Remove auth, fix quota issues with retries, and update agent graph

agentic_rag_graph.py

@@ -0,0 +1,160 @@
+from typing import TypedDict, List, Optional
+import google.generativeai as genai
+from langgraph.graph import StateGraph, END
+
+from rag_store import search_knowledge
+from eval_logger import log_eval
+
+MODEL_NAME = "gemini-2.5-flash"
+
+
+# ===============================
+# STATE
+# ===============================
+class AgentState(TypedDict):
+    query: str
+    decision: str
+    retrieved_chunks: List[dict]
+    answer: Optional[str]
+    confidence: float
+    answer_known: bool
+
+
+# ===============================
+# DECISION NODE
+# ===============================
+def agent_decision_node(state: AgentState) -> AgentState:
+    q = state["query"].lower()
+
+    rag_keywords = [
+        "summarize", "summary", "fee", "fees", "refund",
+        "tuition", "document", "policy", "offer", "scholarship"
+    ]
+
+    decision = "use_rag" if any(k in q for k in rag_keywords) else "no_rag"
+
+    return {**state, "decision": decision}
+
+
+# ===============================
+# RETRIEVAL NODE (TOOL)
+# ===============================
+def retrieve_node(state: AgentState) -> AgentState:
+    chunks = search_knowledge(state["query"])
+    return {**state, "retrieved_chunks": chunks}
+
+
+# ===============================
+# ANSWER WITH RAG
+# ===============================
+def answer_with_rag_node(state: AgentState) -> AgentState:
+    if not state["retrieved_chunks"]:
+        return no_answer_node(state)
+
+    context = "\n\n".join(c["text"] for c in state["retrieved_chunks"])
+
+    prompt = f"""
+Answer using ONLY the context below.
+If the answer is not present, say "I don't know".
+
+Context:
+{context}
+
+Question:
+{state["query"]}
+"""
+
+    model = genai.GenerativeModel(MODEL_NAME)
+    resp = model.generate_content(prompt)
+    answer_text = resp.text
+
+    confidence = min(1.0, len(state["retrieved_chunks"]) / 5)
+    answer_known = "i don't know" not in answer_text.lower()
+
+    log_eval(
+        query=state["query"],
+        retrieved_count=len(state["retrieved_chunks"]),
+        confidence=confidence,
+        answer_known=answer_known
+    )
+
+    return {
+        **state,
+        "answer": answer_text,
+        "confidence": confidence,
+        "answer_known": answer_known
+    }
+
+
+# ===============================
+# ANSWER WITHOUT RAG
+# ===============================
+def answer_direct_node(state: AgentState) -> AgentState:
+    prompt = f"Answer the following question concisely:\n\n{state['query']}"
+
+    model = genai.GenerativeModel(MODEL_NAME)
+    resp = model.generate_content(prompt)
+
+    log_eval(
+        query=state["query"],
+        retrieved_count=0,
+        confidence=0.3,
+        answer_known=True
+    )
+
+    return {
+        **state,
+        "answer": resp.text,
+        "confidence": 0.3,
+        "answer_known": True
+    }
+
+
+# ===============================
+# NO ANSWER
+# ===============================
+def no_answer_node(state: AgentState) -> AgentState:
+    log_eval(
+        query=state["query"],
+        retrieved_count=0,
+        confidence=0.0,
+        answer_known=False
+    )
+
+    return {
+        **state,
+        "answer": "I don't know based on the provided documents.",
+        "confidence": 0.0,
+        "answer_known": False
+    }
+
+
+# ===============================
+# GRAPH BUILDER
+# ===============================
+def build_agentic_rag_graph():
+    graph = StateGraph(AgentState)
+
+    graph.add_node("decide", agent_decision_node)
+    graph.add_node("retrieve", retrieve_node)
+    graph.add_node("answer_rag", answer_with_rag_node)
+    graph.add_node("answer_direct", answer_direct_node)
+    graph.add_node("no_answer", no_answer_node)
+
+    graph.set_entry_point("decide")
+
+    graph.add_conditional_edges(
+        "decide",
+        lambda s: s["decision"],
+        {
+            "use_rag": "retrieve",
+            "no_rag": "answer_direct"
+        }
+    )
+
+    graph.add_edge("retrieve", "answer_rag")
+    graph.add_edge("answer_rag", END)
+    graph.add_edge("answer_direct", END)
+    graph.add_edge("no_answer", END)
+
+    return graph.compile()

frontend/analytics.html

@@ -2,251 +2,252 @@
 <html lang="en">
 
 <head>
-    <meta charset="UTF-8" />
-    <title>Analytics - Gemini RAG</title>
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap" rel="stylesheet">
-
-    <style>
-        :root {
-            --bg: radial-gradient(1200px 600px at top, #e0e7ff 0%, #f8fafc 60%);
-            --card: rgba(255, 255, 255, 0.9);
-            --border: rgba(15, 23, 42, 0.08);
-            --primary: #4f46e5;
-            --secondary: #0ea5e9;
-            --text: #0f172a;
-            --muted: #64748b;
-            --success: #16a34a;
-            --error: #dc2626;
-        }
-
-        [data-theme="dark"] {
-            --bg: radial-gradient(1200px 600px at top, #1e1b4b 0%, #0f172a 60%);
-            --card: rgba(30, 41, 59, 0.9);
-            --border: rgba(148, 163, 184, 0.1);
-            --primary: #818cf8;
-            --secondary: #38bdf8;
-            --text: #f1f5f9;
-            --muted: #94a3b8;
-            --success: #4ade80;
-            --error: #f87171;
-        }
-
-        * {
-            box-sizing: border-box;
-            font-family: Inter, sans-serif;
-        }
-
-        body {
-            margin: 0;
-            min-height: 100vh;
-            background: var(--bg);
-            padding: 40px 16px;
-            color: var(--text);
-            transition: background 0.3s ease, color 0.3s ease;
-        }
-
-        .container {
-            max-width: 1200px;
-            margin: 0 auto;
-        }
-
-        .header {
-            display: flex;
-            justify-content: space-between;
-            align-items: center;
-            margin-bottom: 32px;
-        }
-
-        h1 {
-            font-size: 2.2rem;
-            margin: 0;
-            font-weight: 700;
-            background: linear-gradient(135deg, #4f46e5, #06b6d4);
-            background-clip: text;
-            -webkit-background-clip: text;
-            -webkit-text-fill-color: transparent;
-        }
-
-        .back-btn {
-            padding: 10px 20px;
-            background: var(--primary);
-            color: white;
-            text-decoration: none;
-            border-radius: 12px;
-            font-weight: 600;
-            transition: transform 0.2s ease;
-        }
-
-        .back-btn:hover {
-            transform: translateY(-2px);
-        }
-
-        .stats-grid {
-            display: grid;
-            grid-template-columns: repeat(auto-fit, minmax(240px, 1fr));
-            gap: 20px;
-            margin-bottom: 32px;
-        }
-
-        .stat-card {
-            background: var(--card);
-            backdrop-filter: blur(16px);
-            border-radius: 18px;
-            padding: 24px;
-            border: 1px solid var(--border);
-            box-shadow: 0 4px 12px rgba(0, 0, 0, 0.05);
-        }
-
-        .stat-label {
-            font-size: 0.85rem;
-            color: var(--muted);
-            margin-bottom: 8px;
-            text-transform: uppercase;
-            letter-spacing: 0.5px;
-        }
-
-        .stat-value {
-            font-size: 2.5rem;
-            font-weight: 700;
-            color: var(--primary);
-        }
-
-        .card {
-            background: var(--card);
-            backdrop-filter: blur(16px);
-            border-radius: 18px;
-            padding: 28px;
-            border: 1px solid var(--border);
-            box-shadow: 0 4px 12px rgba(0, 0, 0, 0.05);
-            margin-bottom: 24px;
-        }
-
-        .card h2 {
-            margin-top: 0;
-            margin-bottom: 20px;
-            font-size: 1.3rem;
-        }
-
-        table {
-            width: 100%;
-            border-collapse: collapse;
-        }
-
-        th,
-        td {
-            text-align: left;
-            padding: 12px;
-            border-bottom: 1px solid var(--border);
-        }
-
-        th {
-            font-weight: 600;
-            color: var(--muted);
-            font-size: 0.85rem;
-            text-transform: uppercase;
-            letter-spacing: 0.5px;
-        }
-
-        .badge {
-            display: inline-block;
-            padding: 4px 10px;
-            border-radius: 12px;
-            font-size: 0.75rem;
-            font-weight: 600;
-        }
-
-        .badge-success {
-            background: #dcfce7;
-            color: #166534;
-        }
-
-        .badge-error {
-            background: #fee2e2;
-            color: #991b1b;
-        }
-
-        .theme-toggle {
-            position: fixed;
-            top: 20px;
-            right: 20px;
-            background: var(--card);
-            border: 1px solid var(--border);
-            border-radius: 12px;
-            padding: 10px;
-            cursor: pointer;
-            font-size: 1.4rem;
-            transition: transform 0.2s ease;
-            box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1);
-        }
-
-        .theme-toggle:hover {
-            transform: scale(1.1);
-        }
-
-        .empty-state {
-            text-align: center;
-            padding: 60px 20px;
-            color: var(--muted);
-        }
-    </style>
+  <meta charset="UTF-8" />
+  <title>Analytics - Gemini RAG</title>
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap" rel="stylesheet">
+
+  <style>
+    :root {
+      --bg: radial-gradient(1200px 600px at top, #e0e7ff 0%, #f8fafc 60%);
+      --card: rgba(255, 255, 255, 0.9);
+      --border: rgba(15, 23, 42, 0.08);
+      --primary: #4f46e5;
+      --secondary: #0ea5e9;
+      --text: #0f172a;
+      --muted: #64748b;
+      --success: #16a34a;
+      --error: #dc2626;
+    }
+
+    [data-theme="dark"] {
+      --bg: radial-gradient(1200px 600px at top, #1e1b4b 0%, #0f172a 60%);
+      --card: rgba(30, 41, 59, 0.9);
+      --border: rgba(148, 163, 184, 0.1);
+      --primary: #818cf8;
+      --secondary: #38bdf8;
+      --text: #f1f5f9;
+      --muted: #94a3b8;
+      --success: #4ade80;
+      --error: #f87171;
+    }
+
+    * {
+      box-sizing: border-box;
+      font-family: Inter, sans-serif;
+    }
+
+    body {
+      margin: 0;
+      min-height: 100vh;
+      background: var(--bg);
+      padding: 40px 16px;
+      color: var(--text);
+      transition: background 0.3s ease, color 0.3s ease;
+    }
+
+    .container {
+      max-width: 1200px;
+      margin: 0 auto;
+    }
+
+    .header {
+      display: flex;
+      justify-content: space-between;
+      align-items: center;
+      margin-bottom: 32px;
+    }
+
+    h1 {
+      font-size: 2.2rem;
+      margin: 0;
+      font-weight: 700;
+      background: linear-gradient(135deg, #4f46e5, #06b6d4);
+      background-clip: text;
+      -webkit-background-clip: text;
+      -webkit-text-fill-color: transparent;
+    }
+
+    .back-btn {
+      padding: 10px 20px;
+      background: var(--primary);
+      color: white;
+      text-decoration: none;
+      border-radius: 12px;
+      font-weight: 600;
+      transition: transform 0.2s ease;
+    }
+
+    .back-btn:hover {
+      transform: translateY(-2px);
+    }
+
+    .stats-grid {
+      display: grid;
+      grid-template-columns: repeat(auto-fit, minmax(240px, 1fr));
+      gap: 20px;
+      margin-bottom: 32px;
+    }
+
+    .stat-card {
+      background: var(--card);
+      backdrop-filter: blur(16px);
+      border-radius: 18px;
+      padding: 24px;
+      border: 1px solid var(--border);
+      box-shadow: 0 4px 12px rgba(0, 0, 0, 0.05);
+    }
+
+    .stat-label {
+      font-size: 0.85rem;
+      color: var(--muted);
+      margin-bottom: 8px;
+      text-transform: uppercase;
+      letter-spacing: 0.5px;
+    }
+
+    .stat-value {
+      font-size: 2.5rem;
+      font-weight: 700;
+      color: var(--primary);
+    }
+
+    .card {
+      background: var(--card);
+      backdrop-filter: blur(16px);
+      border-radius: 18px;
+      padding: 28px;
+      border: 1px solid var(--border);
+      box-shadow: 0 4px 12px rgba(0, 0, 0, 0.05);
+      margin-bottom: 24px;
+    }
+
+    .card h2 {
+      margin-top: 0;
+      margin-bottom: 20px;
+      font-size: 1.3rem;
+    }
+
+    table {
+      width: 100%;
+      border-collapse: collapse;
+    }
+
+    th,
+    td {
+      text-align: left;
+      padding: 12px;
+      border-bottom: 1px solid var(--border);
+    }
+
+    th {
+      font-weight: 600;
+      color: var(--muted);
+      font-size: 0.85rem;
+      text-transform: uppercase;
+      letter-spacing: 0.5px;
+    }
+
+    .badge {
+      display: inline-block;
+      padding: 4px 10px;
+      border-radius: 12px;
+      font-size: 0.75rem;
+      font-weight: 600;
+    }
+
+    .badge-success {
+      background: #dcfce7;
+      color: #166534;
+    }
+
+    .badge-error {
+      background: #fee2e2;
+      color: #991b1b;
+    }
+
+    .theme-toggle {
+      position: fixed;
+      top: 20px;
+      right: 20px;
+      background: var(--card);
+      border: 1px solid var(--border);
+      border-radius: 12px;
+      padding: 10px;
+      cursor: pointer;
+      font-size: 1.4rem;
+      transition: transform 0.2s ease;
+      box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1);
+    }
+
+    .theme-toggle:hover {
+      transform: scale(1.1);
+    }
+
+    .empty-state {
+      text-align: center;
+      padding: 60px 20px;
+      color: var(--muted);
+    }
+  </style>
 </head>
 
 <body>
-    <button class="theme-toggle" onclick="toggleTheme()" title="Toggle dark mode">🌙</button>
+  <button class="theme-toggle" onclick="toggleTheme()" title="Toggle dark mode">🌙</button>
 
-    <div class="container">
-        <div class="header">
-            <h1>📊 Analytics Dashboard</h1>
-            <a href="/" class="back-btn">← Back to RAG</a>
-        </div>
-
-        <div id="stats-container">
-            <div class="empty-state">
-                <h2>Loading analytics...</h2>
-            </div>
-        </div>
+  <div class="container">
+    <div class="header">
+      <h1>📊 Analytics Dashboard</h1>
+      <a href="/" class="back-btn">← Back to RAG</a>
     </div>
 
-    <script>
-        // ===== THEME TOGGLE =====
-        function toggleTheme() {
-            const html = document.documentElement;
-            const currentTheme = html.getAttribute('data-theme');
-            const newTheme = currentTheme === 'dark' ? 'light' : 'dark';
-
-            html.setAttribute('data-theme', newTheme);
-            localStorage.setItem('theme', newTheme);
-
-            const btn = document.querySelector('.theme-toggle');
-            btn.textContent = newTheme === 'dark' ? '☀️' : '🌙';
-        }
-
-        // Load saved theme
-        (function () {
-            const savedTheme = localStorage.getItem('theme') || 'light';
-            document.documentElement.setAttribute('data-theme', savedTheme);
-            const btn = document.querySelector('.theme-toggle');
-            if (btn) btn.textContent = savedTheme === 'dark' ? '☀️' : '🌙';
-        })();
-
-        // ===== LOAD ANALYTICS =====
-        async function loadAnalytics() {
-            try {
-                const res = await fetch('/analytics');
-                const data = await res.json();
-
-                if (data.total_queries === 0) {
-                    document.getElementById('stats-container').innerHTML = `
+    <div id="stats-container">
+      <div class="empty-state">
+        <h2>Loading analytics...</h2>
+      </div>
+    </div>
+  </div>
+
+  <script>
+    // ===== THEME TOGGLE =====
+    function toggleTheme() {
+      const html = document.documentElement;
+      const currentTheme = html.getAttribute('data-theme');
+      const newTheme = currentTheme === 'dark' ? 'light' : 'dark';
+
+      html.setAttribute('data-theme', newTheme);
+      localStorage.setItem('theme', newTheme);
+
+      const btn = document.querySelector('.theme-toggle');
+      btn.textContent = newTheme === 'dark' ? '☀️' : '🌙';
+    }
+
+    // Load saved theme
+    (function () {
+      const savedTheme = localStorage.getItem('theme') || 'light';
+      document.documentElement.setAttribute('data-theme', savedTheme);
+      const btn = document.querySelector('.theme-toggle');
+      if (btn) btn.textContent = savedTheme === 'dark' ? '☀️' : '🌙';
+    })();
+
+    // ===== LOAD ANALYTICS =====
+    async function loadAnalytics() {
+      try {
+        const res = await fetch('/analytics');
+
+        const data = await res.json();
+
+        if (data.total_queries === 0) {
+          document.getElementById('stats-container').innerHTML = `
             <div class="empty-state">
               <h2>No data yet</h2>
               <p>Start asking questions to see analytics!</p>
             </div>
           `;
-                    return;
-                }
+          return;
+        }
 
-                const html = `
+        const html = `
           <div class="stats-grid">
             <div class="stat-card">
               <div class="stat-label">Total Queries</div>
@@ -315,20 +316,20 @@
           ` : ''}
         `;
 
-                document.getElementById('stats-container').innerHTML = html;
-            } catch (e) {
-                document.getElementById('stats-container').innerHTML = `
+        document.getElementById('stats-container').innerHTML = html;
+      } catch (e) {
+        document.getElementById('stats-container').innerHTML = `
           <div class="empty-state">
             <h2>Error loading analytics</h2>
             <p>${e.message}</p>
           </div>
         `;
-            }
-        }
+      }
+    }
 
-        // Load on page load
-        loadAnalytics();
-    </script>
+    // Load on page load
+    loadAnalytics();
+  </script>
 </body>
 
 </html>

frontend/index.html

@@ -43,21 +43,83 @@
       min-height: 100vh;
       background: var(--bg);
       display: flex;
-      justify-content: center;
-      padding: 40px 16px;
       color: var(--text);
-      transition: background 0.3s ease, color 0.3s ease;
+    }
+
+    /* Layout */
+    .app_layout {
+      display: grid;
+      grid-template-columns: 260px 1fr;
+      width: 100%;
+      height: 100vh;
+    }
+
+    /* Sidebar */
+    .sidebar {
+      background: rgba(255, 255, 255, 0.5);
+      /* Glass-ish */
+      backdrop-filter: blur(12px);
+      border-right: 1px solid var(--border);
+      padding: 24px;
+      display: flex;
+      flex-direction: column;
+      height: 100%;
+      overflow-y: auto;
+    }
+
+    .main-content {
+      padding: 40px;
+      overflow-y: auto;
+      display: flex;
+      justify-content: center;
     }
 
     .container {
       width: 100%;
       max-width: 800px;
+      /* background: var(--card); Removed container bg for cleaner look in main area */
+      /* border-radius: 24px; */
+      /* padding: 36px; */
+      /* border: 1px solid var(--border); */
+      /* box-shadow: 0 40px 120px rgba(15, 23, 42, .15); */
+    }
+
+    .history-item {
+      padding: 10px 14px;
+      margin-bottom: 8px;
       background: var(--card);
-      backdrop-filter: blur(16px);
-      border-radius: 24px;
-      padding: 36px;
       border: 1px solid var(--border);
-      box-shadow: 0 40px 120px rgba(15, 23, 42, .15);
+      border-radius: 10px;
+      cursor: pointer;
+      font-size: 0.9rem;
+      transition: all 0.2s;
+      white-space: nowrap;
+      overflow: hidden;
+      text-overflow: ellipsis;
+    }
+
+    .history-item:hover {
+      background: var(--primary);
+      color: white;
+    }
+
+    .sidebar-header {
+      margin-bottom: 20px;
+      display: flex;
+      justify-content: space-between;
+      align-items: center;
+    }
+
+    .new-chat-btn {
+      width: 100%;
+      padding: 10px;
+      margin-bottom: 20px;
+      background: var(--primary);
+      color: white;
+      border: none;
+      border-radius: 10px;
+      cursor: pointer;
+      font-weight: 600;
     }
 
     h1 {
@@ -73,44 +135,28 @@
     .subtitle {
       margin-top: 8px;
       color: var(--muted);
-      font-size: 1rem;
     }
 
     .card {
       margin-top: 28px;
-      background: var(--card);
-      border-radius: 18px;
       padding: 24px;
+      border-radius: 18px;
       border: 1px solid var(--border);
+      background: var(--card);
     }
 
-    .card h3 {
-      margin-top: 0;
-      margin-bottom: 16px;
-      font-size: 1.1rem;
-    }
-
-    input[type="file"],
-    textarea {
+    textarea,
+    input[type="file"] {
       width: 100%;
       padding: 14px;
       border-radius: 14px;
       border: 1px solid var(--border);
-      font-size: 0.95rem;
       background: var(--card);
       color: var(--text);
     }
 
     textarea {
       min-height: 100px;
-      resize: vertical;
-    }
-
-    .row {
-      display: flex;
-      gap: 12px;
-      margin-top: 12px;
-      flex-wrap: wrap;
     }
 
     button {
@@ -121,43 +167,30 @@
       color: white;
       font-weight: 600;
       cursor: pointer;
-      transition: all .2s ease;
     }
 
     button.secondary {
       background: var(--secondary);
     }
 
-    button:disabled {
-      opacity: .5;
-      cursor: not-allowed;
-    }
-
-    button:hover:not(:disabled) {
-      transform: translateY(-1px);
-      box-shadow: 0 4px 12px rgba(79, 70, 229, .2);
-    }
-
-    .status {
-      margin-top: 10px;
-      font-size: .9rem;
-      color: var(--muted);
+    .row {
+      display: flex;
+      gap: 12px;
+      margin-top: 12px;
     }
 
     .answer {
       margin-top: 24px;
       padding: 22px;
       border-radius: 16px;
-      background: var(--card);
       border: 1px solid var(--border);
+      background: var(--card);
       line-height: 1.6;
-      box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.05);
-      color: var(--text);
     }
 
     .confidence-badge {
-      display: inline-block;
       margin-top: 12px;
+      display: inline-block;
       padding: 4px 12px;
       border-radius: 20px;
       background: #dcfce7;
@@ -167,184 +200,233 @@
     }
 
     .citations {
-      margin-top: 16px;
-      font-size: .85rem;
+      margin-top: 14px;
+      font-size: 0.85rem;
       color: var(--muted);
-      border-top: 1px solid var(--border);
-      padding-top: 12px;
-    }
-
-    .citations ul {
-      margin: 6px 0 0;
-      padding-left: 20px;
-    }
-
-    .loader {
-      font-weight: 600;
-      color: var(--primary);
-      animation: pulse 1.2s infinite;
-    }
-
-    @keyframes pulse {
-      0% {
-        opacity: .4
-      }
-
-      50% {
-        opacity: 1
-      }
-
-      100% {
-        opacity: .4
-      }
-    }
-
-    .theme-toggle {
-      position: fixed;
-      top: 20px;
-      right: 20px;
-      background: var(--card);
-      border: 1px solid var(--border);
-      border-radius: 12px;
-      padding: 10px;
-      cursor: pointer;
-      font-size: 1.4rem;
-      transition: transform 0.2s ease;
-      box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1);
-    }
-
-    .theme-toggle:hover {
-      transform: scale(1.1);
     }
   </style>
 </head>
 
 <body>
-  <button class="theme-toggle" onclick="toggleTheme()" title="Toggle dark mode">🌙</button>
-  <div class="container">
-    <h1>Gemini RAG Assistant</h1>
-    <div class="subtitle">Upload documents · Ask questions · Get grounded answers · <a href="/frontend/analytics.html"
-        style="color: var(--primary); text-decoration: none; font-weight: 600;">📊 Analytics</a></div>
-
-    <div class="card">
-      <h3>1. Upload Knowledge</h3>
-      <input type="file" id="files" multiple accept=".pdf,.txt" />
-      <div class="row">
-        <button id="uploadBtn" onclick="upload()">Upload & Index Files</button>
+  <div class="app_layout">
+    <div class="sidebar">
+      <div class="sidebar-header">
+        <h2 style="font-size: 1.2rem; margin:0;">History</h2>
+        <button onclick="clearHistory()"
+          style="background:none; border:none; color:var(--error); cursor:pointer; padding:0; font-size:0.8rem; width:auto; text-decoration:underline; margin:0;">Clear</button>
       </div>
-      <div id="uploadStatus" class="status"></div>
+      <button class="new-chat-btn" onclick="newChat()">+ New Chat</button>
+      <div id="historyList"></div>
     </div>
-
-    <div class="card">
-      <h3>2. Ask or Summarize</h3>
-      <textarea id="question" placeholder="E.g., 'What are the main risks?' or 'Summarize the document'"></textarea>
-      <div class="row">
-        <button id="askBtn" onclick="ask()">Ask Question</button>
-        <button class="secondary" id="sumBtn" onclick="summarize()">Generate Summary</button>
+    <div class="main-content">
+      <div class="container">
+        <h1>Gemini RAG Assistant</h1>
+        <div class="subtitle">
+          Upload documents · Ask questions · Get grounded answers ·
+          <a href="/frontend/analytics.html">📊 Analytics</a>
+        </div>
+
+        <div class="card">
+          <h3>Upload Knowledge</h3>
+          <input type="file" id="files" multiple />
+          <div class="row">
+            <button onclick="upload()">Upload</button>
+          </div>
+
+          <!-- Progress Bar Container -->
+          <div id="progressContainer" style="display: none; margin-top: 16px;">
+            <div style="background: var(--border); border-radius: 8px; overflow: hidden; height: 10px;">
+              <div id="progressBar"
+                style="width: 0%; height: 100%; background: var(--primary); transition: width 0.2s;">
+              </div>
+            </div>
+            <div id="progressText"
+              style="margin-top: 6px; font-size: 0.85rem; color: var(--muted); text-align: center;">0%
+            </div>
+          </div>
+
+          <div id="uploadStatus" style="margin-top: 12px; font-weight: 500;"></div>
+        </div>
+
+        <div class="card">
+          <h3>Ask or Summarize</h3>
+          <textarea id="question"></textarea>
+          <div class="row">
+            <button onclick="ask()">Ask</button>
+            <button class="secondary" onclick="summarize()">Summarize</button>
+          </div>
+        </div>
+
+        <div id="answerBox" class="answer" style="display:none;"></div>
       </div>
     </div>
-
-    <div id="answerBox" class="answer" style="display:none;"></div>
-
   </div>
 
   <script>
-    // ===== THEME TOGGLE =====
-    function toggleTheme() {
-      const html = document.documentElement;
-      const currentTheme = html.getAttribute('data-theme');
-      const newTheme = currentTheme === 'dark' ? 'light' : 'dark';
-
-      html.setAttribute('data-theme', newTheme);
-      localStorage.setItem('theme', newTheme);
-
-      // Update button icon
-      const btn = document.querySelector('.theme-toggle');
-      btn.textContent = newTheme === 'dark' ? '☀️' : '🌙';
-    }
-
-    // Load saved theme on page load
-    (function () {
-      const savedTheme = localStorage.getItem('theme') || 'light';
-      document.documentElement.setAttribute('data-theme', savedTheme);
-      const btn = document.querySelector('.theme-toggle');
-      if (btn) btn.textContent = savedTheme === 'dark' ? '☀️' : '🌙';
-    })();
-
-    // ===== APP LOGIC =====
-    let busy = false;
-
-    function setBusy(state) {
-      busy = state;
-      document.getElementById("askBtn").disabled = state;
-      document.getElementById("sumBtn").disabled = state;
-      document.getElementById("uploadBtn").disabled = state;
-    }
+    const MAX_FILE_SIZE = 50 * 1024 * 1024; // 50MB
 
     async function upload() {
-      const files = document.getElementById("files").files;
+      const fileInput = document.getElementById("files");
+      const files = fileInput.files;
+      const statusDiv = document.getElementById("uploadStatus");
+      const progressContainer = document.getElementById("progressContainer");
+      const progressBar = document.getElementById("progressBar");
+      const progressText = document.getElementById("progressText");
+
       if (!files.length) {
-        alert("Please select files first.");
+        alert("Please select at least one file.");
         return;
       }
 
-      setBusy(true);
-      const statusDiv = document.getElementById("uploadStatus");
-      statusDiv.innerText = "Indexing documents... this may take a moment.";
+      // 1. Client-side Validation
+      for (let f of files) {
+        if (f.size > MAX_FILE_SIZE) {
+          alert(`File "${f.name}" is too large (>${MAX_FILE_SIZE / 1024 / 1024}MB). Max allowed is 50MB.`);
+          return;
+        }
+      }
+
+      // Reset UI
+      statusDiv.innerText = "";
+      statusDiv.style.color = "var(--text)";
+      progressContainer.style.display = "block";
+      progressBar.style.width = "0%";
+      progressText.innerText = "0%";
 
       const fd = new FormData();
       for (let f of files) fd.append("files", f);
 
-      try {
-        const res = await fetch("/upload", { method: "POST", body: fd });
-        if (!res.ok) throw new Error("Upload failed");
-        const data = await res.json();
-        statusDiv.innerText = data.message || "Done ✅";
-      } catch (e) {
-        statusDiv.innerText = "Error uploading files.";
+      // 2. Upload via XMLHttpRequest for progress events
+      const xhr = new XMLHttpRequest();
+
+      xhr.upload.addEventListener("progress", (event) => {
+        if (event.lengthComputable) {
+          const percent = Math.round((event.loaded / event.total) * 100);
+          progressBar.style.width = percent + "%";
+          progressText.innerText = percent + "%";
+        }
+      });
+
+
+
+      xhr.addEventListener("load", () => {
+        if (xhr.status >= 200 && xhr.status < 300) {
+          try {
+            const data = JSON.parse(xhr.responseText);
+            statusDiv.innerText = data.message || "Upload complete!";
+            statusDiv.style.color = "var(--success)";
+            progressBar.style.width = "100%";
+            progressText.innerText = "Processing complete";
+            fileInput.value = ""; // Clear input
+          } catch (e) {
+            statusDiv.innerText = "Error parsing server response.";
+            statusDiv.style.color = "var(--error)";
+          }
+        } else {
+          statusDiv.innerText = `Upload failed: ${xhr.statusText || xhr.status}`;
+          statusDiv.style.color = "var(--error)";
+        }
+      });
+
+      xhr.addEventListener("error", () => {
+        statusDiv.innerText = "Network error during upload.";
+        statusDiv.style.color = "var(--error)";
+      });
+
+      xhr.open("POST", "/upload");
+      xhr.send(fd);
+    }
+
+    // --- HISTORY LOGIC ---
+    function loadHistory() {
+      const list = document.getElementById("historyList");
+      list.innerHTML = "";
+      const history = JSON.parse(localStorage.getItem("rag_history") || "[]");
+
+      history.forEach((item, index) => {
+        const div = document.createElement("div");
+        div.className = "history-item";
+        div.innerText = item.query;
+        div.onclick = () => loadSession(index);
+        list.appendChild(div);
+      });
+    }
+
+    function saveToHistory(query, answerHtml) {
+      const history = JSON.parse(localStorage.getItem("rag_history") || "[]");
+      // Prepend new item
+      history.unshift({ query, answerHtml, timestamp: Date.now() });
+      // Keep max 50
+      if (history.length > 50) history.pop();
+
+      localStorage.setItem("rag_history", JSON.stringify(history));
+      loadHistory();
+    }
+
+    function loadSession(index) {
+      const history = JSON.parse(localStorage.getItem("rag_history") || "[]");
+      const item = history[index];
+      if (!item) return;
+
+      document.getElementById("question").value = item.query;
+      const box = document.getElementById("answerBox");
+      box.style.display = "block";
+      box.innerHTML = item.answerHtml;
+    }
+
+    function newChat() {
+      document.getElementById("question").value = "";
+      document.getElementById("answerBox").style.display = "none";
+      document.getElementById("answerBox").innerHTML = "";
+    }
+
+    function clearHistory() {
+      if (confirm("Clear all history?")) {
+        localStorage.removeItem("rag_history");
+        loadHistory();
+        newChat();
       }
-      setBusy(false);
     }
 
+    // Init
+    loadHistory();
+
     async function ask() {
       const q = document.getElementById("question").value.trim();
       if (!q) return;
 
-      setBusy(true);
       const box = document.getElementById("answerBox");
       box.style.display = "block";
-      box.innerHTML = "<span class='loader'>Thinking...</span>";
+      box.innerHTML = "Thinking...";
 
-      try {
-        const res = await fetch("/ask", {
-          method: "POST",
-          headers: { "Content-Type": "application/json" },
-          body: JSON.stringify({ prompt: q })
-        });
+      const res = await fetch("/ask", {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ prompt: q })
+      });
 
-        const data = await res.json();
+      const data = await res.json();
 
-        let html = `<div><strong>Answer:</strong><br>${data.answer.replace(/\n/g, '<br>')}</div>`;
+      let html = `<strong>Answer:</strong><br>${data.answer.replace(/\n/g, "<br>")}`;
 
-        if (data.confidence > 0) {
-          html += `<div class="confidence-badge">Confidence: ${(data.confidence * 100).toFixed(0)}%</div>`;
-        }
+      if (data.confidence > 0) {
+        let label = "Low";
+        if (data.confidence >= 0.7) label = "High";
+        else if (data.confidence >= 0.5) label = "Medium";
 
-        if (data.citations && data.citations.length > 0) {
-          html += `<div class="citations"><strong>Sources:</strong><ul>`;
-          data.citations.forEach(c => {
-            html += `<li>${c.source} (Page ${c.page})</li>`;
-          });
-          html += `</ul></div>`;
-        }
-
-        box.innerHTML = html;
+        html += `<div class="confidence-badge">Confidence: ${label} (${Math.round(data.confidence * 100)}%)</div>`;
+      }
 
-      } catch (e) {
-        box.innerText = "⚠️ Error communicating with the server.";
+      if (data.citations?.length) {
+        html += "<div class='citations'><strong>Sources:</strong><ul>";
+        data.citations.forEach(c => {
+          html += `<li>${c.source} (Page ${c.page})</li>`;
+        });
+        html += "</ul></div>";
       }
 
-      setBusy(false);
+      box.innerHTML = html;
+      saveToHistory(q, html); // Save to history
     }
 
     function summarize() {

langgraph_rag.py

@@ -0,0 +1,110 @@
+from typing import TypedDict, List, Optional
+import google.generativeai as genai
+from langgraph.graph import StateGraph, END
+
+from rag_store import search_knowledge
+from eval_logger import log_eval
+
+MODEL_NAME = "gemini-2.5-flash"
+
+
+# ===============================
+# STATE
+# ===============================
+class RAGState(TypedDict):
+    query: str
+    retrieved_chunks: List[dict]
+    answer: Optional[str]
+    confidence: float
+    answer_known: bool
+
+
+# ===============================
+# RETRIEVAL NODE (TOOL)
+# ===============================
+def retrieve_node(state: RAGState) -> RAGState:
+    results = search_knowledge(state["query"])
+    return {
+        **state,
+        "retrieved_chunks": results
+    }
+
+
+# ===============================
+# ANSWER NODE
+# ===============================
+def answer_node(state: RAGState) -> RAGState:
+    if not state["retrieved_chunks"]:
+        return no_answer_node(state)
+
+    context = "\n\n".join(c["text"] for c in state["retrieved_chunks"])
+
+    prompt = f"""
+Answer using ONLY the context below.
+If the answer is not present, say "I don't know".
+
+Context:
+{context}
+
+Question:
+{state["query"]}
+"""
+
+    model = genai.GenerativeModel(MODEL_NAME)
+    resp = model.generate_content(prompt)
+    answer_text = resp.text
+
+    confidence = min(1.0, len(state["retrieved_chunks"]) / 5)
+    answer_known = "i don't know" not in answer_text.lower()
+
+    log_eval(
+        query=state["query"],
+        retrieved_count=len(state["retrieved_chunks"]),
+        confidence=confidence,
+        answer_known=answer_known
+    )
+
+    return {
+        **state,
+        "answer": answer_text,
+        "confidence": confidence,
+        "answer_known": answer_known
+    }
+
+
+# ===============================
+# NO ANSWER NODE
+# ===============================
+def no_answer_node(state: RAGState) -> RAGState:
+    log_eval(
+        query=state["query"],
+        retrieved_count=0,
+        confidence=0.0,
+        answer_known=False
+    )
+
+    return {
+        **state,
+        "answer": "I don't know based on the provided documents.",
+        "confidence": 0.0,
+        "answer_known": False
+    }
+
+
+# ===============================
+# GRAPH BUILDER
+# ===============================
+def build_rag_graph():
+    graph = StateGraph(RAGState)
+
+    graph.add_node("retrieve", retrieve_node)
+    graph.add_node("answer", answer_node)
+    graph.add_node("no_answer", no_answer_node)
+
+    graph.set_entry_point("retrieve")
+
+    graph.add_edge("retrieve", "answer")
+    graph.add_edge("answer", END)
+    graph.add_edge("no_answer", END)
+
+    return graph.compile()

main.py

@@ -8,28 +8,24 @@ from pydantic import BaseModel
 from dotenv import load_dotenv
 import google.generativeai as genai
 
-from rag_store import ingest_documents, search_knowledge, get_all_chunks, clear_database
-from eval_logger import log_eval
+from rag_store import ingest_documents, get_all_chunks, clear_database
 from analytics import get_analytics
+from agentic_rag_v2_graph import build_agentic_rag_v2_graph
 
 # =========================================================
-# ENV + MODEL SETUP
+# ENV + MODEL
 # =========================================================
 load_dotenv()
 genai.configure(api_key=os.getenv("GEMINI_API_KEY"))
 
 MODEL_NAME = "gemini-2.5-flash"
-USE_MOCK = False # Set to False to use real API
-
-# =========================================================
-# FILE UPLOAD LIMITS
-# =========================================================
-MAX_FILE_SIZE = 50 * 1024 * 1024  # 50 MB
+MAX_FILE_SIZE = 50 * 1024 * 1024
+CACHE_TTL = 300
 
 # =========================================================
 # APP
 # =========================================================
-app = FastAPI(title="Gemini RAG FastAPI")
+app = FastAPI(title="Gemini RAG FastAPI (Agentic RAG v2+)")
 
 app.add_middleware(
     CORSMiddleware,
@@ -41,9 +37,25 @@ app.add_middleware(
 app.mount("/frontend", StaticFiles(directory="frontend"), name="frontend")
 
 # =========================================================
-# CACHE (ANTI-429)
+# SECURITY
+# =========================================================
+from fastapi import Request, HTTPException, Depends
+from fastapi.security import APIKeyCookie
+
+ADMIN_PASSWORD = os.getenv("ADMIN_PASSWORD", "secret")
+COOKIE_NAME = "rag_auth"
+
+api_key_cookie = APIKeyCookie(name=COOKIE_NAME, auto_error=False)
+
+async def verify_admin(cookie: str = Depends(api_key_cookie)):
+    if cookie != ADMIN_PASSWORD:
+        raise HTTPException(status_code=401, detail="Unauthorized")
+    return cookie
+
+# =========================================================
+# STATE
 # =========================================================
-CACHE_TTL = 300  # 5 minutes
+agentic_graph = build_agentic_rag_v2_graph()
 answer_cache: dict[str, tuple[float, dict]] = {}
 
 # =========================================================
@@ -52,64 +64,70 @@ answer_cache: dict[str, tuple[float, dict]] = {}
 class PromptRequest(BaseModel):
     prompt: str
 
+class LoginRequest(BaseModel):
+    password: str
+
 # =========================================================
 # ROUTES
 # =========================================================
+@app.post("/login")
+def login(data: LoginRequest):
+    if data.password != ADMIN_PASSWORD:
+        raise HTTPException(status_code=401, detail="Invalid password")
+    
+    response = JSONResponse(content={"message": "Logged in"})
+    response.set_cookie(key=COOKIE_NAME, value=data.password, httponly=True)
+    return response
+
+@app.get("/me")
+def me(user: str = Depends(verify_admin)):
+    return {"status": "authenticated"}
+
 @app.get("/", response_class=HTMLResponse)
 def serve_ui():
     with open("frontend/index.html", "r", encoding="utf-8") as f:
         return f.read()
 
-@app.get("/analytics")
+@app.get("/analytics", dependencies=[Depends(verify_admin)])
 def analytics():
-    """Return analytics data from evaluation logs."""
     return get_analytics()
 
 # ---------------------------------------------------------
 # UPLOAD
 # ---------------------------------------------------------
-@app.post("/upload")
+@app.post("/upload", dependencies=[Depends(verify_admin)])
 async def upload(files: list[UploadFile] = File(...)):
-    # 1. VALIDATION: File Type and Size Check
     for file in files:
         ext = file.filename.split(".")[-1].lower()
         if ext not in ["pdf", "txt"]:
             return JSONResponse(
-                status_code=400, 
-                content={"error": f"Invalid file type: '{file.filename}'. Only .pdf and .txt files are allowed."}
+                status_code=400,
+                content={"error": "Only PDF and TXT files allowed"}
             )
-        
-        # Check file size
-        file.file.seek(0, 2)  # Seek to end
-        file_size = file.file.tell()
-        file.file.seek(0)  # Reset to beginning
-        
-        if file_size > MAX_FILE_SIZE:
-            size_mb = file_size / (1024 * 1024)
-            max_mb = MAX_FILE_SIZE / (1024 * 1024)
+
+        file.file.seek(0, 2)
+        size = file.file.tell()
+        file.file.seek(0)
+
+        if size > MAX_FILE_SIZE:
             return JSONResponse(
                 status_code=413,
-                content={"error": f"File '{file.filename}' is too large ({size_mb:.1f} MB). Maximum size is {max_mb:.0f} MB."}
+                content={"error": "File too large"}
             )
 
-    try:
-        # 2. CLEAR CONTEXT: Start fresh for every upload session
-        clear_database()
-        answer_cache.clear() # <--- CRITICAL: Clear the questions cache too!
-        
-        # 3. INGEST
-        chunks = ingest_documents(files)
-        return {"message": f"Successfully indexed {chunks} chunks. Previous context cleared."}
-    except Exception as e:
-        return JSONResponse(status_code=400, content={"error": str(e)})
+    clear_database()
+    answer_cache.clear()
+    chunks = ingest_documents(files)
+
+    return {"message": f"Indexed {chunks} chunks successfully."}
 
 # ---------------------------------------------------------
-# ASK / SUMMARIZE
+# ASK
 # ---------------------------------------------------------
 @app.post("/ask")
 async def ask(data: PromptRequest):
-    prompt_text = data.prompt.strip()
-    key = prompt_text.lower()
+    query = data.prompt.strip()
+    key = query.lower()
     now = time()
 
     # ---------- CACHE ----------
@@ -118,212 +136,50 @@ async def ask(data: PromptRequest):
         if now - ts < CACHE_TTL:
             return cached
 
-    model = genai.GenerativeModel(MODEL_NAME)
-    is_summary = "summarize" in key or "summary" in key
-
-    # =====================================================
-    # 🟦 SUMMARY MODE (MAP–REDUCE)
-    # =====================================================
-    # Helper for rate-limit aware generation
-    def generate_safe(prompt_content, retries=5):
-        if USE_MOCK:
-            import time as pytime
-            pytime.sleep(1.5) # Simulate latency
-            class MockResp:
-                def __init__(self, text): self.text = text
-                @property
-                def prompt_feedback(self): return None
-            
-            if "Summarize" in str(prompt_content):
-                return MockResp("- This is a mock summary point 1 (API limit reached).\n- This is point 2 demonstrating the UI works.\n- Point 3: The backend logic is sound.")
-            elif "Combine" in str(prompt_content):
-                 return MockResp("Here are the final summarized points (MOCK MODE):\n\n* **System Integrity**: The RAG system is functioning correctly, handling file ingestion and chunking.\n* **Resilience**: Error handling and retry mechanisms are now in place.\n* **Mocking**: We are currently bypassing the live API to verify the frontend pipeline.\n* **Ready**: Once quotas reset, simply set USE_MOCK = False to resume live intelligence.\n* **Success**: The overall architecture is validated.")
-            else:
-                return MockResp("I am functioning in MOCK MODE because the daily API quota is exhausted. I cannot answer specific questions right now, but I confirm the system received your question: " + str(prompt_content)[:50] + "...")
-
-        import time as pytime
-        base_delay = 10
-        for attempt in range(retries + 1):
-            try:
-                # Always small delay to be nice to the API
-                pytime.sleep(2.0) 
-                response = model.generate_content(prompt_content)
-                return response
-            except Exception as e:
-                err_str = str(e)
-                
-                # API Key Issues
-                if "API_KEY" in err_str or "invalid" in err_str.lower() and "key" in err_str.lower():
-                    raise ValueError("Invalid API key. Please check your GEMINI_API_KEY in the .env file.")
-                
-                # Quota Exhausted
-                if "quota" in err_str.lower() or "limit" in err_str.lower():
-                    raise ValueError("API quota exhausted. Please try again later or upgrade your API plan.")
-                
-                # Rate Limiting (429)
-                if "429" in err_str:
-                    if attempt < retries:
-                        wait_time = base_delay * (2 ** attempt)
-                        print(f"DEBUG: 429 Rate limit hit. Retrying in {wait_time}s...")
-                        pytime.sleep(wait_time)
-                        continue
-                    else:
-                        raise ValueError("Rate limit exceeded. Please try again in a few minutes.")
-                
-                # Safety Filters
-                if "safety" in err_str.lower() or "blocked" in err_str.lower():
-                    raise ValueError("Content was blocked by safety filters. Please rephrase your question.")
-                
-                # Generic error
-                raise ValueError(f"LLM API error: {err_str}")
-
-    if is_summary:
+    # ==========================
+    # 🟦 SUMMARY (BYPASS AGENT)
+    # ==========================
+    if "summary" in key or "summarize" in key:
         chunks = get_all_chunks(limit=80)
-        print(f"DEBUG: Found {len(chunks)} chunks for summary.")
-
-        if not chunks:
-            return {
-                "answer": "No documents available to summarize.",
-                "confidence": 0.0,
-                "citations": []
-            }
-
-        # -----------------------------------------------------
-        # REFACTORED: Single-Shot Summary (Avoids Rate Limits)
-        # -----------------------------------------------------
-        all_text = "\n\n".join(c["text"] for c in chunks)
-        print(f"DEBUG: Total text length for summary: {len(all_text)} chars")
-
-        prompt = f"""
-Summarize the following content in 5 clear, high-level bullet points.
+        context = "\n\n".join(c["text"] for c in chunks)
 
-Content:
-{all_text}
-"""
-        try:
-            # Single call with retry logic
-            resp = generate_safe(prompt)
-            print("DEBUG: Summary generation successful.")
-            
-            final_text = "Analysis complete."
-            try:
-                final_text = resp.text
-            except ValueError:
-                final_text = "Summary generation was blocked by safety filters."
-
-            response = {
-                "answer": final_text,
-                "confidence": 0.95,
-                "citations": list({
-                    (c["metadata"]["source"], c["metadata"]["page"]): c["metadata"]
-                    for c in chunks
-                }.values())
-            }
-
-            answer_cache[key] = (now, response)
-            return response
-            
-        except ValueError as e:
-            # User-friendly error from generate_safe
-            print(f"Summary failed: {e}")
-            return JSONResponse(status_code=200, content={
-                "answer": str(e),
-                "confidence": 0.0,
-                "citations": []
-            })
-        except Exception as e:
-            print(f"Summary failed: {e}")
-            return JSONResponse(status_code=500, content={
-                "answer": f"An unexpected error occurred: {str(e)}",
-                "confidence": 0.0,
-                "citations": []
-            })
-
-    # =====================================================
-    # 🟩 Q&A MODE (RAG)
-    # =====================================================
-    results = search_knowledge(prompt_text)
+        model = genai.GenerativeModel(MODEL_NAME)
+        resp = model.generate_content(
+            f"Summarize the following content clearly:\n\n{context}"
+        )
 
-    if not results:
         response = {
-            "answer": "I don't know based on the provided documents.",
-            "confidence": 0.0,
+            "answer": resp.text,
+            "confidence": 0.95,
             "citations": []
         }
 
-        log_eval(
-            query=prompt_text,
-            retrieved_count=0,
-            confidence=0.0,
-            answer_known=False
-        )
-
         answer_cache[key] = (now, response)
         return response
 
-    context = "\n\n".join(r["text"] for r in results)
-    
-    # DEBUG: Log the context to see what the model is reading
-    print("DEBUG: ------------------- RAG CONTEXT -------------------")
-    print(context[:2000] + ("..." if len(context) > 2000 else ""))
-    print("DEBUG: ---------------------------------------------------")
-
-    prompt = f"""
-Answer using ONLY the context below.
-If the answer is not present, say "I don't know".
-
-Context:
-{context}
-
-Question:
-{prompt_text}
-"""
-    llm = None
-    answer_text = ""
-    
-    try:
-        llm = model.generate_content(prompt)
-        answer_text = llm.text
-    except ValueError as e:
-        # User-friendly error from API
-        response = {
-            "answer": str(e),
-            "confidence": 0.0,
-            "citations": []
-        }
-        answer_cache[key] = (now, response)
-        return response
-    except Exception as e:
-        # Unexpected error
-        response = {
-            "answer": f"An unexpected error occurred: {str(e)}",
-            "confidence": 0.0,
-            "citations": []
-        }
-        return JSONResponse(status_code=500, content=response)
-    
-    # Fix Fake Confidence: If the model says "I don't know", confidence should be 0.
-    confidence = round(min(1.0, len(results) / 5), 2)
-    if "i don't know" in answer_text.lower():
-        confidence = 0.0
+    # ==========================
+    # 🟩 AGENTIC RAG (LLM + EVALUATION)
+    # ==========================
+    result = agentic_graph.invoke({
+        "query": query,
+        "refined_query": "",
+        "decision": "",
+        "retrieved_chunks": [],
+        "retrieval_quality": "",
+        "retries": 0,
+        "answer": None,
+        "confidence": 0.0,
+        "answer_known": False
+    })
 
     response = {
-        "answer": answer_text,
-        "confidence": confidence,
+        "answer": result["answer"],
+        "confidence": result["confidence"],
         "citations": list({
-            (r["metadata"]["source"], r["metadata"]["page"]): r["metadata"]
-            for r in results
+            (c["metadata"]["source"], c["metadata"]["page"]): c["metadata"]
+            for c in result.get("retrieved_chunks", [])
         }.values())
     }
 
-    answer_known = "i don't know" not in answer_text.lower()
-
-    log_eval(
-        query=prompt_text,
-        retrieved_count=len(results),
-        confidence=confidence,
-        answer_known=answer_known
-    )
-
     answer_cache[key] = (now, response)
     return response

migrate_bm25.py

@@ -0,0 +1,23 @@
+from rag_store import load_db, save_db, documents, bm25
+from rank_bm25 import BM25Okapi
+import pickle
+
+print("Loading DB...")
+load_db()
+
+if not documents:
+    print("No documents found. Nothing to do.")
+else:
+    print(f"Found {len(documents)} documents.")
+    print("Building BM25 index...")
+    tokenized_corpus = [doc.split(" ") for doc in documents]
+    
+    # We need to update the global variable in rag_store, but since we imported 'bm25' (by value? no, python imports names), 
+    # we need to actually set it in the module or just use the save logic.
+    # Actually, simplistic way:
+    import rag_store
+    rag_store.bm25 = BM25Okapi(tokenized_corpus)
+    
+    print("Saving DB with BM25...")
+    rag_store.save_db()
+    print("Done!")

rag_eval_logs.jsonl

@@ -21,3 +21,43 @@
 {"timestamp": 1767776180.9555495, "query": "what are the visa conditions?", "retrieved_count": 5, "confidence": 0.0, "answer_known": false}
 {"timestamp": 1767776250.0441537, "query": "tell me about program information?", "retrieved_count": 5, "confidence": 1.0, "answer_known": true}
 {"timestamp": 1767777566.4153016, "query": "what was the role ?", "retrieved_count": 3, "confidence": 0.6, "answer_known": true}
+{"timestamp": 1767800814.2009513, "query": "what is project name?", "retrieved_count": 5, "confidence": 1.0, "answer_known": true}
+{"timestamp": 1767800836.0129147, "query": "what is watson AI?", "retrieved_count": 5, "confidence": 1.0, "answer_known": true}
+{"timestamp": 1767800871.1301703, "query": "what is visa ?", "retrieved_count": 5, "confidence": 0.0, "answer_known": false}
+{"timestamp": 1767947326.4927118, "query": "summarize the uploaded documents", "retrieved_count": 5, "confidence": 1.0, "answer_known": false}
+{"timestamp": 1767947424.8707786, "query": "summarize the uploaded documents", "retrieved_count": 5, "confidence": 1.0, "answer_known": true}
+{"timestamp": 1767947471.041334, "query": "what is the course name?", "retrieved_count": 5, "confidence": 1.0, "answer_known": false}
+{"timestamp": 1767947586.3758693, "query": "what is the application id?", "retrieved_count": 5, "confidence": 1.0, "answer_known": false}
+{"timestamp": 1767948654.5804863, "query": "summarize the uploaded documents", "retrieved_count": 5, "confidence": 1.0, "answer_known": true}
+{"timestamp": 1767948704.0778553, "query": "what is university name?", "retrieved_count": 5, "confidence": 1.0, "answer_known": true}
+{"timestamp": 1767948725.2018607, "query": "what is program name?", "retrieved_count": 5, "confidence": 0.0, "answer_known": false}
+{"timestamp": 1767948743.1319876, "query": "what is course name?", "retrieved_count": 5, "confidence": 0.0, "answer_known": false}
+{"timestamp": 1767948761.2565615, "query": "what is application id ?", "retrieved_count": 5, "confidence": 0.0, "answer_known": false}
+{"timestamp": 1767948799.2538924, "query": "what is name of student?", "retrieved_count": 5, "confidence": 0.0, "answer_known": false}
+{"timestamp": 1767948996.8832078, "query": "what is country name ?", "retrieved_count": 5, "confidence": 1.0, "answer_known": true}
+{"timestamp": 1767949032.4865937, "query": "whats the program name and duration?", "retrieved_count": 5, "confidence": 1.0, "answer_known": true}
+{"timestamp": 1767949053.9461539, "query": "whats the course name and duration?", "retrieved_count": 5, "confidence": 1.0, "answer_known": true}
+{"timestamp": 1767949089.903896, "query": "tell me the process of refund policy?", "retrieved_count": 5, "confidence": 1.0, "answer_known": true}
+{"timestamp": 1767949143.9379044, "query": "application id?", "retrieved_count": 5, "confidence": 0.0, "answer_known": false}
+{"timestamp": 1767949182.8464582, "query": "what is the student id?", "retrieved_count": 5, "confidence": 0.0, "answer_known": false}
+{"timestamp": 1767949216.585371, "query": "what is the program plan code?", "retrieved_count": 5, "confidence": 0.0, "answer_known": false}
+{"timestamp": 1767949349.670479, "query": "what is the commencement date ?", "retrieved_count": 5, "confidence": 0.0, "answer_known": false}
+{"timestamp": 1767949392.793658, "query": "summarize the uploaded documents", "retrieved_count": 5, "confidence": 0.0, "answer_known": false}
+{"timestamp": 1767949909.9175289, "query": "what is the Commencement Date?", "retrieved_count": 5, "confidence": 0.0, "answer_known": false}
+{"timestamp": 1768032006.1977339, "query": "what is duration?", "retrieved_count": 5, "confidence": 0.0, "answer_known": false}
+{"timestamp": 1768032029.513309, "query": "whats the program name?", "retrieved_count": 5, "confidence": 0.0, "answer_known": false}
+{"timestamp": 1768032061.2607996, "query": "what is university name?", "retrieved_count": 5, "confidence": 1.0, "answer_known": true}
+{"timestamp": 1768032085.8119817, "query": "what is program duration?", "retrieved_count": 5, "confidence": 0.0, "answer_known": false}
+{"timestamp": 1768032129.063438, "query": "what are the refund policy rules?", "retrieved_count": 5, "confidence": 1.0, "answer_known": true}
+{"timestamp": 1768032174.2048614, "query": "what is the master of artificial intelligence program durtaion?", "retrieved_count": 5, "confidence": 0.0, "answer_known": false}
+{"timestamp": 1768032198.3882332, "query": "what is the course duration?", "retrieved_count": 5, "confidence": 1.0, "answer_known": true}
+{"timestamp": 1768032243.0287364, "query": "what are the visa rules ?", "retrieved_count": 5, "confidence": 1.0, "answer_known": true}
+{"timestamp": 1768034581.1382725, "query": "how much of deposit to pay?", "retrieved_count": 5, "confidence": 1.0, "answer_known": true}
+{"timestamp": 1768036053.733725, "query": "What is the tuition fee?", "retrieved_count": 5, "confidence": 1.0, "answer_known": true}
+{"timestamp": 1768036088.6985803, "query": "Is Melbourne a good city?", "retrieved_count": 0, "confidence": 0.3, "answer_known": true}
+{"timestamp": 1768056940.6508985, "query": "what is the AIDA framework", "retrieved_count": 0, "confidence": 0.3, "answer_known": true}
+{"timestamp": 1768056996.7694573, "query": "what is meant by Landing Page Hero Section (AIDA Framework)?", "retrieved_count": 0, "confidence": 0.3, "answer_known": true}
+{"timestamp": 1768057049.903009, "query": "what is Error Messages & Microcopy", "retrieved_count": 0, "confidence": 0.3, "answer_known": true}
+{"timestamp": 1768121326.683464, "query": "what is the location ?", "retrieved_count": 5, "confidence": 1.0, "answer_known": true}
+{"timestamp": 1768121412.7319663, "query": "what is the conditions for OSHC?", "retrieved_count": 5, "confidence": 1.0, "answer_known": true}
+{"timestamp": 1768207382.1637495, "query": "what is application ID number?", "retrieved_count": 5, "confidence": 0.95, "answer_known": false}

rag_store.py

@@ -3,10 +3,8 @@ import os
 import pickle
 from pypdf import PdfReader
 from sentence_transformers import SentenceTransformer, CrossEncoder
+from rank_bm25 import BM25Okapi
 
-# =========================================================
-# CONFIG
-# =========================================================
 USE_HNSW = True
 USE_RERANKER = True
 
@@ -15,24 +13,20 @@ CHUNK_OVERLAP = 200
 
 DB_FILE_INDEX = "vector.index"
 DB_FILE_META = "metadata.pkl"
+DB_FILE_BM25 = "bm25.pkl"
 
-# =========================================================
-# GLOBAL STATE
-# =========================================================
 index = None
 documents = []
 metadata = []
+bm25 = None
+tokenized_corpus = []
 
 embedder = SentenceTransformer("all-MiniLM-L6-v2")
 reranker = CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2")
 
-# =========================================================
-# HELPERS
-# =========================================================
 def chunk_text(text):
     import re
     sentences = re.split(r'(?<=[.!?])\s+', text)
-
     chunks, current = [], ""
     for s in sentences:
         if len(current) + len(s) > CHUNK_SIZE and current:
@@ -41,152 +35,154 @@ def chunk_text(text):
             current = current[overlap:] + " " + s
         else:
             current += " " + s if current else s
-
     if current.strip():
         chunks.append(current.strip())
     return chunks
 
-
 def save_db():
     if index:
         faiss.write_index(index, DB_FILE_INDEX)
     if documents:
         with open(DB_FILE_META, "wb") as f:
             pickle.dump({"documents": documents, "metadata": metadata}, f)
-
+    if bm25:
+        with open(DB_FILE_BM25, "wb") as f:
+            pickle.dump(bm25, f)
 
 def load_db():
-    global index, documents, metadata
+    global index, documents, metadata, bm25
     if os.path.exists(DB_FILE_INDEX) and os.path.exists(DB_FILE_META):
         index = faiss.read_index(DB_FILE_INDEX)
         with open(DB_FILE_META, "rb") as f:
             data = pickle.load(f)
             documents = data["documents"]
             metadata = data["metadata"]
-        print(f"DEBUG: Loaded {len(documents)} chunks")
-
+    
+    if os.path.exists(DB_FILE_BM25):
+        with open(DB_FILE_BM25, "rb") as f:
+            bm25 = pickle.load(f)
+    elif documents:
+        # Auto-backfill if documents exist but BM25 is missing
+        print("Backfilling BM25 index on first load...")
+        tokenized_corpus = [doc.split(" ") for doc in documents]
+        bm25 = BM25Okapi(tokenized_corpus)
+        with open(DB_FILE_BM25, "wb") as f:
+            pickle.dump(bm25, f)
 
 load_db()
 
-
 def clear_database():
-    global index, documents, metadata
+    global index, documents, metadata, bm25
     index = None
     documents = []
     metadata = []
-
+    bm25 = None
     if os.path.exists(DB_FILE_INDEX):
         os.remove(DB_FILE_INDEX)
     if os.path.exists(DB_FILE_META):
         os.remove(DB_FILE_META)
+    if os.path.exists(DB_FILE_BM25):
+        os.remove(DB_FILE_BM25)
 
-
-# =========================================================
-# INGEST
-# =========================================================
 def ingest_documents(files):
     global index, documents, metadata
-
     texts, meta = [], []
 
     for file in files:
-        name = file.filename
-
-        if name.endswith(".pdf"):
+        if file.filename.endswith(".pdf"):
             reader = PdfReader(file.file)
             for i, page in enumerate(reader.pages):
-                try:
-                    text = page.extract_text()
-                except Exception:
-                    text = None
-
+                text = page.extract_text()
                 if text:
                     for chunk in chunk_text(text):
                         texts.append(chunk)
-                        meta.append({"source": name, "page": i + 1})
+                        meta.append({"source": file.filename, "page": i + 1})
 
-        elif name.endswith(".txt"):
+        elif file.filename.endswith(".txt"):
             content = file.file.read().decode("utf-8", errors="ignore")
             for chunk in chunk_text(content):
                 texts.append(chunk)
-                meta.append({"source": name, "page": "N/A"})
+                meta.append({"source": file.filename, "page": "N/A"})
 
     if not texts:
-        raise ValueError(
-            "No readable text found. "
-            "If this is a scanned PDF, OCR is required."
-        )
+        raise ValueError("No readable text found (OCR needed for scanned PDFs).")
 
-    embeddings = embedder.encode(
-        texts,
-        convert_to_numpy=True,
-        normalize_embeddings=True
-    )
+    embeddings = embedder.encode(texts, convert_to_numpy=True, normalize_embeddings=True)
 
     if index is None:
         dim = embeddings.shape[1]
-        if USE_HNSW:
-            index = faiss.IndexHNSWFlat(dim, 32)
-            index.hnsw.efConstruction = 200
-            index.hnsw.efSearch = 64
-        else:
-            index = faiss.IndexFlatIP(dim)
+        index = faiss.IndexHNSWFlat(dim, 32) if USE_HNSW else faiss.IndexFlatIP(dim)
+        index.hnsw.efConstruction = 200
+        index.hnsw.efSearch = 64
 
     index.add(embeddings)
     documents.extend(texts)
     metadata.extend(meta)
-
+    
+    # Update BM25
+    tokenized_corpus = [doc.split(" ") for doc in documents]
+    bm25 = BM25Okapi(tokenized_corpus)
+    
     save_db()
     return len(documents)
 
-
-# =========================================================
-# SEARCH
-# =========================================================
-def search_knowledge(query, top_k=8, min_similarity=0.25):
+def search_knowledge(query, top_k=8):
     if index is None:
         return []
 
-    qvec = embedder.encode(
-        [query],
-        convert_to_numpy=True,
-        normalize_embeddings=True
-    )
-
+    # 1. Vector Search
+    qvec = embedder.encode([query], convert_to_numpy=True, normalize_embeddings=True)
     scores, indices = index.search(qvec, top_k)
-    candidates = []
-    ql = query.lower()
-
-    for idx, score in zip(indices[0], scores[0]):
-        if idx == -1:
-            continue
-
-        text = documents[idx]
-        meta = metadata[idx]
-        keyword_hits = sum(w in text.lower() for w in ql.split())
-        hybrid_score = float(score) + (0.05 * keyword_hits)
-
-        if hybrid_score >= min_similarity:
-            candidates.append({
-                "text": text,
-                "metadata": meta,
-                "hybrid_score": hybrid_score
-            })
+    
+    vector_results = {}
+    for i, (idx, score) in enumerate(zip(indices[0], scores[0])):
+        if idx == -1: continue
+        vector_results[idx] = i  # Store rank (0-based)
+
+    # 2. Keyword Search (BM25)
+    bm25_results = {}
+    if bm25:
+        tokenized_query = query.split(" ")
+        bm25_scores = bm25.get_scores(tokenized_query)
+        # Get top_k indices
+        top_n = sorted(range(len(bm25_scores)), key=lambda i: bm25_scores[i], reverse=True)[:top_k]
+        for i, idx in enumerate(top_n):
+            bm25_results[idx] = i  # Store rank
+
+    # 3. Reciprocal Rank Fusion (RRF)
+    # score = 1 / (k + rank)
+    k = 60
+    candidates_idx = set(vector_results.keys()) | set(bm25_results.keys())
+    merged_candidates = []
+
+    for idx in candidates_idx:
+        v_rank = vector_results.get(idx, float('inf'))
+        b_rank = bm25_results.get(idx, float('inf'))
+        
+        rrf_score = (1 / (k + v_rank)) + (1 / (k + b_rank))
+        
+        merged_candidates.append({
+            "text": documents[idx],
+            "metadata": metadata[idx],
+            "score": rrf_score,  # This is RRF score, not cosine/BM25 score
+            "vector_rank": v_rank if v_rank != float('inf') else None,
+            "bm25_rank": b_rank if b_rank != float('inf') else None
+        })
+
+    # Sort by RRF score
+    merged_candidates.sort(key=lambda x: x["score"], reverse=True)
+    
+    # 4. Rerank Top Candidates
+    candidates = merged_candidates[:10] # Take top 10 for reranking
 
     if USE_RERANKER and candidates:
         pairs = [(query, c["text"]) for c in candidates]
-        scores = reranker.predict(pairs)
-        for c, s in zip(candidates, scores):
-            c["rerank"] = float(s)
+        rerank_scores = reranker.predict(pairs)
+        for c, rs in zip(candidates, rerank_scores):
+            c["rerank"] = float(rs)
         candidates.sort(key=lambda x: x["rerank"], reverse=True)
-    else:
-        candidates.sort(key=lambda x: x["hybrid_score"], reverse=True)
 
     return candidates[:5]
 
-
-def get_all_chunks(limit=50):
-    return [
-        {"text": t, "metadata": m}
-        for t, m in zip(documents[:limit], metadata[:limit])
-    ]
+def get_all_chunks(limit=80):
+    return [{"text": t, "metadata": m} for t, m in zip(documents[:limit], metadata[:limit])]

requirements.txt

@@ -7,3 +7,5 @@ sentence-transformers
 pypdf
 numpy
 python-multipart
+
+rank_bm25