Update main.py

main.py

@@ -108,8 +108,12 @@ def clean_url(url: str) -> str:
     if not url:
         return ""
 
-    if url.startswith('//duckduckgo.com/l/'):
-        url = f"https:{url}"
+    # Handle DuckDuckGo redirect links like //duckduckgo.com/l/?uddg=... or /l/?uddg=...
+    if url.startswith('//duckduckgo.com/l/') or url.startswith('/l/?'):
+        if url.startswith('//'):
+            url = f"https:{url}"
+        elif url.startswith('/'):
+            url = f"https://duckduckgo.com{url}"
         try:
             parsed = urlparse(url)
             query_params = parsed.query
@@ -117,7 +121,7 @@ def clean_url(url: str) -> str:
                 match = re.search(r'uddg=([^&]+)', query_params)
                 if match:
                     return unquote(match.group(1))
-        except:
+        except Exception:
             pass
 
     if url.startswith('//'):
@@ -153,68 +157,135 @@ async def check_robots_txt(url: str) -> bool:
         return False
 
 async def fetch_search_results(query: str, max_results: int = 5) -> List[dict]:
-    """Perform a real search using DuckDuckGo's HTML interface with robust retry logic."""
-    headers = {
-        "User-Agent": await get_real_user_agent(),
-        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
-        "Accept-Language": "en-US,en;q=0.5",
+    """Perform a real search using DuckDuckGo (Lite/HTML) with multi-endpoint fallback to reduce 202 issues."""
+    ua_hdr = await get_real_user_agent()
+    common_headers = {
+        "User-Agent": ua_hdr,
+        "Accept-Language": "en-US,en;q=0.9",
+        "DNT": "1",
+        "Cache-Control": "no-cache",
+        "Pragma": "no-cache",
         "Referer": "https://duckduckgo.com/",
-        "DNT": "1"
     }
 
+    # Try Lite first (very lightweight HTML), then HTML mirrors
+    endpoints = [
+        {"name": "lite-get", "method": "GET", "url": lambda q: f"https://lite.duckduckgo.com/lite/?q={quote_plus(q)}&kl=us-en&bing_market=us-en"},
+        # Per provided openapi.json: POST /lite/ with query params
+        {"name": "lite-post", "method": "POST", "url": lambda q: f"https://lite.duckduckgo.com/lite/?q={quote_plus(q)}&kl=us-en&bing_market=us-en"},
+        {"name": "html-mirror", "method": "GET", "url": lambda q: f"https://html.duckduckgo.com/html/?q={quote_plus(q)}"},
+        {"name": "html", "method": "GET", "url": lambda q: f"https://duckduckgo.com/html/?q={quote_plus(q)}"},
+    ]
+
+    def parse_results_from_html(html: str) -> List[dict]:
+        soup = BeautifulSoup(html, 'html.parser')
+        results: List[dict] = []
+
+        # Primary selectors (full HTML interface)
+        candidates = soup.select('.result__body')
+        if not candidates:
+            candidates = soup.select('.result')
+
+        for result in candidates:
+            try:
+                title_elem = result.select_one('.result__title .result__a') or result.select_one('.result__a')
+                if not title_elem:
+                    # Lite fallback: find first anchor in this block
+                    title_elem = result.find('a')
+                    if not title_elem:
+                        continue
+                link = title_elem.get('href')
+                if not link:
+                    continue
+                snippet_elem = result.select_one('.result__snippet') or result.find('p')
+                clean_link = clean_url(link)
+                if not clean_link or clean_link.startswith('javascript:'):
+                    continue
+                snippet = snippet_elem.get_text(strip=True) if snippet_elem else ""
+                title_text = title_elem.get_text(strip=True)
+                results.append({'title': title_text, 'link': clean_link, 'snippet': snippet})
+            except Exception as e:
+                logging.warning(f"Error parsing search result: {e}")
+                continue
+
+        # DuckDuckGo Lite often uses simple anchors; target likely link patterns first
+        if not results:
+            lite_links = soup.select('a[href*="/l/?uddg="]')
+            for a in lite_links:
+                try:
+                    href = a.get('href')
+                    title_text = a.get_text(strip=True)
+                    if not href or not title_text:
+                        continue
+                    clean_link = clean_url(href)
+                    if not clean_link or clean_link.startswith('javascript:'):
+                        continue
+                    results.append({'title': title_text, 'link': clean_link, 'snippet': ''})
+                    if len(results) >= max_results:
+                        break
+                except Exception:
+                    continue
+
+        # If still empty, do a very generic anchor scrape (fallback)
+        if not results:
+            anchors = soup.find_all('a', href=True)
+            for a in anchors:
+                text = a.get_text(strip=True)
+                href = a['href']
+                if not text or not href:
+                    continue
+                if '/l/?' in href or href.startswith('http') or href.startswith('//'):
+                    clean_link = clean_url(href)
+                    if clean_link and not clean_link.startswith('javascript:'):
+                        results.append({'title': text, 'link': clean_link, 'snippet': ''})
+                if len(results) >= max_results * 2:
+                    break
+
+        return results[:max_results]
+
     for attempt in range(RETRY_ATTEMPTS):
         try:
-            search_url = f"https://html.duckduckgo.com/html/?q={quote_plus(query)}"
             async with aiohttp.ClientSession() as session:
-                async with session.get(search_url, headers=headers, timeout=10) as response:
-                    if response.status != 200:
-                        if response.status == 202:
-                            logging.warning(f"Search attempt {attempt + 1} failed with status 202 for query '{query}'")
-                            if attempt < RETRY_ATTEMPTS - 1:
-                                await asyncio.sleep(RETRY_DELAY)
+                for ep in endpoints:
+                    url = ep['url'](query)
+                    headers = {**common_headers, "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}
+                    try:
+                        if ep['method'] == 'GET':
+                            resp = await session.get(url, headers=headers, timeout=12)
+                        else:
+                            # POST with querystring parameters as specified; no body required
+                            resp = await session.post(url, headers=headers, timeout=12)
+                        async with resp as response:
+                            if response.status == 200:
+                                html = await response.text()
+                                results = parse_results_from_html(html)
+                                if results:
+                                    logging.info(f"Found {len(results)} real search results for '{query}' via {ep['name']}")
+                                    return results
+                                # If empty, try next endpoint
+                                logging.warning(f"No results parsed from {ep['name']} for '{query}', trying next endpoint...")
                                 continue
-                        logging.warning(f"Search failed with status {response.status} for query '{query}'")
-                        return []
-
-                    html = await response.text()
-                    soup = BeautifulSoup(html, 'html.parser')
-
-                    results = []
-                    for selector in ['.result__body', '.result__a', '.result']:
-                        if len(results) >= max_results:
-                            break
-
-                        for result in soup.select(selector)[:max_results]:
-                            try:
-                                title_elem = result.select_one('.result__title .result__a') or result.select_one('.result__a')
-                                if not title_elem:
-                                    continue
-
-                                link = title_elem['href']
-                                snippet_elem = result.select_one('.result__snippet')
-
-                                clean_link = clean_url(link)
-                                if not clean_link or clean_link.startswith('javascript:'):
-                                    continue
-
-                                snippet = snippet_elem.get_text(strip=True) if snippet_elem else ""
-                                results.append({
-                                    'title': title_elem.get_text(strip=True),
-                                    'link': clean_link,
-                                    'snippet': snippet
-                                })
-                            except Exception as e:
-                                logging.warning(f"Error parsing search result: {e}")
+                            elif response.status == 202:
+                                logging.warning(f"Search attempt {attempt + 1} got 202 at {ep['name']} for '{query}', trying next endpoint")
+                                continue
+                            else:
+                                logging.warning(f"Search failed with status {response.status} at {ep['name']} for '{query}'")
                                 continue
+                    except asyncio.TimeoutError:
+                        logging.warning(f"Timeout contacting {ep['name']} for '{query}'")
+                        continue
+                    except Exception as e:
+                        logging.warning(f"Error contacting {ep['name']} for '{query}': {e}")
+                        continue
 
-                    logging.info(f"Found {len(results)} real search results for '{query}'")
-                    return results[:max_results]
         except Exception as e:
             logging.error(f"Search attempt {attempt + 1} failed for '{query}': {e}")
-            if attempt < RETRY_ATTEMPTS - 1:
-                await asyncio.sleep(RETRY_DELAY)
-                continue
-    logging.error(f"All {RETRY_ATTEMPTS} search attempts failed for '{query}'")
+
+        # Backoff before next multi-endpoint attempt
+        if attempt < RETRY_ATTEMPTS - 1:
+            await asyncio.sleep(RETRY_DELAY)
+
+    logging.error(f"All {RETRY_ATTEMPTS} search attempts failed across endpoints for '{query}'")
     return []
 
 async def process_web_source(session: aiohttp.ClientSession, source: dict, timeout: int = 15) -> Tuple[str, dict]:
@@ -780,7 +851,10 @@ Evidence and notes from crawled sources (trimmed):
                     f"[{s['id']}] {s['title']} — {s['url']}" for s in sources_catalog
                 ]
                 refs_md = "\n".join(refs_md_lines)
-                yield format_sse({"event": "chunk", "data": {"text": refs_md, "section": "references"}})
+                # Back-compat: plain chunk
+                yield format_sse({"event": "chunk", "data": refs_md})
+                # New: section-tagged chunk
+                yield format_sse({"event": "section_chunk", "data": {"text": refs_md, "section": "references"}})
 
             duration = time.time() - start_time
             stats = {