sanitize web urls

app.py

@@ -4,8 +4,8 @@ from typing import Optional, Tuple
 import gradio as gr
 import pandas as pd
 from buster.completers import Completion
-from buster.utils import extract_zip
 
+# from embed_docs import embed_rtd_website
 from rtd_scraper.scrape_rtd import scrape_rtd
 import cfg
 from cfg import setup_buster
@@ -18,8 +18,8 @@ if os.getenv("OPENAI_API_KEY") is None:
     )
 
 
-homepage_url = os.getenv("READTHEDOCS_URL") # e.g. "https://orion.readthedocs.io/"
-target_version = os.getenv("READTHEDOCS_VERSION") # e.g. "en/stable"
+homepage_url = os.getenv("READTHEDOCS_URL")  # e.g. "https://orion.readthedocs.io/"
+target_version = os.getenv("READTHEDOCS_VERSION")  # e.g. "en/stable"
 
 # scrape and embed content from readthedocs website
 # comment out if already embedded locally to avoid extra costs
@@ -117,10 +117,8 @@ with demo:
         answer questions.
         View the code on the [project homepage](https://github.com/jerpint/RAGTheDocs)
         """
-
     )
 
-
     chatbot = gr.Chatbot()
 
     with gr.Row():
@@ -135,7 +133,7 @@ with demo:
         examples=[
             "How can I install the library?",
             "What dependencies are required?",
-            "Give a brief overview of the library."
+            "Give a brief overview of the library.",
         ],
         inputs=question,
     )

rtd_scraper/scrape_rtd.py

@@ -7,7 +7,7 @@ from buster.parser import SphinxParser
 from scrapy.crawler import CrawlerProcess
 from scrapy.utils.project import get_project_settings
 
-from rtd_scraper.tutorial.spiders.docs_spider import DocsSpider
+from rtd_scraper.tutorial.spiders.docs_spider import DocsSpider, sanitize_url
 
 # from tutorial.spiders.docs_spider import DocsSpider
 
@@ -34,6 +34,10 @@ def run_spider(homepage_url, save_directory, target_version=None):
 
 
 def scrape_rtd(homepage_url, save_directory, target_version=None):
+
+    # adds https:// and trailing backslash
+    homepage_url = sanitize_url(homepage_url)
+
     # Crawl the website using scrapy
     run_spider(
         homepage_url, save_directory=save_directory, target_version=target_version

rtd_scraper/tutorial/spiders/docs_spider.py

@@ -27,21 +27,30 @@ def extract_domain(url):
     return domain
 
 
+def sanitize_url(url: str) -> str:
+    """Adds https:// and trailing backslash."""
+    if not url.startswith("https://"):
+        url = "https://" + url
+
+    if not url.endswith("/"):
+        url = url + "/"
+    return url
+
+
 class DocsSpider(scrapy.Spider):
     name = "docs"
 
     def __init__(
         self,
         homepage_url: str,
-        save_dir="crawled_pages",
+        save_dir="outputs/",
         target_version=None,
         *args,
         **kwargs,
     ):
         super(DocsSpider, self).__init__(*args, **kwargs)
 
-        if not homepage_url.startswith("https://"):
-            homepage_url = "https://" + homepage_url
+        homepage_url = sanitize_url(homepage_url)
 
         self.allowed_domains = [extract_domain(homepage_url)]
         self.start_urls = [homepage_url]