lint

cfg.py

@@ -24,7 +24,9 @@ homepage_url = os.getenv("RTD_URL", "https://orion.readthedocs.io/")
 target_version = os.getenv("RTD_VERSION", "en/stable")
 
 # scrape and embed content from readthedocs website
-scrape_rtd(homepage_url=homepage_url, save_directory="outputs/", target_version=target_version)
+scrape_rtd(
+    homepage_url=homepage_url, save_directory="outputs/", target_version=target_version
+)
 
 # Disable logging for third-party libraries at DEBUG level
 for name in logging.root.manager.loggerDict:

rtd_scraper/scrape_rtd.py

@@ -8,6 +8,7 @@ from scrapy.crawler import CrawlerProcess
 from scrapy.utils.project import get_project_settings
 
 from rtd_scraper.tutorial.spiders.docs_spider import DocsSpider
+
 # from tutorial.spiders.docs_spider import DocsSpider
 
 # When using scrapy it seems to set logging for all apps at DEBUG, so simply shut it off here...
@@ -18,7 +19,12 @@ for name in logging.root.manager.loggerDict:
 
 def run_spider(homepage_url, save_directory, target_version=None):
     process = CrawlerProcess(settings=get_project_settings())
-    process.crawl(DocsSpider, homepage_url=homepage_url, save_dir=save_directory, target_version=target_version)
+    process.crawl(
+        DocsSpider,
+        homepage_url=homepage_url,
+        save_dir=save_directory,
+        target_version=target_version,
+    )
 
     # Start the crawling process
     process.start()
@@ -29,7 +35,9 @@ def run_spider(homepage_url, save_directory, target_version=None):
 
 def scrape_rtd(homepage_url, save_directory, target_version=None):
     # Crawl the website using scrapy
-    run_spider(homepage_url, save_directory=save_directory, target_version=target_version)
+    run_spider(
+        homepage_url, save_directory=save_directory, target_version=target_version
+    )
 
     # # Convert the .html pages into chunks using Buster's SphinxParser
     root_dir = os.path.join(save_directory, homepage_url.split("https://")[1])
@@ -62,7 +70,8 @@ def scrape_rtd(homepage_url, save_directory, target_version=None):
     )
 
 
-
 if __name__ == "__main__":
     homepage_url = "https://orion.readthedocs.io/"
-    scrape_rtd(homepage_url=homepage_url, target_version="v0.2.7", save_directory="outputs/")
+    scrape_rtd(
+        homepage_url=homepage_url, target_version="v0.2.7", save_directory="outputs/"
+    )

rtd_scraper/tutorial/spiders/docs_spider.py

@@ -8,6 +8,7 @@ logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.ERROR)
 
 from urllib.parse import urlparse
 
+
 def extract_domain(url):
     """
     Extract the domain (including subdomains) from a given URL.
@@ -29,7 +30,14 @@ def extract_domain(url):
 class DocsSpider(scrapy.Spider):
     name = "docs"
 
-    def __init__(self, homepage_url: str, save_dir="crawled_pages", target_version=None, *args, **kwargs):
+    def __init__(
+        self,
+        homepage_url: str,
+        save_dir="crawled_pages",
+        target_version=None,
+        *args,
+        **kwargs,
+    ):
         super(DocsSpider, self).__init__(*args, **kwargs)
 
         if not homepage_url.startswith("https://"):
@@ -67,4 +75,4 @@ class DocsSpider(scrapy.Spider):
                     yield response.follow(href, self.parse)
             else:
                 # no version specified, follow all links
-                yield response.follow(href, self.parse)
+                yield response.follow(href, self.parse)