Create scraper.py

scraper.py

@@ -0,0 +1,77 @@
+import requests
+from bs4 import BeautifulSoup
+import re
+import time
+
+def get_course_links(page_url):
+    """
+    Scrapes the links of individual courses from the given page URL.
+
+    Args:
+        page_url (str): The URL of the page to scrape.
+
+    Returns:
+        list: A list of course URLs.
+    """
+    response = requests.get(page_url)
+    soup = BeautifulSoup(response.content, 'html.parser')
+    course_links = [a['href'] for a in soup.select('a.course-card.course-card__public.published') if 'href' in a.attrs]
+    course_links = [link if link.startswith('http') else f"https://courses.analyticsvidhya.com{link}" for link in course_links]
+    return course_links
+
+
+def parse_course_page(url):
+    """
+    Extracts course details from a given course page URL.
+
+    Args:
+        url (str): The URL of the course page.
+
+    Returns:
+        dict: A dictionary containing course information.
+    """
+    response = requests.get(url)
+    soup = BeautifulSoup(response.content, 'html.parser')
+    
+    title = soup.select_one('h1[class^="section__heading"]')
+    title = title.text.strip() if title else 'N/A'
+    
+    description = soup.select_one('div.fr-view')
+    description = description.text.strip() if description else 'No description available'
+    
+    curriculum = [re.sub(r'\s+', ' ', item.text.strip()) for item in soup.select('div.course-curriculum__chapter-content-wrapper')]
+    curriculum = ' | '.join(curriculum) if curriculum else 'No curriculum available'
+    
+    return {
+        'Title': title,
+        'Description': description,
+        'Curriculum': curriculum,
+        'Link': url
+    }
+
+
+def scrape_courses_json(base_url, num_pages=5):
+    """
+    Scrapes courses from multiple pages and returns course data.
+
+    Args:
+        base_url (str): Base URL of the course pages.
+        num_pages (int): Number of pages to scrape.
+
+    Returns:
+        list: A list of dictionaries, each containing course information.
+    """
+    data = []
+    for page_num in range(1, num_pages + 1):
+        page_url = f"{base_url}{page_num}"
+        course_links = get_course_links(page_url)
+        
+        for link in course_links:
+            try:
+                course_data = parse_course_page(link)
+                data.append(course_data)
+                time.sleep(1)  # Be courteous to the server
+            except Exception as e:
+                print(f"Failed to scrape {link}: {e}")
+
+    return data