Update other_function.py

other_function.py

@@ -112,40 +112,25 @@ def get_weather(city):
   celcius=str(round((int(degree) - 32)* 5/9,1))+temperature[-2]+'C'
   return (celcius)
 
-def get_rates():
-    chrome_options = Options()
-    chrome_options.add_argument("--headless")
-    chrome_options.add_argument("--no-sandbox")
-    chrome_options.add_argument("--disable-dev-shm-usage")
-    chrome_options.add_argument("--disable-gpu")
-    chrome_options.add_argument("--disable-extensions")
-    chrome_options.add_argument("--disable-infobars")
-    chrome_options.add_argument("--disable-images")
-    chrome_options.add_argument("--blink-settings=imagesEnabled=false")  # Disable image loading
-    chrome_options.add_argument("--enable-automation")
-
-    # Initialize the WebDriver with the headless option
-
-    driver = webdriver.Chrome(options=chrome_options)
-    
-    # Open the website
-    driver.get('https://www.kisandeals.com/mandiprices/ALL/TAMIL-NADU/ALL')
-    
-    # Wait for the table to be present (instead of waiting for the entire page)
-    table = WebDriverWait(driver, 5).until(
-        EC.presence_of_element_located((By.XPATH, '//table'))
-    )
-    
-    # Parse the table using pandas
-    df = pd.read_html(table.get_attribute('outerHTML'))[0]
-    
-    # Drop the 'Quintal Price' column
-    df.drop(columns=['Quintal Price'], inplace=True)
-    
-    # Convert the data to a dictionary
-    d = {df.iloc[i, 0]: df.iloc[i, 1] for i in range(len(df))}
-    
-    # Close the WebDriver
-    driver.quit()
-    
-    return str(d) + ' These prices are for 1 kg'
+import scrapy
+from scrapy.selector import Selector
+from urllib.parse import urljoin
+import pandas as pd
+
+class KisanDealsSpider(scrapy.Spider):
+    name = 'kisan_deals'
+    start_urls = ['https://www.kisandeals.com/mandiprices/ALL/TAMIL-NADU/ALL']
+
+    def parse(self, response):
+        table = response.xpath('//table')
+        rows = table.xpath('//tr')
+
+        data = []
+        for row in rows[1:]:  # Skip the header row
+            crop = row.xpath('//td[1]/text()').get()
+            rate = row.xpath('//td[2]/text()').get()
+            data.append({'crop': crop, 'rate': rate})
+
+        df = pd.DataFrame(data)
+        df = df.drop(columns=['Quintal Price'])
+        return str(df.to_dict('index')) + ' These prices are for 1 kg'