Attempting to add URL crawler

app.py

@@ -1,10 +1,11 @@
 import streamlit as st
 from haystack.document_stores import InMemoryDocumentStore
-from haystack.nodes import TransformersSummarizer, PreProcessor, PDFToTextConverter
+from haystack.nodes import TransformersSummarizer, PreProcessor, PDFToTextConverter, Crawler
 from haystack.schema import Document
 import logging
 import base64
 from PIL import Image
+import validators
 
 @st.cache(hash_funcs={"builtins.SwigPyObject": lambda _: None},allow_output_mutation=True)
 def start_haystack():
@@ -32,8 +33,20 @@ def pdf_to_document_store(pdf_file):
         document_store.write_documents(preprocessed_docs)
         temp_file.close()
 
-def summarize(file):
-    pdf_to_document_store(file)
+def crawl_url(url):
+    crawler = Crawler(output_dir="crawled_files", overwrite_existing_files=True, crawler_depth=1)
+    try:
+        docs = crawler.crawl(urls=[url])
+        preprocessed_docs = preprocessor.process(docs)
+        document_store.write_documents(preprocessed_docs)
+    except:
+        st.write('We were unable to crawl the contents of that URL, please try something else')
+    
+def summarize(content):
+    if st.session_state.pdf:
+        pdf_to_document_store(content)
+    elif st.session_state.url:
+        crawl_url(content)
     summaries = summarizer.predict(documents=document_store.get_all_documents(), generate_single_summary=True)
     return summaries
 
@@ -41,7 +54,10 @@ def set_state_if_absent(key, value):
     if key not in st.session_state:
         st.session_state[key] = value
         
-set_state_if_absent("summaries", None)        
+set_state_if_absent("summaries", None)
+set_state_if_absent("url", False)
+set_state_if_absent("pdf", False)
+        
 document_store, summarizer, preprocessor = start_haystack()
 
 st.title('TL;DR with Haystack')
@@ -53,12 +69,25 @@ This Summarization demo uses a [Haystack TransformerSummarizer node](https://hay
 """, unsafe_allow_html=True)
 
 uploaded_file = st.file_uploader("Choose a PDF file", accept_multiple_files=False)
+url = st.text_input(label="enter a URL", type="URL")
 
-if uploaded_file is not None:
+if (validators.url(url)) and (upload_file is None):
+    if st.button('Summarize contents of URL'):
+        with st.spinner("📚    Please wait while we produce a summary..."):
+            try:
+                st.session_state.pdf = False
+                st.session_state.url = True
+                st. session_state.summaries = summarize(url)
+            except Exception as e:
+                logging.exception(e)
+                
+if (uploaded_file is not None) and !(validators.url(url)):
     if st.button('Summarize Document'):
         with st.spinner("📚    Please wait while we produce a summary..."):
             try:
-                st. session_state.summaries = summarize(uploaded_file)
+                st.session_state.pdf = True
+                st.session_state.url = False
+                st.session_state.summaries = summarize(uploaded_file)
             except Exception as e:
                 logging.exception(e)