Delete non-used code

.idea/HFSummSpace.iml

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/venv" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="PyDocumentationSettings">
+    <option name="format" value="PLAIN" />
+    <option name="myDocStringFormat" value="Plain" />
+  </component>
+  <component name="TestRunnerService">
+    <option name="PROJECT_TEST_RUNNER" value="py.test" />
+  </component>
+</module>

.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,12 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="PyPep8NamingInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
+      <option name="ignoredErrors">
+        <list>
+          <option value="N806" />
+        </list>
+      </option>
+    </inspection_tool>
+  </profile>
+</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7 (HFSummSpace)" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/HFSummSpace.iml" filepath="$PROJECT_DIR$/.idea/HFSummSpace.iml" />
+    </modules>
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

.idea/workspace.xml

@@ -0,0 +1,107 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ChangeListManager">
+    <list default="true" id="57f23431-346d-451d-8d77-db859508e831" name="Changes" comment="">
+      <change beforePath="$PROJECT_DIR$/app.py" beforeDir="false" afterPath="$PROJECT_DIR$/app.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/custom_renderer.py" beforeDir="false" afterPath="$PROJECT_DIR$/custom_renderer.py" afterDir="false" />
+    </list>
+    <option name="SHOW_DIALOG" value="false" />
+    <option name="HIGHLIGHT_CONFLICTS" value="true" />
+    <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
+    <option name="LAST_RESOLUTION" value="IGNORE" />
+  </component>
+  <component name="FileTemplateManagerImpl">
+    <option name="RECENT_TEMPLATES">
+      <list>
+        <option value="Python Script" />
+      </list>
+    </option>
+  </component>
+  <component name="Git.Settings">
+    <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
+  </component>
+  <component name="GitSEFilterConfiguration">
+    <file-type-list>
+      <filtered-out-file-type name="LOCAL_BRANCH" />
+      <filtered-out-file-type name="REMOTE_BRANCH" />
+      <filtered-out-file-type name="TAG" />
+      <filtered-out-file-type name="COMMIT_BY_MESSAGE" />
+    </file-type-list>
+  </component>
+  <component name="HighlightingSettingsPerFile">
+    <setting file="file://$PROJECT_DIR$/venv/lib/python3.7/site-packages/flair/models/sequence_tagger_model.py" root0="SKIP_INSPECTION" />
+  </component>
+  <component name="MarkdownSettingsMigration">
+    <option name="stateVersion" value="1" />
+  </component>
+  <component name="ProjectId" id="27jdqgqsSB1v523dZaR7czhkX4c" />
+  <component name="ProjectLevelVcsManager" settingsEditedManually="true" />
+  <component name="ProjectViewState">
+    <option name="hideEmptyMiddlePackages" value="true" />
+    <option name="showLibraryContents" value="true" />
+  </component>
+  <component name="PropertiesComponent"><![CDATA[{
+  "keyToString": {
+    "last_opened_file_path": "/home/matthias/Documents/Summarization-fact-checker/HugginfaceSpace/HFSummSpace",
+    "settings.editor.selected.configurable": "editor.preferences.fonts.default"
+  }
+}]]></component>
+  <component name="RecentsManager">
+    <key name="CopyFile.RECENT_KEYS">
+      <recent name="$PROJECT_DIR$" />
+    </key>
+    <key name="MoveFile.RECENT_KEYS">
+      <recent name="$PROJECT_DIR$/sample-articles-temp" />
+    </key>
+  </component>
+  <component name="RunManager">
+    <configuration name="app" type="PythonConfigurationType" factoryName="Python" temporary="true" nameIsGenerated="true">
+      <module name="HFSummSpace" />
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <envs>
+        <env name="PYTHONUNBUFFERED" value="1" />
+      </envs>
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
+      <option name="IS_MODULE_SDK" value="true" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/app.py" />
+      <option name="PARAMETERS" value="" />
+      <option name="SHOW_COMMAND_LINE" value="false" />
+      <option name="EMULATE_TERMINAL" value="false" />
+      <option name="MODULE_MODE" value="false" />
+      <option name="REDIRECT_INPUT" value="false" />
+      <option name="INPUT_FILE" value="" />
+      <method v="2" />
+    </configuration>
+    <recent_temporary>
+      <list>
+        <item itemvalue="Python.app" />
+      </list>
+    </recent_temporary>
+  </component>
+  <component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="application-level" UseSingleDictionary="true" transferred="true" />
+  <component name="TaskManager">
+    <task active="true" id="Default" summary="Default task">
+      <changelist id="57f23431-346d-451d-8d77-db859508e831" name="Changes" comment="" />
+      <created>1649837622575</created>
+      <option name="number" value="Default" />
+      <option name="presentableId" value="Default" />
+      <updated>1649837622575</updated>
+    </task>
+    <servers />
+  </component>
+  <component name="Vcs.Log.Tabs.Properties">
+    <option name="TAB_STATES">
+      <map>
+        <entry key="MAIN">
+          <value>
+            <State />
+          </value>
+        </entry>
+      </map>
+    </option>
+  </component>
+</project>

app.py

@@ -1,10 +1,6 @@
-import random
-from typing import AnyStr, List, Dict
-# import tensorflow_hub as hub
+from typing import AnyStr, Dict
 
 import itertools
-
-#import en_core_web_sm
 import streamlit as st
 import en_core_web_lg
 
@@ -13,25 +9,15 @@ from bs4 import BeautifulSoup
 import numpy as np
 import base64
 
-import validators
 from spacy_streamlit.util import get_svg
-from validators import ValidationFailure
 
 from custom_renderer import render_sentence_custom
-# from flair.data import Sentence
-# from flair.models import SequenceTagger
 from sentence_transformers import SentenceTransformer
 
-import spacy
-from spacy import displacy
-from spacy_streamlit import visualize_parser
-
-from transformers import AutoTokenizer, AutoModelForSequenceClassification, AutoModelForTokenClassification
+from transformers import AutoTokenizer, AutoModelForTokenClassification
 from transformers import pipeline
 import os
-from transformers_interpret import SequenceClassificationExplainer
 
-# USE_model = hub.load("https://tfhub.dev/google/universal-sentence-encoder/4")
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
 
@@ -42,19 +28,10 @@ def get_sentence_embedding_model():
 
 @st.experimental_singleton
 def get_spacy():
-    # nlp = spacy.load('en_core_web_lg')
     nlp = en_core_web_lg.load()
     return nlp
 
 
-# TODO: might look into which one is the best here
-# TODO: might be useful to make an ml6 preloaded model for flair as this takes ridiculously long to load the first time
-# @st.experimental_singleton
-# @st.cache(suppress_st_warning=True, allow_output_mutation=True)
-# def get_flair_tagger():
-#     return SequenceTagger.load("flair/ner-english-ontonotes-fast")
-
-
 @st.experimental_singleton
 def get_transformer_pipeline():
     tokenizer = AutoTokenizer.from_pretrained("xlm-roberta-large-finetuned-conll03-english")
@@ -96,7 +73,7 @@ def list_all_article_names() -> list:
 
 
 def fetch_article_contents(filename: str) -> AnyStr:
-    if (filename == "Provide your own input"):
+    if filename == "Provide your own input":
         return " "
     with open(f'./sample-articles/{filename.lower()}.txt', 'r') as f:
         data = f.read()
@@ -174,13 +151,13 @@ def get_all_entities(text):
 
 # TODO: this functionality can be cached (e.g. by storing html file output) if wanted (or just store list of entities idk)
 def get_and_compare_entities():
-    #article_content = fetch_article_contents(article_name)
+    # article_content = fetch_article_contents(article_name)
     article_content = st.session_state.article_text
     all_entities_per_sentence = get_all_entities_per_sentence(article_content)
     # st.session_state.entities_per_sentence_article = all_entities_per_sentence
     entities_article = list(itertools.chain.from_iterable(all_entities_per_sentence))
 
-    #summary_content = fetch_summary_contents(article_name)
+    # summary_content = fetch_summary_contents(article_name)
     summary_content = st.session_state.summary_output
     all_entities_per_sentence = get_all_entities_per_sentence(summary_content)
     # st.session_state.entities_per_sentence_summary = all_entities_per_sentence
@@ -193,7 +170,8 @@ def get_and_compare_entities():
         if any(entity.lower() in substring_entity.lower() for substring_entity in entities_article):
             matched_entities.append(entity)
         elif any(
-                np.inner(sentence_embedding_model.encode(entity, show_progress_bar=False), sentence_embedding_model.encode(art_entity, show_progress_bar=False)) > 0.9 for
+                np.inner(sentence_embedding_model.encode(entity, show_progress_bar=False),
+                         sentence_embedding_model.encode(art_entity, show_progress_bar=False)) > 0.9 for
                 art_entity in entities_article):
             matched_entities.append(entity)
         else:
@@ -202,7 +180,7 @@ def get_and_compare_entities():
 
 
 def highlight_entities():
-    #summary_content = fetch_summary_contents(article_name)
+    # summary_content = fetch_summary_contents(article_name)
     summary_content = st.session_state.summary_output
     markdown_start_red = "<mark class=\"entity\" style=\"background: rgb(238, 135, 135);\">"
     markdown_start_green = "<mark class=\"entity\" style=\"background: rgb(121, 236, 121);\">"
@@ -277,13 +255,6 @@ def check_dependency(article: bool):
     # return all_deps
 
 
-def is_valid_url(url: str) -> bool:
-    result = validators.url(url)
-    if isinstance(result, ValidationFailure):
-        return False
-    return True
-
-
 def render_svg(svg_file):
     with open(svg_file, "r") as f:
         lines = f.readlines()
@@ -296,7 +267,6 @@ def render_svg(svg_file):
 
 
 def generate_abstractive_summary(text, type, min_len=120, max_len=512, **kwargs):
-    summarization_model = get_summarizer_model()
     text = text.strip().replace("\n", " ")
     if type == "top_p":
         text = summarization_model(text, min_length=min_len,
@@ -316,10 +286,6 @@ def generate_abstractive_summary(text, type, min_len=120, max_len=512, **kwargs)
     return summary
 
 
-# Start session
-if 'results' not in st.session_state:
-    st.session_state.results = []
-
 # Page
 st.title('Summarization fact checker')
 
@@ -341,11 +307,11 @@ metric, indicating the trustworthiness of the generated summary. Throughout this
 results for some methods on specific examples. These text blocks will be indicated and they change according to the 
 currently selected article.""")
 
+# Load all different models (cached) at start time of the hugginface space
 sentence_embedding_model = get_sentence_embedding_model()
-# tagger = get_flair_tagger()
 ner_model = get_transformer_pipeline()
 nlp = get_spacy()
-# nlp = en_core_web_sm.load()
+summarization_model = get_summarizer_model()
 
 # GENERATING SUMMARIES PART
 st.header("Generating summaries")
@@ -353,7 +319,6 @@ st.markdown("Let’s start by selecting an article text for which we want to gen
             "text yourself. Note that it’s suggested to provide a sufficiently large text, as otherwise the summary "
             "generated from it might not be optimal, leading to suboptimal performance of the post-processing steps.")
 
-# TODO: NEED TO CHECK ARTICLE TEXT INSTEAD OF ARTICLE NAME ALSO FREE INPUT OPTION
 selected_article = st.selectbox('Select an article or provide your own:',
                                 list_all_article_names())  # index=0, format_func=special_internal_function, key=None, help=None, on_change=None, args=None, kwargs=None, *, disabled=False)
 st.session_state.article_text = fetch_article_contents(selected_article)
@@ -363,23 +328,27 @@ article_text = st.text_area(
     height=150
 )
 
-summarize_button = st.button(label='Process article content', help="Generates summary and applies entity matching and dependency parsing for given article")
+summarize_button = st.button(label='Process article content',
+                             help="Generates summary and applies entity matching and dependency parsing for given article")
 
 if summarize_button:
     st.session_state.article_text = article_text
-    st.markdown("Below you can find the generated summary for the article. Based on empirical research, we will discuss "
-                "two main methods that detect some common errors. We can then score different summaries, to indicate how "
-                "factual a summary is for a given article. The idea is that in production, you could generate a set of "
-                "summaries for the same article, with different parameters (or even different models). By using "
-                "post-processing error detection, we can then select the best possible summary.")
+    st.markdown(
+        "Below you can find the generated summary for the article. Based on empirical research, we will discuss "
+        "two main methods that detect some common errors. We can then score different summaries, to indicate how "
+        "factual a summary is for a given article. The idea is that in production, you could generate a set of "
+        "summaries for the same article, with different parameters (or even different models). By using "
+        "post-processing error detection, we can then select the best possible summary.")
     if st.session_state.article_text:
         with st.spinner('Generating summary...'):
             # classify_comment(article_text, selected_model)
-            if selected_article != "Provide your own input" and article_text == fetch_article_contents(selected_article):
+            if selected_article != "Provide your own input" and article_text == fetch_article_contents(
+                    selected_article):
                 st.session_state.unchanged_text = True
                 summary_content = fetch_summary_contents(selected_article)
             else:
-                summary_content = generate_abstractive_summary(article_text, type="beam", do_sample=True, num_beams=15, no_repeat_ngram_size=4)
+                summary_content = generate_abstractive_summary(article_text, type="beam", do_sample=True, num_beams=15,
+                                                               no_repeat_ngram_size=4)
                 st.session_state.unchanged_text = False
             summary_displayed = display_summary(summary_content)
             st.write("**Generated summary:**", summary_displayed, unsafe_allow_html=True)
@@ -428,10 +397,11 @@ if summarize_button:
 
     # DEPENDENCY PARSING PART
     st.header("Dependency comparison")
-    st.markdown("The second method we use for post-processing is called **Dependency parsing**: the process in which the "
-                "grammatical structure in a sentence is analysed, to find out related words as well as the type of the "
-                "relationship between them. For the sentence “Jan’s wife is called Sarah” you would get the following "
-                "dependency graph:")
+    st.markdown(
+        "The second method we use for post-processing is called **Dependency parsing**: the process in which the "
+        "grammatical structure in a sentence is analysed, to find out related words as well as the type of the "
+        "relationship between them. For the sentence “Jan’s wife is called Sarah” you would get the following "
+        "dependency graph:")
 
     # TODO: I wonder why the first doesn't work but the second does (it doesn't show deps otherwise)
     # st.image("ExampleParsing.svg")
@@ -442,14 +412,15 @@ if summarize_button:
                 "are still correct. “The borders of Ukraine” have a different dependency between “borders” and “Ukraine” "
                 "than “Ukraine’s borders”, while both descriptions have the same meaning. So just matching all "
                 "dependencies between article and summary (as we did with entity matching) would not be a robust method.")
-    st.markdown("However, by empirical testing, we have found that there are certain dependencies which can be used for "
-                "such matching techniques. When unmatched, these specific dependencies are often an indication of a "
-                "wrongly constructed sentence. **Should I explain this more/better or is it enough that I explain by "
-                "example specific run throughs?**. We found 2(/3 TODO) common dependencies which, when present in the "
-                "summary but not in the article, are highly indicative of factualness errors. Furthermore, we only check "
-                "dependencies between an existing **entity** and its direct connections. Below we highlight all unmatched "
-                "dependencies that satisfy the discussed constraints. We also discuss the specific results for the "
-                "currently selected article.")
+    st.markdown(
+        "However, by empirical testing, we have found that there are certain dependencies which can be used for "
+        "such matching techniques. When unmatched, these specific dependencies are often an indication of a "
+        "wrongly constructed sentence. **Should I explain this more/better or is it enough that I explain by "
+        "example specific run throughs?**. We found 2(/3 TODO) common dependencies which, when present in the "
+        "summary but not in the article, are highly indicative of factualness errors. Furthermore, we only check "
+        "dependencies between an existing **entity** and its direct connections. Below we highlight all unmatched "
+        "dependencies that satisfy the discussed constraints. We also discuss the specific results for the "
+        "currently selected article.")
     with st.spinner("Doing dependency parsing..."):
         # TODO RIGHT IF FUNCTION (IF EXAMPLE AND IF INPUT UNCHANGED)
         # if selected_article == 'article11':
@@ -474,12 +445,13 @@ if summarize_button:
 
     # OUTRO/CONCLUSION
     st.header("Wrapping up")
-    st.markdown("We have presented 2 methods that try to improve summaries via post-processing steps. Entity matching can "
-                "be used to solve hallucinations, while dependency comparison can be used to filter out some bad "
-                "sentences (and thus worse summaries). These methods highlight the possibilities of post-processing "
-                "AI-made summaries, but are only a basic introduction. As the methods were empirically tested they are "
-                "definitely not sufficiently robust for general use-cases. (something about that we tested also RE and "
-                "maybe other things).")
+    st.markdown(
+        "We have presented 2 methods that try to improve summaries via post-processing steps. Entity matching can "
+        "be used to solve hallucinations, while dependency comparison can be used to filter out some bad "
+        "sentences (and thus worse summaries). These methods highlight the possibilities of post-processing "
+        "AI-made summaries, but are only a basic introduction. As the methods were empirically tested they are "
+        "definitely not sufficiently robust for general use-cases. (something about that we tested also RE and "
+        "maybe other things).")
     st.markdown("####")
     st.markdown("Below we generated 5 different kind of summaries from the article in which their ranks are estimated, "
                 "and hopefully the best summary (read: the one that a human would prefer or indicate as the best one) "

custom_renderer.py

@@ -1,6 +1,4 @@
 from typing import Dict
-
-import spacy
 from PIL import ImageFont