Allow free input and some fixes

app.py

@@ -1,5 +1,5 @@
 import random
-from typing import AnyStr
+from typing import AnyStr, List, Dict
 # import tensorflow_hub as hub
 
 import itertools
@@ -42,8 +42,8 @@ def get_sentence_embedding_model():
 
 @st.experimental_singleton
 def get_spacy():
-    #nlp = spacy.load('en_core_web_lg')
-    nlp = en_core_web_sm.load()
+    # nlp = spacy.load('en_core_web_lg')
+    nlp = en_core_web_lg.load()
     return nlp
 
 
@@ -62,6 +62,15 @@ def get_transformer_pipeline():
     return pipeline("ner", model=model, tokenizer=tokenizer, grouped_entities=True)
 
 
+@st.experimental_singleton
+def get_summarizer_model():
+    model_name = 'google/pegasus-cnn_dailymail'
+    summarizer_model = pipeline("summarization", model=model_name, tokenizer=model_name,
+                                device=0 if torch.cuda.is_available() else -1)
+
+    return summarizer_model
+
+
 # Page setup
 st.set_page_config(
     page_title="Post-processing summarization fact checker",
@@ -81,10 +90,14 @@ def list_all_article_names() -> list:
     for file in sorted(os.listdir('./sample-articles/')):
         if file.endswith('.txt'):
             filenames.append(file.replace('.txt', ''))
+        # Append free use possibility:
+    filenames.append("Provide your own input")
     return filenames
 
 
 def fetch_article_contents(filename: str) -> AnyStr:
+    if (filename == "Provide your own input"):
+        return " "
     with open(f'./sample-articles/{filename.lower()}.txt', 'r') as f:
         data = f.read()
     return data
@@ -110,13 +123,12 @@ def fetch_dependency_specific_contents(filename: str) -> AnyStr:
 
 def fetch_dependency_svg(filename: str) -> AnyStr:
     with open(f'./dependency-images/{filename.lower()}.txt', 'r') as f:
-        #data = f.read()
-        lines=[line.rstrip() for line in f]
+        # data = f.read()
+        lines = [line.rstrip() for line in f]
     return lines
 
 
-def display_summary(article_name: str):
-    summary_content = fetch_summary_contents(article_name)
+def display_summary(summary_content: str):
     st.session_state.summary_output = summary_content
     soup = BeautifulSoup(summary_content, features="html.parser")
     HTML_WRAPPER = """<div style="overflow-x: auto; border: 1px solid #e6e9ef; border-radius: 0.25rem; padding: 1rem; margin-bottom: 2.5rem">{}</div>"""
@@ -161,13 +173,15 @@ def get_all_entities(text):
 
 
 # TODO: this functionality can be cached (e.g. by storing html file output) if wanted (or just store list of entities idk)
-def get_and_compare_entities(article_name: str):
-    article_content = fetch_article_contents(article_name)
+def get_and_compare_entities():
+    #article_content = fetch_article_contents(article_name)
+    article_content = st.session_state.article_text
     all_entities_per_sentence = get_all_entities_per_sentence(article_content)
     # st.session_state.entities_per_sentence_article = all_entities_per_sentence
     entities_article = list(itertools.chain.from_iterable(all_entities_per_sentence))
 
-    summary_content = fetch_summary_contents(article_name)
+    #summary_content = fetch_summary_contents(article_name)
+    summary_content = st.session_state.summary_output
     all_entities_per_sentence = get_all_entities_per_sentence(summary_content)
     # st.session_state.entities_per_sentence_summary = all_entities_per_sentence
     entities_summary = list(itertools.chain.from_iterable(all_entities_per_sentence))
@@ -179,7 +193,7 @@ def get_and_compare_entities(article_name: str):
         if any(entity.lower() in substring_entity.lower() for substring_entity in entities_article):
             matched_entities.append(entity)
         elif any(
-                np.inner(sentence_embedding_model.encode(entity), sentence_embedding_model.encode(art_entity)) > 0.9 for
+                np.inner(sentence_embedding_model.encode(entity, show_progress_bar=False), sentence_embedding_model.encode(art_entity, show_progress_bar=False)) > 0.9 for
                 art_entity in entities_article):
             matched_entities.append(entity)
         else:
@@ -187,14 +201,14 @@ def get_and_compare_entities(article_name: str):
     return matched_entities, unmatched_entities
 
 
-def highlight_entities(article_name: str):
-    summary_content = fetch_summary_contents(article_name)
-
+def highlight_entities():
+    #summary_content = fetch_summary_contents(article_name)
+    summary_content = st.session_state.summary_output
     markdown_start_red = "<mark class=\"entity\" style=\"background: rgb(238, 135, 135);\">"
     markdown_start_green = "<mark class=\"entity\" style=\"background: rgb(121, 236, 121);\">"
     markdown_end = "</mark>"
 
-    matched_entities, unmatched_entities = get_and_compare_entities(article_name)
+    matched_entities, unmatched_entities = get_and_compare_entities()
 
     for entity in matched_entities:
         summary_content = summary_content.replace(entity, markdown_start_green + entity + markdown_end)
@@ -209,10 +223,9 @@ def highlight_entities(article_name: str):
     return HTML_WRAPPER.format(soup)
 
 
-def render_dependency_parsing(text: str):
-    html = render_sentence_custom(text)
+def render_dependency_parsing(text: Dict):
+    html = render_sentence_custom(text, nlp)
     html = html.replace("\n\n", "\n")
-    print(get_svg(html))
     st.write(get_svg(html), unsafe_allow_html=True)
 
 
@@ -237,7 +250,6 @@ def check_dependency(article: bool):
         start_id = sentence.start
         end_id = sentence.end
         for t in tok_l:
-            # print(t)
             if t["id"] < start_id or t["id"] > end_id:
                 continue
             head = tok_l[t['head']]
@@ -261,7 +273,6 @@ def check_dependency(article: bool):
                                                   "identifier": identifier, "sentence": str(sentence)})
                 else:
                     continue
-    # print(f'NOW TEST LIST DICT: {test_list_dict_output}')
     return test_list_dict_output
     # return all_deps
 
@@ -273,6 +284,38 @@ def is_valid_url(url: str) -> bool:
     return True
 
 
+def render_svg(svg_file):
+    with open(svg_file, "r") as f:
+        lines = f.readlines()
+        svg = "".join(lines)
+
+        # """Renders the given svg string."""
+        b64 = base64.b64encode(svg.encode("utf-8")).decode("utf-8")
+        html = r'<img src="data:image/svg+xml;base64,%s"/>' % b64
+        return html
+
+
+def generate_abstractive_summary(text, type, min_len=120, max_len=512, **kwargs):
+    summarization_model = get_summarizer_model()
+    text = text.strip().replace("\n", " ")
+    if type == "top_p":
+        text = summarization_model(text, min_length=min_len,
+                                   max_length=max_len,
+                                   top_k=50, top_p=0.95, clean_up_tokenization_spaces=True)
+    elif type == "greedy":
+        text = summarization_model(text, min_length=min_len,
+                                   max_length=max_len, clean_up_tokenization_spaces=True)
+    elif type == "top_k":
+        text = summarization_model(text, min_length=min_len, max_length=max_len, top_k=50,
+                                   clean_up_tokenization_spaces=True)
+    elif type == "beam":
+        text = summarization_model(text, min_length=min_len,
+                                   max_length=max_len,
+                                   clean_up_tokenization_spaces=True, **kwargs)
+    summary = text[0]['summary_text'].replace("<n>", " ")
+    return summary
+
+
 # Start session
 if 'results' not in st.session_state:
     st.session_state.results = []
@@ -298,12 +341,11 @@ metric, indicating the trustworthiness of the generated summary. Throughout this
 results for some methods on specific examples. These text blocks will be indicated and they change according to the 
 currently selected article.""")
 
-
 sentence_embedding_model = get_sentence_embedding_model()
 # tagger = get_flair_tagger()
 ner_model = get_transformer_pipeline()
 nlp = get_spacy()
-#nlp = en_core_web_sm.load()
+# nlp = en_core_web_sm.load()
 
 # GENERATING SUMMARIES PART
 st.header("Generating summaries")
@@ -321,131 +363,125 @@ article_text = st.text_area(
     height=150
 )
 
-st.markdown("Below you can find the generated summary for the article. Based on empirical research, we will discuss "
-            "two main methods that detect some common errors. We can then score different summaries, to indicate how "
-            "factual a summary is for a given article. The idea is that in production, you could generate a set of "
-            "summaries for the same article, with different parameters (or even different models). By using "
-            "post-processing error detection, we can then select the best possible summary.")
-if st.session_state.article_text:
-    with st.spinner('Generating summary...'):
-        # classify_comment(article_text, selected_model)
-
-        summary_displayed = display_summary(selected_article)
-
-        st.write("**Generated summary:**", summary_displayed, unsafe_allow_html=True)
-else:
-    st.error('**Error**: No comment to classify. Please provide a comment.',
-             help="Generate summary for the given article text")
-
-
-def render_svg(svg_file):
-    with open(svg_file, "r") as f:
-        lines = f.readlines()
-        svg = "".join(lines)
-
-        # """Renders the given svg string."""
-        b64 = base64.b64encode(svg.encode("utf-8")).decode("utf-8")
-        html = r'<img src="data:image/svg+xml;base64,%s"/>' % b64
-        return html
-
-
-# ENTITY MATCHING PART
-st.header("Entity matching")
-st.markdown("The first method we will discuss is called **Named Entity Recognition** (NER). NER is the task of "
-            "identifying and categorising key information (entities) in text. An entity can be a singular word or a "
-            "series of words that consistently refers to the same thing. Common entity classes are person names, "
-            "organisations, locations and so on. By applying NER to both the article and its summary, we can spot "
-            "possible **hallucinations**. Hallucinations are words generated by the model that are not supported by "
-            "the source input. In theory all entities in the summary (such as dates, locations and so on), "
-            "should also be present in the article. Thus we can extract all entities from the summary and compare "
-            "them to the entities of the original article, spotting potential hallucinations. The more unmatched "
-            "entities we find, the lower the factualness score of the summary. ")
-with st.spinner("Calculating and matching entities..."):
-    entity_match_html = highlight_entities(selected_article)
-    st.write(entity_match_html, unsafe_allow_html=True)
-    red_text = """<font color="black"><span style="background-color: rgb(238, 135, 135); opacity: 
-    1;">red</span></font> """
-    green_text = """<font color="black">
-        <span style="background-color: rgb(121, 236, 121); opacity: 1;">green</span>
-    </font>"""
-
-    markdown_start_red = "<mark class=\"entity\" style=\"background: rgb(238, 135, 135);\">"
-    markdown_start_green = "<mark class=\"entity\" style=\"background: rgb(121, 236, 121);\">"
-    st.markdown("We call this technique “entity matching” and here you can see what this looks like when we apply "
-                "this method on the summary. Entities in the summary are marked  " + green_text + " when the entity "
-                                                                                                  "also exists in the "
-                                                                                                  "article, "
-                                                                                                  "while unmatched "
-                                                                                                  "entities are "
-                                                                                                  "marked " +
-                red_text + ". Several of the example articles and their summaries indicate different errors we find "
-                           "by using this technique. Based on which article you choose, we provide a short "
-                           "explanation of the results below.",
-                unsafe_allow_html=True)
-    entity_specific_text = fetch_entity_specific_contents(selected_article)
-    soup = BeautifulSoup(entity_specific_text, features="html.parser")
-    HTML_WRAPPER = """<div style="overflow-x: auto; border: 1px solid #e6e9ef; border-radius: 0.25rem; padding: 1rem;
-    margin-bottom: 2.5rem">{}</div> """
-    st.write("💡👇 **Specific example explanation** 👇💡", HTML_WRAPPER.format(soup), unsafe_allow_html=True)
-
-# DEPENDENCY PARSING PART
-st.header("Dependency comparison")
-st.markdown("The second method we use for post-processing is called **Dependency parsing**: the process in which the "
-            "grammatical structure in a sentence is analysed, to find out related words as well as the type of the "
-            "relationship between them. For the sentence “Jan’s wife is called Sarah” you would get the following "
-            "dependency graph:")
-
-# TODO: I wonder why the first doesn't work but the second does (it doesn't show deps otherwise)
-# st.image("ExampleParsing.svg")
-st.write(render_svg('ExampleParsing.svg'), unsafe_allow_html=True)
-st.markdown("Here, “Jan” is the “poss” (possession modifier) of “wife”. If suddenly the summary would read “Jan’s "
-            "husband…”, there would be a dependency in the summary that is non-existent in the article itself (namely "
-            "“Jan” is the “poss” of “husband”). However, often new dependencies are introduced in the summary that "
-            "are still correct. “The borders of Ukraine” have a different dependency between “borders” and “Ukraine” "
-            "than “Ukraine’s borders”, while both descriptions have the same meaning. So just matching all "
-            "dependencies between article and summary (as we did with entity matching) would not be a robust method.")
-st.markdown("However, by empirical testing, we have found that there are certain dependencies which can be used for "
-            "such matching techniques. When unmatched, these specific dependencies are often an indication of a "
-            "wrongly constructed sentence. **Should I explain this more/better or is it enough that I explain by "
-            "example specific run throughs?**. We found 2(/3 TODO) common dependencies which, when present in the "
-            "summary but not in the article, are highly indicative of factualness errors. Furthermore, we only check "
-            "dependencies between an existing **entity** and its direct connections. Below we highlight all unmatched "
-            "dependencies that satisfy the discussed constraints. We also discuss the specific results for the "
-            "currently selected article.")
-with st.spinner("Doing dependency parsing..."):
-    # TODO RIGHT IF FUNCTION (IF EXAMPLE AND IF INPUT UNCHANGED)
-    #if selected_article == 'article11':
-    if True:
-        for cur_svg_image in fetch_dependency_svg(selected_article):
-            st.write(cur_svg_image, unsafe_allow_html=True)
+summarize_button = st.button(label='Process article content', help="Generates summary and applies entity matching and dependency parsing for given article")
+
+if summarize_button:
+    st.session_state.article_text = article_text
+    st.markdown("Below you can find the generated summary for the article. Based on empirical research, we will discuss "
+                "two main methods that detect some common errors. We can then score different summaries, to indicate how "
+                "factual a summary is for a given article. The idea is that in production, you could generate a set of "
+                "summaries for the same article, with different parameters (or even different models). By using "
+                "post-processing error detection, we can then select the best possible summary.")
+    if st.session_state.article_text:
+        with st.spinner('Generating summary...'):
+            # classify_comment(article_text, selected_model)
+            if selected_article != "Provide your own input" and article_text == fetch_article_contents(selected_article):
+                st.session_state.unchanged_text = True
+                summary_content = fetch_summary_contents(selected_article)
+            else:
+                summary_content = generate_abstractive_summary(article_text, type="beam", do_sample=True, num_beams=15, no_repeat_ngram_size=4)
+                st.session_state.unchanged_text = False
+            summary_displayed = display_summary(summary_content)
+            st.write("**Generated summary:**", summary_displayed, unsafe_allow_html=True)
     else:
-        summary_deps = check_dependency(False)
-        article_deps = check_dependency(True)
-        total_unmatched_deps = []
-        for summ_dep in summary_deps:
-            if not any(summ_dep['identifier'] in art_dep['identifier'] for art_dep in article_deps):
-                total_unmatched_deps.append(summ_dep)
-        # print(f'ALL UNMATCHED DEPS ARE: {total_unmatched_deps}')
-        # render_dependency_parsing(check_dependency(False))
-        if total_unmatched_deps:
-            for current_drawing_list in total_unmatched_deps:
-                render_dependency_parsing(current_drawing_list)
-    dep_specific_text = fetch_dependency_specific_contents(selected_article)
-    soup = BeautifulSoup(dep_specific_text, features="html.parser")
-    HTML_WRAPPER = """<div style="overflow-x: auto; border: 1px solid #e6e9ef; border-radius: 0.25rem; padding: 1rem;
-    margin-bottom: 2.5rem">{}</div> """
-    st.write("💡👇 **Specific example explanation** 👇💡", HTML_WRAPPER.format(soup), unsafe_allow_html=True)
-
-# OUTRO/CONCLUSION
-st.header("Wrapping up")
-st.markdown("We have presented 2 methods that try to improve summaries via post-processing steps. Entity matching can "
-            "be used to solve hallucinations, while dependency comparison can be used to filter out some bad "
-            "sentences (and thus worse summaries). These methods highlight the possibilities of post-processing "
-            "AI-made summaries, but are only a basic introduction. As the methods were empirically tested they are "
-            "definitely not sufficiently robust for general use-cases. (something about that we tested also RE and "
-            "maybe other things).")
-st.markdown("####")
-st.markdown("Below we generated 5 different kind of summaries from the article in which their ranks are estimated, "
-            "and hopefully the best summary (read: the one that a human would prefer or indicate as the best one) "
-            "will be at the top. TODO: implement this (at the end I think) and also put something in the text with "
-            "the actual parameters or something? ")
+        st.error('**Error**: No comment to classify. Please provide a comment.')
+
+    # ENTITY MATCHING PART
+    st.header("Entity matching")
+    st.markdown("The first method we will discuss is called **Named Entity Recognition** (NER). NER is the task of "
+                "identifying and categorising key information (entities) in text. An entity can be a singular word or a "
+                "series of words that consistently refers to the same thing. Common entity classes are person names, "
+                "organisations, locations and so on. By applying NER to both the article and its summary, we can spot "
+                "possible **hallucinations**. Hallucinations are words generated by the model that are not supported by "
+                "the source input. In theory all entities in the summary (such as dates, locations and so on), "
+                "should also be present in the article. Thus we can extract all entities from the summary and compare "
+                "them to the entities of the original article, spotting potential hallucinations. The more unmatched "
+                "entities we find, the lower the factualness score of the summary. ")
+    with st.spinner("Calculating and matching entities..."):
+        entity_match_html = highlight_entities()
+        st.write(entity_match_html, unsafe_allow_html=True)
+        red_text = """<font color="black"><span style="background-color: rgb(238, 135, 135); opacity: 
+        1;">red</span></font> """
+        green_text = """<font color="black">
+            <span style="background-color: rgb(121, 236, 121); opacity: 1;">green</span>
+        </font>"""
+
+        markdown_start_red = "<mark class=\"entity\" style=\"background: rgb(238, 135, 135);\">"
+        markdown_start_green = "<mark class=\"entity\" style=\"background: rgb(121, 236, 121);\">"
+        st.markdown("We call this technique “entity matching” and here you can see what this looks like when we apply "
+                    "this method on the summary. Entities in the summary are marked  " + green_text + " when the entity "
+                                                                                                      "also exists in the "
+                                                                                                      "article, "
+                                                                                                      "while unmatched "
+                                                                                                      "entities are "
+                                                                                                      "marked " +
+                    red_text + ". Several of the example articles and their summaries indicate different errors we find "
+                               "by using this technique. Based on which article you choose, we provide a short "
+                               "explanation of the results below.",
+                    unsafe_allow_html=True)
+        if st.session_state.unchanged_text:
+            entity_specific_text = fetch_entity_specific_contents(selected_article)
+            soup = BeautifulSoup(entity_specific_text, features="html.parser")
+            HTML_WRAPPER = """<div style="overflow-x: auto; border: 1px solid #e6e9ef; border-radius: 0.25rem; padding: 1rem;
+            margin-bottom: 2.5rem">{}</div> """
+            st.write("💡👇 **Specific example explanation** 👇💡", HTML_WRAPPER.format(soup), unsafe_allow_html=True)
+
+    # DEPENDENCY PARSING PART
+    st.header("Dependency comparison")
+    st.markdown("The second method we use for post-processing is called **Dependency parsing**: the process in which the "
+                "grammatical structure in a sentence is analysed, to find out related words as well as the type of the "
+                "relationship between them. For the sentence “Jan’s wife is called Sarah” you would get the following "
+                "dependency graph:")
+
+    # TODO: I wonder why the first doesn't work but the second does (it doesn't show deps otherwise)
+    # st.image("ExampleParsing.svg")
+    st.write(render_svg('ExampleParsing.svg'), unsafe_allow_html=True)
+    st.markdown("Here, “Jan” is the “poss” (possession modifier) of “wife”. If suddenly the summary would read “Jan’s "
+                "husband…”, there would be a dependency in the summary that is non-existent in the article itself (namely "
+                "“Jan” is the “poss” of “husband”). However, often new dependencies are introduced in the summary that "
+                "are still correct. “The borders of Ukraine” have a different dependency between “borders” and “Ukraine” "
+                "than “Ukraine’s borders”, while both descriptions have the same meaning. So just matching all "
+                "dependencies between article and summary (as we did with entity matching) would not be a robust method.")
+    st.markdown("However, by empirical testing, we have found that there are certain dependencies which can be used for "
+                "such matching techniques. When unmatched, these specific dependencies are often an indication of a "
+                "wrongly constructed sentence. **Should I explain this more/better or is it enough that I explain by "
+                "example specific run throughs?**. We found 2(/3 TODO) common dependencies which, when present in the "
+                "summary but not in the article, are highly indicative of factualness errors. Furthermore, we only check "
+                "dependencies between an existing **entity** and its direct connections. Below we highlight all unmatched "
+                "dependencies that satisfy the discussed constraints. We also discuss the specific results for the "
+                "currently selected article.")
+    with st.spinner("Doing dependency parsing..."):
+        # TODO RIGHT IF FUNCTION (IF EXAMPLE AND IF INPUT UNCHANGED)
+        # if selected_article == 'article11':
+        if st.session_state.unchanged_text:
+            for cur_svg_image in fetch_dependency_svg(selected_article):
+                st.write(cur_svg_image, unsafe_allow_html=True)
+            dep_specific_text = fetch_dependency_specific_contents(selected_article)
+            soup = BeautifulSoup(dep_specific_text, features="html.parser")
+            HTML_WRAPPER = """<div style="overflow-x: auto; border: 1px solid #e6e9ef; border-radius: 0.25rem; padding: 1rem;
+            margin-bottom: 2.5rem">{}</div> """
+            st.write("💡👇 **Specific example explanation** 👇💡", HTML_WRAPPER.format(soup), unsafe_allow_html=True)
+        else:
+            summary_deps = check_dependency(False)
+            article_deps = check_dependency(True)
+            total_unmatched_deps = []
+            for summ_dep in summary_deps:
+                if not any(summ_dep['identifier'] in art_dep['identifier'] for art_dep in article_deps):
+                    total_unmatched_deps.append(summ_dep)
+            if total_unmatched_deps:
+                for current_drawing_list in total_unmatched_deps:
+                    render_dependency_parsing(current_drawing_list)
+
+    # OUTRO/CONCLUSION
+    st.header("Wrapping up")
+    st.markdown("We have presented 2 methods that try to improve summaries via post-processing steps. Entity matching can "
+                "be used to solve hallucinations, while dependency comparison can be used to filter out some bad "
+                "sentences (and thus worse summaries). These methods highlight the possibilities of post-processing "
+                "AI-made summaries, but are only a basic introduction. As the methods were empirically tested they are "
+                "definitely not sufficiently robust for general use-cases. (something about that we tested also RE and "
+                "maybe other things).")
+    st.markdown("####")
+    st.markdown("Below we generated 5 different kind of summaries from the article in which their ranks are estimated, "
+                "and hopefully the best summary (read: the one that a human would prefer or indicate as the best one) "
+                "will be at the top. TODO: implement this (at the end I think) and also put something in the text with "
+                "the actual parameters or something? ")

custom_renderer.py

@@ -1,10 +1,8 @@
-from typing import Dict, Any
+from typing import Dict
 
-import numpy as np
 import spacy
 from PIL import ImageFont
 
-from spacy.tokens import Doc
 
 
 def get_pil_text_size(text, font_size, font_name):
@@ -78,8 +76,7 @@ def get_arrowhead(direction: str, x: int, y: int, end: int) -> str:
     return f"M{p1},{y + 2} L{p2},{y - arrow_width} {p3},{y - arrow_width}"
 
 
-# parsed = [{'words': [{'text': 'The', 'tag': 'DET', 'lemma': None}, {'text': 'OnePlus', 'tag': 'PROPN', 'lemma': None}, {'text': '10', 'tag': 'NUM', 'lemma': None}, {'text': 'Pro', 'tag': 'PROPN', 'lemma': None}, {'text': 'is', 'tag': 'AUX', 'lemma': None}, {'text': 'the', 'tag': 'DET', 'lemma': None}, {'text': 'company', 'tag': 'NOUN', 'lemma': None}, {'text': "'s", 'tag': 'PART', 'lemma': None}, {'text': 'first', 'tag': 'ADJ', 'lemma': None}, {'text': 'flagship', 'tag': 'NOUN', 'lemma': None}, {'text': 'phone.', 'tag': 'NOUN', 'lemma': None}], 'arcs': [{'start': 0, 'end': 3, 'label': 'det', 'dir': 'left'}, {'start': 1, 'end': 3, 'label': 'nmod', 'dir': 'left'}, {'start': 1, 'end': 2, 'label': 'nummod', 'dir': 'right'}, {'start': 3, 'end': 4, 'label': 'nsubj', 'dir': 'left'}, {'start': 5, 'end': 6, 'label': 'det', 'dir': 'left'}, {'start': 6, 'end': 10, 'label': 'poss', 'dir': 'left'}, {'start': 6, 'end': 7, 'label': 'case', 'dir': 'right'}, {'start': 8, 'end': 10, 'label': 'amod', 'dir': 'left'}, {'start': 9, 'end': 10, 'label': 'compound', 'dir': 'left'}, {'start': 4, 'end': 10, 'label': 'attr', 'dir': 'right'}], 'settings': {'lang': 'en', 'direction': 'ltr'}}]
-def render_sentence_custom(unmatched_list: Dict):
+def render_sentence_custom(unmatched_list: Dict, nlp):
     TPL_DEP_WORDS = """
   <text class="displacy-token" fill="currentColor" text-anchor="start" y="{y}">
       <tspan class="displacy-word" fill="currentColor" x="{x}">{text}</tspan>
@@ -91,64 +88,14 @@ def render_sentence_custom(unmatched_list: Dict):
   <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xml:lang="{lang}" id="{id}" class="displacy" width="{width}" height="{height}" direction="{dir}" style="max-width: none; height: {height}px; color: {color}; background: {bg}; font-family: {font}; direction: {dir}">{content}</svg>
   """
     arcs_svg = []
-    nlp = spacy.load('en_core_web_lg')
+    #nlp = spacy.load('en_core_web_lg')
     doc = nlp(unmatched_list["sentence"])
-    # words = {}
-    # unmatched_list = [parse_deps(doc)]
-    # #print(parsed)
-    # for i, p in enumerate(unmatched_list):
-    #     arcs = p["arcs"]
-    #     words = p["words"]
-    # for i, a in enumerate(arcs):
-    #     #CHECK CERTAIN DEPS (ALSO ADD/CHANGE BELOW WHEN CHANGING HERE)
-    #     if a["label"] == "amod":
-    #         couples = (a["start"], a["end"])
-    #     elif a["label"] == "pobj":
-    #         couples = (a["start"], a["end"])
-    # #couples = (3,5)
-    #
-    # x_value_counter = 10
-    # index_counter = 0
-    # svg_words = []
-    # coords_test = []
-    # for i, word in enumerate(words):
-    #     word = word["text"]
-    #     word = word + " "
-    #     pixel_x_length = get_pil_text_size(word, 16, 'arial.ttf')[0]
-    #     svg_words.append(TPL_DEP_WORDS.format(text=word, tag="", x=x_value_counter, y=70))
-    #     if index_counter >= couples[0] and index_counter <= couples[1]:
-    #         coords_test.append(x_value_counter)
-    #         x_value_counter += 50
-    #     index_counter += 1
-    #     x_value_counter += pixel_x_length + 4
-    # for i, a in enumerate(arcs):
-    #     if a["label"] == "amod":
-    #         arcs_svg.append(render_arrow(a["label"], coords_test[0], coords_test[-1], a["dir"], i))
-    #     elif a["label"] == "pobj":
-    #         arcs_svg.append(render_arrow(a["label"], coords_test[0], coords_test[-1], a["dir"], i))
-    #
-    # content = "".join(svg_words) + "".join(arcs_svg)
-    #
-    # full_svg = TPL_DEP_SVG.format(
-    #     id=0,
-    #     width=1200, #600
-    #     height=250, #125
-    #     color="#00000",
-    #     bg="#ffffff",
-    #     font="Arial",
-    #     content=content,
-    #     dir="ltr",
-    #     lang="en",
-    # )
 
     x_value_counter = 10
     index_counter = 0
     svg_words = []
-    words = unmatched_list["sentence"].split(" ")
+    #words = unmatched_list["sentence"].split(" ")
     coords_test = []
-    #print(unmatched_list)
-    #print(words)
-    #print("NOW")
     direction_current = "rtl"
     if unmatched_list["cur_word_index"] < unmatched_list["target_word_index"]:
         min_index = unmatched_list["cur_word_index"]
@@ -169,8 +116,6 @@ def render_sentence_custom(unmatched_list: Dict):
         index_counter += 1
         x_value_counter += pixel_x_length + 4
 
-    # TODO: DYNAMIC DIRECTION MAKING (SHOULD GIVE WITH DICT I THINK)
-    #print(coords_test)
     arcs_svg.append(render_arrow(unmatched_list['dep'], coords_test[0], coords_test[-1], direction_current, i))
 
     content = "".join(svg_words) + "".join(arcs_svg)
@@ -189,69 +134,3 @@ def render_sentence_custom(unmatched_list: Dict):
     return full_svg
 
 
-def parse_deps(orig_doc: Doc, options: Dict[str, Any] = {}) -> Dict[str, Any]:
-    """Generate dependency parse in {'words': [], 'arcs': []} format.
-
-    doc (Doc): Document do parse.
-    RETURNS (dict): Generated dependency parse keyed by words and arcs.
-    """
-    doc = Doc(orig_doc.vocab).from_bytes(orig_doc.to_bytes(exclude=["user_data"]))
-    if not doc.has_annotation("DEP"):
-        print("WARNING")
-    if options.get("collapse_phrases", False):
-        with doc.retokenize() as retokenizer:
-            for np in list(doc.noun_chunks):
-                attrs = {
-                    "tag": np.root.tag_,
-                    "lemma": np.root.lemma_,
-                    "ent_type": np.root.ent_type_,
-                }
-                retokenizer.merge(np, attrs=attrs)
-    if options.get("collapse_punct", True):
-        spans = []
-        for word in doc[:-1]:
-            if word.is_punct or not word.nbor(1).is_punct:
-                continue
-            start = word.i
-            end = word.i + 1
-            while end < len(doc) and doc[end].is_punct:
-                end += 1
-            span = doc[start:end]
-            spans.append((span, word.tag_, word.lemma_, word.ent_type_))
-        with doc.retokenize() as retokenizer:
-            for span, tag, lemma, ent_type in spans:
-                attrs = {"tag": tag, "lemma": lemma, "ent_type": ent_type}
-                retokenizer.merge(span, attrs=attrs)
-    fine_grained = options.get("fine_grained")
-    add_lemma = options.get("add_lemma")
-    words = [
-        {
-            "text": w.text,
-            "tag": w.tag_ if fine_grained else w.pos_,
-            "lemma": w.lemma_ if add_lemma else None,
-        }
-        for w in doc
-    ]
-    arcs = []
-    for word in doc:
-        if word.i < word.head.i:
-            arcs.append(
-                {"start": word.i, "end": word.head.i, "label": word.dep_, "dir": "left"}
-            )
-        elif word.i > word.head.i:
-            arcs.append(
-                {
-                    "start": word.head.i,
-                    "end": word.i,
-                    "label": word.dep_,
-                    "dir": "right",
-                }
-            )
-    return {"words": words, "arcs": arcs, "settings": get_doc_settings(orig_doc)}
-
-
-def get_doc_settings(doc: Doc) -> Dict[str, Any]:
-    return {
-        "lang": doc.lang_,
-        "direction": doc.vocab.writing_system.get("direction", "ltr"),
-    }

requirements.txt

@@ -5,5 +5,6 @@ transformers-interpret==0.5.2
 sentence-transformers==2.2.0
 spacy==3.0.0
 spacy_streamlit==1.0.3
-en_core_web_lg @ https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.0.0/en_core_web_lg-3.0.0.tar.gz
-en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.0.0/en_core_web_sm-3.0.0.tar.gz
+###### en_core_web_lg @ https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.0.0/en_core_web_lg-3.0.0.tar.gz
+en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.0.0/en_core_web_sm-3.0.0.tar.gz
+en_core_web_lg @ https://huggingface.co/spacy/en_core_web_lg/resolve/main/en_core_web_lg-any-py3-none-any.whl