Add some more whitespaces and implement scoring and showing different summaries at the end of the blogpost

app.py

@@ -10,6 +10,7 @@ import numpy as np
 import base64
 
 from spacy_streamlit.util import get_svg
+from streamlit.proto.SessionState_pb2 import SessionState
 
 from custom_renderer import render_sentence_custom
 from sentence_transformers import SentenceTransformer
@@ -174,15 +175,28 @@ def get_and_compare_entities(first_time: bool):
 
     matched_entities = list(dict.fromkeys(matched_entities))
     unmatched_entities = list(dict.fromkeys(unmatched_entities))
+
+    matched_entities_to_remove = []
+    unmatched_entities_to_remove = []
+
     for entity in matched_entities:
         for substring_entity in matched_entities:
             if entity != substring_entity and entity.lower() in substring_entity.lower():
-                matched_entities.remove(entity)
+                matched_entities_to_remove.append(entity)
 
     for entity in unmatched_entities:
         for substring_entity in unmatched_entities:
             if entity != substring_entity and entity.lower() in substring_entity.lower():
-                unmatched_entities.remove(entity)
+                unmatched_entities_to_remove.append(entity)
+
+    matched_entities_to_remove = list(dict.fromkeys(matched_entities_to_remove))
+    unmatched_entities_to_remove = list(dict.fromkeys(unmatched_entities_to_remove))
+
+    for entity in matched_entities_to_remove:
+        matched_entities.remove(entity)
+    for entity in unmatched_entities_to_remove:
+        unmatched_entities.remove(entity)
+
     return matched_entities, unmatched_entities
 
 
@@ -292,7 +306,7 @@ st.title('📜 Summarization fact checker 📜')
 # INTRODUCTION
 st.header("🧑‍🏫 Introduction")
 
-introduction_checkbox = st.checkbox("Show introduction text", value = True)
+introduction_checkbox = st.checkbox("Show introduction text", value=True)
 if introduction_checkbox:
     st.markdown("""
     Recent work using 🤖 **transformers** 🤖 on large text corpora has shown great success when fine-tuned on 
@@ -346,6 +360,7 @@ if summarize_button:
         "factual a summary is for a given article. The idea is that in production, you could generate a set of "
         "summaries for the same article, with different parameters (or even different models). By using "
         "post-processing error detection, we can then select the best possible summary.*")
+    st.markdown("####")
     if st.session_state.article_text:
         with st.spinner('Generating summary, this might take a while...'):
             if selected_article != "Provide your own input" and article_text == fetch_article_contents(
@@ -381,6 +396,7 @@ if summarize_button:
                 "entities we find, the lower the factualness score of the summary. ")
     with st.spinner("Calculating and matching entities..."):
         entity_match_html = highlight_entities()
+        st.markdown("####")
         st.write(entity_match_html, unsafe_allow_html=True)
         red_text = """<font color="black"><span style="background-color: rgb(238, 135, 135); opacity: 
         1;">red</span></font> """
@@ -463,43 +479,79 @@ if summarize_button:
                 "empirically tested they are definitely not sufficiently robust for general use-cases.")
     st.markdown("####")
     st.markdown(
-        "(TODO) Below we generated 5 different kind of summaries from the article in which their ranks are estimated, "
-        "and hopefully the best summary (read: the one that a human would prefer or indicate as the best one) "
-        "will be at the top. TODO: implement this (at the end I think) and also put something in the text with "
-        "the actual parameters or something? ")
-
-    # with st.spinner("Calculating more summaries and scoring them, might take while..."):
-    #     # ENTITIES
-    #     _, amount_unmatched = get_and_compare_entities(False)
-    #     st.write(len(amount_unmatched))
-    #     st.write(amount_unmatched)
-    #
-    #     # DEPS
-    #     summary_deps = check_dependency(False)
-    #     article_deps = check_dependency(True)
-    #     total_unmatched_deps = []
-    #     for summ_dep in summary_deps:
-    #         if not any(summ_dep['identifier'] in art_dep['identifier'] for art_dep in article_deps):
-    #             total_unmatched_deps.append(summ_dep)
-    #
-    #     st.write(len(total_unmatched_deps))
-    #     st.write(total_unmatched_deps)
-    #
-    #     # FOR NEW GENERATED SUMMARY
-    #     st.session_state.summary_output = generate_abstractive_summary(st.session_state.article_text,
-    #                                                                    type="beam",
-    #                                                                    do_sample=True, num_beams=15,
-    #                                                                    no_repeat_ngram_size=5)
-    #     _, amount_unmatched = get_and_compare_entities(False)
-    #     st.write(len(amount_unmatched))
-    #     st.write(amount_unmatched)
-    #
-    #     summary_deps = check_dependency(False)
-    #     article_deps = check_dependency(True)
-    #     total_unmatched_deps = []
-    #     for summ_dep in summary_deps:
-    #         if not any(summ_dep['identifier'] in art_dep['identifier'] for art_dep in article_deps):
-    #             total_unmatched_deps.append(summ_dep)
-    #
-    #     st.write(len(total_unmatched_deps))
-    #     st.write(total_unmatched_deps)
+        "Below we generate 3 different kind of summaries, and based on the two discussed methods, their errors are "
+        "detected to estimate a factualness score. Based on this basic approach, "
+        "the best summary (read: the one that a human would prefer or indicate as the best one) "
+        "will hopefully be at the top. Summaries with the same scores will get the same rank displayed.")
+    st.markdown("####")
+
+    with st.spinner("Calculating more summaries and scoring them, this might take while..."):
+        summaries_list = []
+        deduction_points = []
+        # ENTITIES
+        _, amount_unmatched = get_and_compare_entities(False)
+
+        # DEPS
+        summary_deps = check_dependency(False)
+        article_deps = check_dependency(True)
+        total_unmatched_deps = []
+        for summ_dep in summary_deps:
+            if not any(summ_dep['identifier'] in art_dep['identifier'] for art_dep in article_deps):
+                total_unmatched_deps.append(summ_dep)
+
+        summaries_list.append(st.session_state.summary_output)
+        deduction_points.append(len(amount_unmatched) + len(total_unmatched_deps))
+
+        # FOR NEW GENERATED SUMMARY
+        st.session_state.summary_output = generate_abstractive_summary(st.session_state.article_text,
+                                                                       type="beam",
+                                                                       do_sample=True, num_beams=15,
+                                                                       no_repeat_ngram_size=5)
+        _, amount_unmatched = get_and_compare_entities(False)
+
+        summary_deps = check_dependency(False)
+        article_deps = check_dependency(True)
+        total_unmatched_deps = []
+        for summ_dep in summary_deps:
+            if not any(summ_dep['identifier'] in art_dep['identifier'] for art_dep in article_deps):
+                total_unmatched_deps.append(summ_dep)
+
+        summaries_list.append(st.session_state.summary_output)
+        deduction_points.append(len(amount_unmatched) + len(total_unmatched_deps))
+
+        # FOR NEW GENERATED SUMMARY
+        st.session_state.summary_output = generate_abstractive_summary(st.session_state.article_text,
+                                                                       type="top_p",
+                                                                       do_sample=True,
+                                                                       no_repeat_ngram_size=5)
+        _, amount_unmatched = get_and_compare_entities(False)
+
+        summary_deps = check_dependency(False)
+        article_deps = check_dependency(True)
+        total_unmatched_deps = []
+        for summ_dep in summary_deps:
+            if not any(summ_dep['identifier'] in art_dep['identifier'] for art_dep in article_deps):
+                total_unmatched_deps.append(summ_dep)
+
+        summaries_list.append(st.session_state.summary_output)
+        deduction_points.append(len(amount_unmatched) + len(total_unmatched_deps))
+
+        # RANKING AND SHOWING THE SUMMARIES
+        deduction_points, summaries_list = (list(t) for t in zip(*sorted(zip(deduction_points, summaries_list))))
+
+        cur_rank = 1
+        rank_downgrade = 0
+        for i in range(len(deduction_points)):
+            st.write(f'🏆 Rank {cur_rank} summary: 🏆', display_summary(summaries_list[i]), unsafe_allow_html=True)
+            if i < len(deduction_points) - 1:
+                rank_downgrade += 1
+                if not deduction_points[i + 1] == deduction_points[i]:
+                    cur_rank += rank_downgrade
+                    rank_downgrade = 0
+
+# session = SessionState.get(code=print("TEST"))
+# a = st.radio("Edit or show", ['Edit', 'Show'], 1)
+# if a == 'Edit':
+#     session.code = st.text_input('Edit code', session.code)
+# else:
+#     st.write(session.code)