Upload 16 files (#8)

- Upload 16 files (654c92761e49f26e7c52e337cdd12c207ebeb3e5)

app.py

@@ -22,9 +22,9 @@ from utils.models import (
     get_data,
     get_flan_alpaca_xl_model,
     get_flan_t5_model,
+    get_instructor_embedding_model,
     get_mpnet_embedding_model,
     get_sgpt_embedding_model,
-    get_instructor_embedding_model,
     get_spacy_model,
     get_splade_sparse_embedding_model,
     get_t5_model,
@@ -248,7 +248,13 @@ with st.sidebar:
 
 # Choose encoder model
 
-encoder_models_choice = ["MPNET", "Instructor", "SGPT", "Hybrid MPNET - SPLADE"]
+encoder_models_choice = [
+    "MPNET",
+    "Instructor",
+    "Hybrid Instructor - SPLADE",
+    "SGPT",
+    "Hybrid MPNET - SPLADE",
+]
 with st.sidebar:
     encoder_model = st.selectbox("Select Encoder Model", encoder_models_choice)
 
@@ -285,12 +291,32 @@ elif encoder_model == "SGPT":
 elif encoder_model == "Instructor":
     # Connect to pinecone environment
     pinecone.init(
-        api_key=st.secrets["pinecone_instructor"], environment="us-west4-gcp-free"
+        api_key=st.secrets["pinecone_instructor"],
+        environment="us-west4-gcp-free",
     )
     pinecone_index_name = "week13-instructor-xl"
     pinecone_index = pinecone.Index(pinecone_index_name)
     retriever_model = get_instructor_embedding_model()
-    instruction = "Represent the financial question for retrieving supporting documents:"
+    instruction = (
+        "Represent the financial question for retrieving supporting documents:"
+    )
+
+elif encoder_model == "Hybrid Instructor - SPLADE":
+    # Connect to pinecone environment
+    pinecone.init(
+        api_key=st.secrets["pinecone_instructor_splade"],
+        environment="us-west4-gcp-free",
+    )
+    pinecone_index_name = "week13-splade-instructor-xl"
+    pinecone_index = pinecone.Index(pinecone_index_name)
+    retriever_model = get_instructor_embedding_model()
+    (
+        sparse_retriever_model,
+        sparse_retriever_tokenizer,
+    ) = get_splade_sparse_embedding_model()
+    instruction = (
+        "Represent the financial question for retrieving supporting documents:"
+    )
 
 elif encoder_model == "Hybrid MPNET - SPLADE":
     pinecone.init(
@@ -332,10 +358,15 @@ with st.sidebar:
 data = get_data()
 
 if document_type == "Single-Document":
-    if encoder_model == "Hybrid SGPT - SPLADE":
-        dense_query_embedding = create_dense_embeddings(
-            query_text, retriever_model
-        )
+    if encoder_model in ["Hybrid SGPT - SPLADE", "Hybrid Instructor - SPLADE"]:
+        if encoder_model == "Hybrid Instructor - SPLADE":
+            dense_query_embedding = create_dense_embeddings(
+                query_text, retriever_model, instruction
+            )
+        else:
+            dense_query_embedding = create_dense_embeddings(
+                query_text, retriever_model
+            )
         sparse_query_embedding = create_sparse_embeddings(
             query_text, sparse_retriever_model, sparse_retriever_tokenizer
         )
@@ -383,10 +414,18 @@ else:
     # Multi-Document Retreival
     # Single Company
     if multi_company_choice == "Single-Company":
-        if encoder_model == "Hybrid SGPT - SPLADE":
-            dense_query_embedding = create_dense_embeddings(
-                query_text, retriever_model
-            )
+        if encoder_model in [
+            "Hybrid SGPT - SPLADE",
+            "Hybrid Instructor - SPLADE",
+        ]:
+            if encoder_model == "Hybrid Instructor - SPLADE":
+                dense_query_embedding = create_dense_embeddings(
+                    query_text, retriever_model, instruction
+                )
+            else:
+                dense_query_embedding = create_dense_embeddings(
+                    query_text, retriever_model
+                )
             sparse_query_embedding = create_sparse_embeddings(
                 query_text, sparse_retriever_model, sparse_retriever_tokenizer
             )
@@ -448,10 +487,18 @@ else:
         multi_doc_context = generate_multi_doc_context(context_group)
     # Companies Comparison
     else:
-        if encoder_model == "Hybrid SGPT - SPLADE":
-            dense_query_embedding = create_dense_embeddings(
-                query_text, retriever_model
-            )
+        if encoder_model in [
+            "Hybrid SGPT - SPLADE",
+            "Hybrid Instructor - SPLADE",
+        ]:
+            if encoder_model == "Hybrid Instructor - SPLADE":
+                dense_query_embedding = create_dense_embeddings(
+                    query_text, retriever_model, instruction
+                )
+            else:
+                dense_query_embedding = create_dense_embeddings(
+                    query_text, retriever_model
+                )
             sparse_query_embedding = create_sparse_embeddings(
                 query_text, sparse_retriever_model, sparse_retriever_tokenizer
             )
@@ -766,22 +813,41 @@ with tab2:
             for year, quarter in year_quarter_list:
                 file_text = retrieve_transcript(data, year, quarter, ticker)
                 with st.expander(f"See Transcript - {quarter} {year}"):
-                    st.subheader("Earnings Call Transcript - {quarter} {year}:")
+                    st.subheader(
+                        "Earnings Call Transcript - {quarter} {year}:"
+                    )
                     stx.scrollableTextbox(
-                        file_text, height=700, border=False, fontFamily="Helvetica"
+                        file_text,
+                        height=700,
+                        border=False,
+                        fontFamily="Helvetica",
                     )
         else:
             for year, quarter in year_quarter_list:
-                file_text = retrieve_transcript(data, year, quarter, ticker_first)
+                file_text = retrieve_transcript(
+                    data, year, quarter, ticker_first
+                )
                 with st.expander(f"See Transcript - {quarter} {year}"):
-                    st.subheader("Earnings Call Transcript - {quarter} {year}:")
+                    st.subheader(
+                        "Earnings Call Transcript - {quarter} {year}:"
+                    )
                     stx.scrollableTextbox(
-                        file_text, height=700, border=False, fontFamily="Helvetica"
+                        file_text,
+                        height=700,
+                        border=False,
+                        fontFamily="Helvetica",
                     )
             for year, quarter in year_quarter_list:
-                file_text = retrieve_transcript(data, year, quarter, ticker_second)
+                file_text = retrieve_transcript(
+                    data, year, quarter, ticker_second
+                )
                 with st.expander(f"See Transcript - {quarter} {year}"):
-                    st.subheader("Earnings Call Transcript - {quarter} {year}:")
+                    st.subheader(
+                        "Earnings Call Transcript - {quarter} {year}:"
+                    )
                     stx.scrollableTextbox(
-                        file_text, height=700, border=False, fontFamily="Helvetica"
+                        file_text,
+                        height=700,
+                        border=False,
+                        fontFamily="Helvetica",
                     )

utils/models.py

@@ -8,8 +8,8 @@ import spacy
 import spacy_transformers
 import streamlit_scrollable_textbox as stx
 import torch
-from sentence_transformers import SentenceTransformer
 from InstructorEmbedding import INSTRUCTOR
+from sentence_transformers import SentenceTransformer
 from tqdm import tqdm
 from transformers import (
     AutoModelForMaskedLM,