clustering works

.gitignore

@@ -1,2 +1,3 @@
 /__pycache__*
-recursive-exclude * *.py[co]
+recursive-exclude * *.py[co]
+/.vscode*

.vscode/settings.json

@@ -0,0 +1,24 @@
+{
+    "editor.tokenColorCustomizations": {
+        "textMateRules": [
+            {
+                "scope": "googletest.failed",
+                "settings": {
+                    "foreground": "#f00"
+                }
+            },
+            {
+                "scope": "googletest.passed",
+                "settings": {
+                    "foreground": "#0f0"
+                }
+            },
+            {
+                "scope": "googletest.run",
+                "settings": {
+                    "foreground": "#0f0"
+                }
+            }
+        ]
+    }
+}

clustering.py

@@ -0,0 +1,2 @@
+import spacy 
+import pandas as pd

topics.py

@@ -1,37 +1,29 @@
 import openai 
 from utils import *
-import mdforest 
-import pandas as pd
-import spacy
 
-class Insights:
+class TopicModelling:
     
     EMBEDDING_MAX_TOKENS = 1023
     
     def __init__(self, text:str) -> None:
-        cleaned_text = mdforest.clean_markdown(text)
         self.keywords = []
-        
-        self.corpus = preprocess(cleaned_text)        
-        self.text = create_nest_sentences(self.corpus, self.EMBEDDING_MAX_TOKENS)
+        self.corpus = text
+        # self.text = create_nest_sentences(self.corpus, self.EMBEDDING_MAX_TOKENS)
         self.model = load_keyword_model()
-        self.embedder = load_embedder()
-        
         
     def generate_topics(self) -> list:
-        for sentence in self.text:
-            self.keywords = self.keywords  + generate_keywords(self.model, sentence)
-        return self.keywords
+
+        keywords = self.model.extract_keywords(self.corpus, keyphrase_ngram_range=(1, 1), stop_words=None)
+        topics = self.model.extract_keywords(self.corpus, keyphrase_ngram_range=(1, 2), stop_words=None)
+        keywords = [kw[0] for kw in keywords]  + [kw[0] for kw in topics]
+        concepts = self.model.extract_keywords(self.corpus, keyphrase_ngram_range=(3, 3), stop_words='english', top_n=5)
+        concepts = [kw[0] for kw in concepts]
+        
+        return keywords, concepts
+        
+        
+    
     
-    def generate_embeddings(self) -> list:
-        # generate embeddings for all the sentences
-        nlp = spacy.load("en_core_web_sm")
-        final_embeddings = []
-        for text in self.text:
-            print(text[0])
-            doc = nlp(text[0])
-            sentence_embeddings = [sent.vector for sent in doc.sents]    
-            final_embeddings += sentence_embeddings
 
     
     

utils.py

@@ -4,12 +4,11 @@ from nltk.corpus import stopwords
 from transformers import AutoTokenizer
 import re
 import spacy
+from sklearn.cluster import KMeans, AgglomerativeClustering
+import numpy as np
 from sentence_transformers import SentenceTransformer
 
-# @st.cache_data
-# def load_nlp():
-#   nlp = 
-
+MODEL = 'all-MiniLM-L6-v2' 
 
 @st.cache_data
 def load_autotoken():
@@ -18,12 +17,13 @@ def load_autotoken():
 
 @st.cache_data
 def load_keyword_model():
-  kw_model = KeyBERT()
-  return kw_model
+	sentence_model = load_model()
+	kw_model = KeyBERT(model=sentence_model)
+	return kw_model
 
 @st.cache_data
-def load_embedder():
-  embedder = SentenceTransformer('all-MiniLM-L6-v2')
+def load_model():
+  embedder = SentenceTransformer(MODEL)
   return embedder 
 
 def create_nest_sentences(document:str, token_max_length = 1023):
@@ -32,7 +32,7 @@ def create_nest_sentences(document:str, token_max_length = 1023):
   length = 0
   tokenizer = load_autotoken()
 
-  for sentence in re.split(r'(?<=[^A-Z].[.?]) +(?=[A-Z])', document.replace("\n", ' ')):
+  for sentence in re.split(r'(?<=[^A-Z].[.?]) +(?=[A-Z])', document.replace("\n", '.')):
     tokens_in_sentence = tokenizer(str(sentence), truncation=False, padding=False)[0] # hugging face transformer tokenizer
     length += len(tokens_in_sentence)
 
@@ -51,7 +51,7 @@ def preprocess(text) -> str:
     stop_words = set(stopwords.words("english"))
     text = text.lower()
     text = ''.join([c for c in text if c not in ('!', '.', ',', '?', ':', ';', '"', "'", '-', '(', ')')])
-    words = text.split()
+    words = text.split() 
     words = [w for w in words if not w in stop_words]
     return " ".join(words)
 
@@ -64,3 +64,44 @@ def generate_keywords(kw_model, document: str) -> list:
     for extraction in complex_extractions:
         final_topics.append(extraction[0])
     return final_topics
+
+def cluster_based_on_topics(embedder, text1:str, text2:str, num_clusters:int = 2):
+  nlp = spacy.load("en_core_web_sm")
+  
+  # Preprocess and tokenize the texts
+  doc1 = nlp(preprocess(text1))
+  doc2 = nlp(preprocess(text2))
+  
+  # Extract sentences from the texts  
+  sentences1 = [sent.text for sent in doc1.sents]
+  sentences2 = [sent.text for sent in doc2.sents]
+  all_sentences = sentences1 + sentences2
+
+  with open('insight1_sent.txt', 'w') as f:
+    for item in sentences1:
+      f.write("%s\n" % item)
+            
+  with open('insight2_sent.txt', 'w') as f:
+    for item in sentences2:
+      f.write("%s\n" % item)
+        
+  # Generate sentence embeddings for each sentence
+  sentence_embeddings1 = embedder.encode(sentences1)
+  sentence_embeddings2 = embedder.encode(sentences2)
+  all_embeddings = np.concatenate((sentence_embeddings1, sentence_embeddings2), axis=0)
+
+  # Normalize the embeddings to unit length
+  all_embeddings = all_embeddings /  np.linalg.norm(all_embeddings, axis=1, keepdims=True)
+
+    # Perform kmean clustering
+  clustering_model = AgglomerativeClustering(n_clusters=None, distance_threshold=1.5)
+  clustering_model.fit(all_embeddings)
+  cluster_assignment = clustering_model.labels_
+
+  clustered_sentences = {}
+  for sentence_id, cluster_id in enumerate(cluster_assignment):
+    if cluster_id not in clustered_sentences:
+      clustered_sentences[cluster_id] = []
+    clustered_sentences[cluster_id].append(all_sentences[sentence_id])
+
+  return clustered_sentences