Update app.py

app.py

@@ -234,28 +234,29 @@ def classify_emotion(text, classifier):
     return final_emotion
         
 def get_embedding_for_text(text, tokenizer, model):
-    """Get embedding for complete text."""
-    chunks = split_text(text)
+    """Get embedding for complete text while preserving all content."""
+    # Split into optimal chunks of exactly 512 tokens
+    tokenized_text = tokenizer.encode(text)
+    chunks = [tokenized_text[i:i + 512] for i in range(0, len(tokenized_text), 512)]
     chunk_embeddings = []
     
     for chunk in chunks:
-        inputs = tokenizer(
-            chunk,
+        inputs = tokenizer.encode(
+            tokenizer.decode(chunk),
             return_tensors="pt",
-            padding=True,
-            truncation=True,
+            padding='max_length',
             max_length=512
         )
-        inputs = {k: v.to(model.device) for k, v in inputs.items()}
+        inputs = inputs.to(model.device)
         
         with torch.no_grad():
-            # Access the first element of the tuple which contains the hidden states
-            outputs = model(**inputs)[0]
+            outputs = model(inputs)[0]
             embedding = outputs[:, 0, :].cpu().numpy()
             chunk_embeddings.append(embedding[0])
     
     if chunk_embeddings:
-        weights = np.array([len(chunk.split()) for chunk in chunks])
+        # Weight each chunk based on its content length
+        weights = np.array([len(chunk) for chunk in chunks])
         weights = weights / weights.sum()
         weighted_embedding = np.average(chunk_embeddings, axis=0, weights=weights)
         return weighted_embedding