Add SinaTools and update app file

app.py

@@ -11,6 +11,7 @@ from Nested.utils.data import get_dataloaders, text2segments
 import json
 from pydantic import BaseModel
 from fastapi.responses import JSONResponse
+from sinatools.utils.tokenizer import sentence_tokenizer
 
 app = FastAPI()
 print("Version 2...")
@@ -53,50 +54,86 @@ with open("Nested/utils/tag_vocab.pkl", "rb") as f:
 label_vocab = label_vocab[0]  # the list loaded from pickle
 id2label = {i: s for i, s in enumerate(label_vocab.itos)}
 
+def split_text_into_groups_of_Ns(sentence, max_words_per_sentence):
+    # Split the text into words
+    words = simple_word_tokenize(sentence)
+    
+    # Initialize variables
+    groups = []
+    current_group = ""
+    group_size = 0
+    
+    # Iterate through the words
+    for word in words:
+        if group_size < max_words_per_sentence - 1:
+            if len(current_group) == 0:
+                current_group = word
+            else:
+                current_group += " " + word
+            group_size += 1
+        else:
+            current_group += " " + word
+            groups.append(current_group)
+            current_group = ""
+            group_size = 0
+    
+    # Add the last group if it contains less than n words
+    if current_group:
+        groups.append(current_group)
+    
+    return groups
 
 class NERRequest(BaseModel):
     text: str
-
+    mode: str
 
 @app.post("/predict")
 def predict(request: NERRequest):
-
-    sentence = request.text  # 👈 user input
-
     # Load tagger
     tagger, tag_vocab, train_config = load_checkpoint(checkpoint_path)
 
-    dataset, token_vocab = text2segments(sentence)
+    text = request.text  
+    mode = request.mode  
 
-    vocabs = namedtuple("Vocab", ["tags", "tokens"])
-    vocab = vocabs(tokens=token_vocab, tags=tag_vocab)
+    sentences = sentence_tokenizer(
+        text, dot=False, new_line=True, question_mark=False, exclamation_mark=False
+    )
 
-    dataloader = get_dataloaders(
-        (dataset,),
-        vocab,
-        args_data,
-        batch_size=32,
-        shuffle=(False,),
-    )[0]
+    lists = []
+    for sentence in sentences:
+        se = split_text_into_groups_of_Ns(sentence, max_words_per_sentence=300)
+        for s in se:
+            dataset, token_vocab = text2segments(sentence)
 
-    segments = tagger.infer(dataloader)
+            vocabs = namedtuple("Vocab", ["tags", "tokens"])
+            vocab = vocabs(tokens=token_vocab, tags=tag_vocab)
 
-    lists = []
+            dataloader = get_dataloaders(
+                (dataset,),
+                vocab,
+                args_data,
+                batch_size=32,
+                shuffle=(False,),
+            )[0]
 
-    for segment in segments:
-        for token in segment:
-            item = {}
-            item["token"] = token.text
+            segments = tagger.infer(dataloader)
 
-            list_of_tags = [t["tag"] for t in token.pred_tag]
-            list_of_tags = [i for i in list_of_tags if i not in ("O", " ", "")]
+            # lists = []
 
-            if not list_of_tags:
-                item["tags"] = ["O"]
-            else:
-                item["tags"] = list_of_tags
+            for segment in segments:
+                for token in segment:
+                    item = {}
+                    item["token"] = token.text
+
+                    list_of_tags = [t["tag"] for t in token.pred_tag]
+                    list_of_tags = [i for i in list_of_tags if i not in ("O", " ", "")]
+
+                    if not list_of_tags:
+                        item["tags"] = ["O"]
+                    else:
+                        item["tags"] = list_of_tags
 
-            lists.append(item)
+                    lists.append(item)
 
     content = {
         "resp": lists,

requirements.txt

@@ -5,4 +5,5 @@ numpy
 huggingface_hub
 transformers
 natsort
-seqeval
+seqeval
+sinatools