req added, changed spacy to recursive

app.py

@@ -9,7 +9,7 @@ import requests
 import logging
 
 from aiohttp import ClientSession
-from langchain.text_splitter import SpacyTextSplitter
+from langchain.text_splitter import RecursiveCharacterTextSplitter
 from datasets import Dataset, load_dataset
 from tqdm import tqdm
 from tqdm.asyncio import tqdm_asyncio
@@ -26,8 +26,8 @@ class Chunker:
     def __init__(self, strategy, split_seq=".", chunk_len=512):
         self.split_seq = split_seq
         self.chunk_len = chunk_len
-        if strategy == "spacy":
-            self.split = SpacyTextSplitter().split_text
+        if strategy == "recursive":
+            self.split = RecursiveCharacterTextSplitter().split_text
         if strategy == "sequence":
             self.split = self.seq_splitter
         if strategy == "constant":
@@ -138,7 +138,7 @@ def run_embed(input_ds, input_splits, embed_in_text_col, output_ds, tei_url, pri
 
 
 def change_dropdown(choice):
-    if choice == "spacy" or choice == "sequence":
+    if choice == "recursive" or choice == "sequence":
         return [
             gr.Textbox(visible=True),
             gr.Textbox(visible=False)
@@ -166,8 +166,8 @@ with gr.Blocks() as demo:
             chunk_private = gr.Checkbox(label="Make chunked dataset private")
         with gr.Row():
             dropdown = gr.Dropdown(
-                ["spacy", "sequence", "constant"], label="Chunking strategy",
-                info="'spacy' uses a Spacy tokenizer, 'sequence' splits texts by a chosen sequence, "
+                ["recursive", "sequence", "constant"], label="Chunking strategy",
+                info="'recursive' uses a Langchain recursive tokenizer, 'sequence' splits texts by a chosen sequence, "
                      "'constant' makes chunks of the constant size",
                 scale=2
             )

requirements.txt

@@ -0,0 +1,4 @@
+langchain==0.0.*
+aiohttp==3.8.*
+datasets==2.16.*
+numpy==1.25.*