Update app.py

app.py

@@ -1,32 +1,18 @@
 import streamlit as st
-from langchain.text_splitter import RecursiveCharacterTextSplitter, CharacterTextSplitter, Language
+from langchain.text_splitter import RecursiveCharacterTextSplitter
 import tiktoken
 
 
 CHARACTER_LENGTH = "length_function=lambda x: len(x)"
 TOKEN_LENGTH = """enc = tiktoken.get_encoding("cl100k_base")
 length_function = lambda text: len(enc.encode(text))
-"""
-CHARACTER = """CharacterTextSplitter(
-    separator="\\n\\n",
-    chunk_size={chunk_size},
-    chunk_overlap={chunk_overlap},
-    length_function={length_function}
-)
-"""
+
 RECURSIVE_CHARACTER = """RecursiveCharacterTextSplitter(
     chunk_size={chunk_size},
     chunk_overlap={chunk_overlap},
     length_function={length_function}
 )
-"""
-LANGUAGE = """RecursiveCharacterTextSplitter.from_language(
-    language="{language}",
-    chunk_size={chunk_size},
-    chunk_overlap={chunk_overlap},
-    length_function={length_function}
-)
-"""
+
 
 # Streamlit UI
 st.title("Understand Chunk and Token")
@@ -77,30 +63,13 @@ doc = st.text_area("Füge hier deinen Text ein:")
 
 # Button zum Splitten des Textes
 if st.button("Split Text"):
-    # Erzeugen des Splitter-Objekts basierend auf der Auswahl
-    if splitter_choice == "Character":
-        splitter = CharacterTextSplitter(
-            separator="\n\n",
-            chunk_size=chunk_size,
-            chunk_overlap=chunk_overlap,
-            length_function=length_function
-        )
-    elif splitter_choice == "RecursiveCharacter":
-        splitter = RecursiveCharacterTextSplitter(
-            chunk_size=chunk_size,
-            chunk_overlap=chunk_overlap,
-            length_function=length_function
-        )
-    elif "Language." in splitter_choice:
-        lang = splitter_choice.split(".")[1].lower()
-        splitter = RecursiveCharacterTextSplitter.from_language(
-            language=lang,
+
+    splitter = RecursiveCharacterTextSplitter(
             chunk_size=chunk_size,
             chunk_overlap=chunk_overlap,
             length_function=length_function
-        )
-    else:
-        raise ValueError("Ungültige Wahl beim Text Splitter.")
+    )
+
     
     # Aufteilen des Textes
     splits = splitter.split_text(doc)