Update app.py

app.py

@@ -1,56 +1,32 @@
 import streamlit as st
 from langchain.text_splitter import RecursiveCharacterTextSplitter
-import tiktoken
-
-
-CHARACTER_LENGTH = "length_function=lambda x: len(x)"
-TOKEN_LENGTH = enc = tiktoken.get_encoding("cl100k_base")
-length_function = lambda text: len(enc.encode(text))
-
-# Streamlit UI
-st.title("Understand Chunk and Token")
-
 
+st.title("Understand Chunking")
 
 chunk_size = st.number_input(
         min_value=1,
         label="Chunk Size",
-        value=1000
+        value=50
 )
 
-
 chunk_overlap = st.number_input(
         min_value=1,
         max_value=chunk_size - 1,
         label="Chunk Overlap",
-        value=int(chunk_size * 0.2)
+        value=10
 )
 
+docs = st.text_area("Put your text:")
 
-length_function_option = st.selectbox(
-        "Length Function",
-        ["Characters", "Tokens"]
-    )
-
-
-length_function_option = Tokens
-
-# Text-Eingabe
-doc = st.text_area("Füge hier deinen Text ein:")
-
-# Button zum Splitten des Textes
-if st.button("Split Text"):
+if st.button("Split"):
 
     splitter = RecursiveCharacterTextSplitter(
             chunk_size=chunk_size,
             chunk_overlap=chunk_overlap,
-            length_function=length_function
+            length_function=len
     )
 
-    
-    # Aufteilen des Textes
-    splits = splitter.split_text(doc)
+    splits = splitter.split_text(docs)
 
-    # Ausgabe der erstellten Textsplitter
     for idx, split in enumerate(splits, start=1):
-        st.text_area(f"Teilstück {idx}", split, height=150)
+        st.text_area(f"Chunk {idx}", split)