ap token_size updated

app.py

@@ -167,6 +167,10 @@ def main():
 
     # File uploader widget
     if uploaded_file is not None:
+        # Select token size:
+        st.sidebar.success("Note: 1 Token ~ 4 Characters.")
+        token_size = st.sidebar.slider('Select a token size (when we scrape the document)', 5, 150, 20)
+
         # To read file as bytes:
         bytes_data = uploaded_file.getvalue()
         st.success("Your PDF is uploaded successfully.")
@@ -199,7 +203,7 @@ def main():
         # Tokenize it
         st.warning("Start tokenzing ...")
         token_splitter = SentenceTransformersTokenTextSplitter(
-            chunk_overlap=0, tokens_per_chunk=20
+            chunk_overlap=5, tokens_per_chunk=token_size
         )
         token_split_texts = []
         for text in character_split_texts: