Update ingest.py

Changed the chunk size to 880 from 1000, the original size seems to be causing some issues for larger files. The best way is to around with it.
PromtEngineer · Aug 17, 2023 · 0d20544 · 0d20544
1 parent a45a138
commit 0d20544
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/ingest.py b/ingest.py
@@ -123,7 +123,7 @@ def main(device_type):
     text_documents, python_documents = split_documents(documents)
     text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
     python_splitter = RecursiveCharacterTextSplitter.from_language(
-        language=Language.PYTHON, chunk_size=1000, chunk_overlap=200
+        language=Language.PYTHON, chunk_size=880, chunk_overlap=200
     )
     texts = text_splitter.split_documents(text_documents)
     texts.extend(python_splitter.split_documents(python_documents))