From 326783f7f1406e1b68a3a562ab34d0c106a4bedd Mon Sep 17 00:00:00 2001 From: yichuan520030910320 Date: Mon, 14 Jul 2025 23:07:46 -0700 Subject: [PATCH] fix mem compare fix split --- examples/faiss_only.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/examples/faiss_only.py b/examples/faiss_only.py index bd2822a..28d9db6 100644 --- a/examples/faiss_only.py +++ b/examples/faiss_only.py @@ -77,13 +77,7 @@ def main(): chunk_size=256, chunk_overlap=20, separator=" ", paragraph_separator="\n\n" ) - all_texts = [] - for doc in documents: - nodes = node_parser.get_nodes_from_documents([doc]) - for node in nodes: - all_texts.append(node.get_content()) - - tracker.checkpoint("After text chunking") + tracker.checkpoint("After text splitter setup") # Check if index already exists and try to load it index_loaded = False @@ -115,7 +109,9 @@ def main(): vector_store = FaissVectorStore(faiss_index=faiss_index) storage_context = StorageContext.from_defaults(vector_store=vector_store) index = VectorStoreIndex.from_documents( - documents, storage_context=storage_context + documents, + storage_context=storage_context, + transformations=[node_parser] ) tracker.checkpoint("After index building")