TBNilles/LEANN
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix mem compare fix split

Browse Source
This commit is contained in:
yichuan520030910320
2025-07-14 23:07:46 -07:00
parent e5a9ca8787
commit 326783f7f1
1 changed files with 4 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
examples/faiss_only.py
View File

@@ -77,13 +77,7 @@ def main():
chunk_size=256, chunk_overlap=20, separator=" ", paragraph_separator="\n\n" chunk_size=256, chunk_overlap=20, separator=" ", paragraph_separator="\n\n"
) )
all_texts = [] tracker.checkpoint("After text splitter setup")
for doc in documents:
nodes = node_parser.get_nodes_from_documents([doc])
for node in nodes:
all_texts.append(node.get_content())
tracker.checkpoint("After text chunking")
# Check if index already exists and try to load it # Check if index already exists and try to load it
index_loaded = False index_loaded = False
@@ -115,7 +109,9 @@ def main():
vector_store = FaissVectorStore(faiss_index=faiss_index) vector_store = FaissVectorStore(faiss_index=faiss_index)
storage_context = StorageContext.from_defaults(vector_store=vector_store) storage_context = StorageContext.from_defaults(vector_store=vector_store)
index = VectorStoreIndex.from_documents( index = VectorStoreIndex.from_documents(
documents, storage_context=storage_context documents,
storage_context=storage_context,
transformations=[node_parser]
) )
tracker.checkpoint("After index building") tracker.checkpoint("After index building")