TBNilles/LEANN
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add dynamic add test

Browse Source
This commit is contained in:
yichuan520030910320
2025-09-30 00:48:46 -07:00
parent e588100674
commit e2b37914ce
3 changed files with 16 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
examples/dynamic_update_no_recompute.py
View File

@@ -43,7 +43,11 @@ from apps.chunking import create_text_chunks
REPO_ROOT = Path(__file__).resolve().parents[1] REPO_ROOT = Path(__file__).resolve().parents[1]
DEFAULT_QUERY = "What's LEANN?" DEFAULT_QUERY = "What's LEANN?"
DEFAULT_INITIAL_FILES = [REPO_ROOT / "data" / "2501.14312v1 (1).pdf"] DEFAULT_INITIAL_FILES = [
REPO_ROOT / "data" / "2501.14312v1 (1).pdf",
REPO_ROOT / "data" / "huawei_pangu.md",
REPO_ROOT / "data" / "PrideandPrejudice.txt",
]
DEFAULT_UPDATE_FILES = [REPO_ROOT / "data" / "2506.08276v1.pdf"] DEFAULT_UPDATE_FILES = [REPO_ROOT / "data" / "2506.08276v1.pdf"]

2
packages/leann-backend-hnsw/third_party/faiss vendored

Submodule packages/leann-backend-hnsw/third_party/faiss updated: ed96ff7dba...1d51f0c074

11
packages/leann-core/src/leann/api.py
View File

@@ -728,6 +728,7 @@ class LeannBuilder:
index = faiss.read_index(str(index_file)) index = faiss.read_index(str(index_file))
if hasattr(index, "is_recompute"): if hasattr(index, "is_recompute"):
index.is_recompute = needs_recompute index.is_recompute = needs_recompute
print(f"index.is_recompute: {index.is_recompute}")
if getattr(index, "storage", None) is None: if getattr(index, "storage", None) is None:
if index.metric_type == faiss.METRIC_INNER_PRODUCT: if index.metric_type == faiss.METRIC_INNER_PRODUCT:
storage_index = faiss.IndexFlatIP(index.d) storage_index = faiss.IndexFlatIP(index.d)
@@ -760,7 +761,15 @@ class LeannBuilder:
chunk.setdefault("metadata", {})["id"] = new_id chunk.setdefault("metadata", {})["id"] = new_id
chunk["id"] = new_id chunk["id"] = new_id
index.add(embeddings.shape[0], faiss.swig_ptr(embeddings)) if needs_recompute:
# sequengtially add embeddings
for i in range(embeddings.shape[0]):
print(f"add {i} embeddings")
index.add(1, faiss.swig_ptr(embeddings[i : i + 1]))
else:
index.add(embeddings.shape[0], faiss.swig_ptr(embeddings))
# index.add(embeddings.shape[0], faiss.swig_ptr(embeddings))
faiss.write_index(index, str(index_file)) faiss.write_index(index, str(index_file))
with open(passages_file, "a", encoding="utf-8") as f: with open(passages_file, "a", encoding="utf-8") as f: