add configuable funcname
This commit is contained in:
979
demo.ipynb
979
demo.ipynb
File diff suppressed because it is too large
Load Diff
BIN
examples/data/FairTree__OSDI_25_ (1).pdf
Normal file
BIN
examples/data/FairTree__OSDI_25_ (1).pdf
Normal file
Binary file not shown.
@@ -21,7 +21,7 @@ file_extractor: dict[str, BaseReader] = {
|
|||||||
".xlsx": reader,
|
".xlsx": reader,
|
||||||
}
|
}
|
||||||
node_parser = DoclingNodeParser(
|
node_parser = DoclingNodeParser(
|
||||||
chunker=HybridChunker(tokenizer="Qwen/Qwen3-Embedding-4B", max_tokens=512)
|
chunker=HybridChunker(tokenizer="Qwen/Qwen3-Embedding-4B", max_tokens=64)
|
||||||
)
|
)
|
||||||
|
|
||||||
documents = SimpleDirectoryReader(
|
documents = SimpleDirectoryReader(
|
||||||
@@ -67,7 +67,7 @@ async def main():
|
|||||||
print(f"\n[PHASE 2] Starting Leann chat session...")
|
print(f"\n[PHASE 2] Starting Leann chat session...")
|
||||||
chat = LeannChat(index_path=INDEX_PATH)
|
chat = LeannChat(index_path=INDEX_PATH)
|
||||||
|
|
||||||
query = "Based on the paper, what are the main techniques LEANN explores to reduce the storage overhead?"
|
query = "Based on the paper, what are the main techniques LEANN and DLPM explores to reduce the storage overhead?"
|
||||||
print(f"You: {query}")
|
print(f"You: {query}")
|
||||||
chat_response = chat.ask(query, top_k=10, recompute_beighbor_embeddings=True)
|
chat_response = chat.ask(query, top_k=10, recompute_beighbor_embeddings=True)
|
||||||
print(f"Leann: {chat_response}")
|
print(f"Leann: {chat_response}")
|
||||||
|
|||||||
@@ -192,6 +192,7 @@ class HNSWBuilder(LeannBackendBuilderInterface):
|
|||||||
build_kwargs = {**self.build_params, **kwargs}
|
build_kwargs = {**self.build_params, **kwargs}
|
||||||
metric_str = build_kwargs.get("distance_metric", "mips").lower()
|
metric_str = build_kwargs.get("distance_metric", "mips").lower()
|
||||||
metric_enum = get_metric_map().get(metric_str)
|
metric_enum = get_metric_map().get(metric_str)
|
||||||
|
print('metric_enum', metric_enum,' metric_str', metric_str)
|
||||||
if metric_enum is None:
|
if metric_enum is None:
|
||||||
raise ValueError(f"Unsupported distance_metric '{metric_str}'.")
|
raise ValueError(f"Unsupported distance_metric '{metric_str}'.")
|
||||||
|
|
||||||
@@ -217,7 +218,9 @@ class HNSWBuilder(LeannBackendBuilderInterface):
|
|||||||
faiss.normalize_L2(data)
|
faiss.normalize_L2(data)
|
||||||
|
|
||||||
# Add vectors to index
|
# Add vectors to index
|
||||||
|
print('starting to add vectors to index')
|
||||||
index.add(data.shape[0], faiss.swig_ptr(data))
|
index.add(data.shape[0], faiss.swig_ptr(data))
|
||||||
|
print('vectors added to index')
|
||||||
|
|
||||||
# Save index
|
# Save index
|
||||||
index_file = index_dir / f"{index_prefix}.index"
|
index_file = index_dir / f"{index_prefix}.index"
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user