{ "cells": [ { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "INFO:sentence_transformers.SentenceTransformer:Use pytorch device_name: mps\n", "INFO:sentence_transformers.SentenceTransformer:Load pretrained SentenceTransformer: facebook/contriever-msmarco\n", "WARNING:sentence_transformers.SentenceTransformer:No sentence-transformers model found with name facebook/contriever-msmarco. Creating a new one with mean pooling.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO: Computing embeddings for 1 chunks using SentenceTransformer model 'facebook/contriever-msmarco'...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Batches: 100%|██████████| 1/1 [00:00<00:00, 65.35it/s]\n", "INFO:sentence_transformers.SentenceTransformer:Use pytorch device_name: mps\n", "INFO:sentence_transformers.SentenceTransformer:Load pretrained SentenceTransformer: facebook/contriever-msmarco\n", "WARNING:sentence_transformers.SentenceTransformer:No sentence-transformers model found with name facebook/contriever-msmarco. Creating a new one with mean pooling.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO: Computing embeddings for 5 chunks using SentenceTransformer model 'facebook/contriever-msmarco'...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Batches: 100%|██████████| 1/1 [00:00<00:00, 50.38it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "M: 64 for level: 0\n", "INFO: Converting HNSW index to CSR-pruned format...\n", "Starting conversion: knowledge.index -> knowledge.csr.tmp\n", "[0.00s] Reading Index HNSW header...\n", "[0.00s] Header read: d=768, ntotal=5\n", "[0.00s] Reading HNSW struct vectors...\n", " Reading vector (dtype=, fmt='d')... Count=6, Bytes=48\n", "[0.00s] Read assign_probas (6)\n", " Reading vector (dtype=, fmt='i')... Count=7, Bytes=28\n", "[0.12s] Read cum_nneighbor_per_level (7)\n", " Reading vector (dtype=, fmt='i')... " ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Count=5, Bytes=20\n", "[0.21s] Read levels (5)\n", "[0.30s] Probing for compact storage flag...\n", "[0.30s] Found compact flag: False\n", "[0.30s] Compact flag is False, reading original format...\n", "[0.30s] Probing for potential extra byte before non-compact offsets...\n", "[0.30s] Found and consumed an unexpected 0x00 byte.\n", " Reading vector (dtype=, fmt='Q')... Count=6, Bytes=48\n", "[0.30s] Read offsets (6)\n", "[0.39s] Attempting to read neighbors vector...\n", " Reading vector (dtype=, fmt='i')... Count=320, Bytes=1280\n", "[0.39s] Read neighbors (320)\n", "[0.47s] Read scalar params (ep=4, max_lvl=0)\n", "[0.47s] Checking for storage data...\n", "[0.47s] Found storage fourcc: 49467849.\n", "[0.47s] Converting to CSR format...\n", "[0.47s] Conversion loop finished. \n", "[0.47s] Running validation checks...\n", " Checking total valid neighbor count...\n", " OK: Total valid neighbors = 20\n", " Checking final pointer indices...\n", " OK: Final pointers match data size.\n", "[0.47s] Deleting original neighbors and offsets arrays...\n", " CSR Stats: |data|=20, |level_ptr|=10\n", "[0.56s] Writing CSR HNSW graph data in FAISS-compatible order...\n", " Pruning embeddings: Writing NULL storage marker.\n", "[0.64s] Conversion complete." ] }, { "name": "stderr", "output_type": "stream", "text": [ "INFO:sentence_transformers.SentenceTransformer:Use pytorch device_name: mps\n", "INFO:sentence_transformers.SentenceTransformer:Load pretrained SentenceTransformer: facebook/contriever-msmarco\n", "WARNING:sentence_transformers.SentenceTransformer:No sentence-transformers model found with name facebook/contriever-msmarco. Creating a new one with mean pooling.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "✅ CSR conversion successful.\n", "INFO: Replaced original index with CSR-pruned version at 'knowledge.index'\n", "[read_HNSW - CSR NL v4] Reading metadata & CSR indices (manual offset)...\n", "[read_HNSW NL v4] Read levels vector, size: 5\n", "[read_HNSW NL v4] Reading Compact Storage format indices...\n", "[read_HNSW NL v4] Read compact_level_ptr, size: 10\n", "INFO: Terminating session server process (PID: 21439)...\n", "[read_HNSW NL v4] Read compact_node_offsets, size: 6\n", "[read_HNSW NL v4] Read entry_point: 4, max_level: 0\n", "[read_HNSW NL v4] Read storage fourcc: 0x6c6c756e\n", "[read_HNSW NL v4 FIX] Detected FileIOReader. Neighbors size field offset: 326\n", "[read_HNSW NL v4] Reading neighbors data into memory.\n", "[read_HNSW NL v4] Read neighbors data, size: 20\n", "[read_HNSW NL v4] Finished reading metadata and CSR indices.\n", "INFO: Skipping external storage loading, since is_recompute is true.\n", "INFO: Server process terminated.\n", "🔍 DEBUG LeannSearcher.search() called:\n", " Query: 'C++ programming languages'\n", " Top_k: 2\n", " Search kwargs: {'recompute_beighbor_embeddings': True}\n", "INFO: Computing embeddings for 1 chunks using SentenceTransformer model 'facebook/contriever-msmarco'...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Batches: 100%|██████████| 1/1 [00:00<00:00, 85.08it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Generated embedding shape: (1, 768)\n", "🔍 DEBUG Query embedding first 10 values: [ 0.04288 -0.04135 0.0666 0.02197 -0.0881 -0.04367 -0.02835 -0.0408\n", " -0.1254 -0.08594]\n", "🔍 DEBUG Query embedding norm: 1.3876953125\n", "INFO: Starting session-level embedding server for 'leann_backend_hnsw.hnsw_embedding_server'...\n", "INFO: Running command from project root: /Users/yichuan/Desktop/code/LEANN/leann\n", "INFO: Command: /Users/yichuan/Desktop/code/LEANN/leann/.venv/bin/python -m leann_backend_hnsw.hnsw_embedding_server --zmq-port 5557 --model-name facebook/contriever-msmarco --passages-file knowledge.leann.meta.json --disable-warmup\n", "INFO: Server process started with PID: 21622\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "✅ Embedding server is up and ready for this session.\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: INFO: Starting backend auto-discovery...\n", "ZmqDistanceComputer initialized: d=768, metric=0\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: INFO: Registering backend 'diskann'\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: INFO: Backend auto-discovery finished.\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: INFO: Registering backend 'hnsw'\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: Loading tokenizer for facebook/contriever-msmarco...\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: Tokenizer loaded successfully!\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: MPS available: True\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: CUDA available: False\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: Using MPS device (Apple Silicon)\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: Starting HNSW server on port 5557 with model facebook/contriever-msmarco\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: Loading model facebook/contriever-msmarco... (this may take a while if downloading)\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: Model facebook/contriever-msmarco loaded successfully!\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: Loaded label map with 5 entries\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: Initialized lazy passage loading for 5 passages\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: Warmup disabled or no passages available (enable_warmup=False, passages=5)\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: HNSW ZMQ server listening on port 5557\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: Started HNSW ZMQ server thread on port 5557\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: Received ZMQ request of size 3 bytes\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: DEBUG: request_payload length: 1\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: DEBUG: request_payload[0]: - [4]\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: Request for 1 node embeddings\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: Total batch size: 1, max_batch_size: 128\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: DEBUG zmq_server_thread: Final 'hidden' array | Shape: (1, 768) | Dtype: float32 | Has NaN/Inf: False\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: Serialize time: 0.000150 seconds\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: ZMQ E2E time: 0.142946 seconds\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: Received ZMQ request of size 3849 bytes\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: DEBUG: request_payload length: 2\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: DEBUG: request_payload[0]: - [0, 1, 2, 3]\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: DEBUG: request_payload[1]: - [0.042877197265625, -0.041351318359375, 0.06658935546875, 0.02197265625, -0.08807373046875, -0.04367...\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: DEBUG: Distance calculation request received\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: Node IDs: [0, 1, 2, 3]\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: Query vector dim: 768\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: Passages loaded: 5\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: DEBUG: Looking up passage ID 0\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: DEBUG: Found text for ID 0, length: 37\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: DEBUG: Looking up passage ID 1\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: DEBUG: Found text for ID 1, length: 41\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: DEBUG: Looking up passage ID 2\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: DEBUG: Found text for ID 2, length: 38\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: DEBUG: Looking up passage ID 3\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: DEBUG: Found text for ID 3, length: 36\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: Sending distance response with 4 distances\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: Distance calculation E2E time: 0.173929 seconds\n", " Backend returned: labels=2 results\n", " Processing 2 passage IDs:\n", " 1. passage_id='28b7b6b9-d0a4-408d-9e7f-9a7fcb7d8186' -> SUCCESS: C# is a powerful programming language...\n", " 2. passage_id='1bccf691-a571-4e9d-aaed-424a30ba8604' -> SUCCESS: Python is a powerful programming language...\n", " Final enriched results: 2 passages\n", "[SearchResult(id='28b7b6b9-d0a4-408d-9e7f-9a7fcb7d8186', score=np.float32(1.5213046), text='C# is a powerful programming language', metadata={}), SearchResult(id='1bccf691-a571-4e9d-aaed-424a30ba8604', score=np.float32(1.2999034), text='Python is a powerful programming language', metadata={})]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[leann_backend_hnsw.hnsw_embedding_server LOG]: ZMQ socket timeout, continuing to listen\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: ZMQ socket timeout, continuing to listen\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: ZMQ socket timeout, continuing to listen\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: ZMQ socket timeout, continuing to listen\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: ZMQ socket timeout, continuing to listen\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: ZMQ socket timeout, continuing to listen\n", "[leann_backend_hnsw.hnsw_embedding_server LOG]: ZMQ socket timeout, continuing to listen\n" ] } ], "source": [ "from leann.api import LeannBuilder, LeannSearcher\n", "# 1. Build index (no embeddings stored!)\n", "builder = LeannBuilder(backend_name=\"hnsw\")\n", "builder.add_text(\"C# is a powerful programming language\")\n", "builder.add_text(\"Python is a powerful programming language\")\n", "builder.add_text(\"Machine learning transforms industries\") \n", "builder.add_text(\"Neural networks process complex data\")\n", "builder.add_text(\"Leann is a great storage saving engine for RAG on your macbook\")\n", "builder.build_index(\"knowledge.leann\")\n", "# 2. Search with real-time embeddings\n", "searcher = LeannSearcher(\"knowledge.leann\")\n", "results = searcher.search(\"C++ programming languages\", top_k=2, recompute_beighbor_embeddings=True)\n", "print(results)" ] } ], "metadata": { "kernelspec": { "display_name": ".venv", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.12" } }, "nbformat": 4, "nbformat_minor": 2 }