fix: remove whitespace from blank line

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>

.gitattributes

@@ -0,0 +1 @@
+paper_plot/data/big_graph_degree_data.npz filter=lfs diff=lfs merge=lfs -text

.github/workflows/build-and-publish.yml

@@ -5,7 +5,6 @@ on:
     branches: [ main ]
   pull_request:
     branches: [ main ]
-  workflow_dispatch:
 
 jobs:
   build:

.github/workflows/build-reusable.yml

@@ -28,7 +28,7 @@ jobs:
 
       - name: Install ruff
         run: |
-          uv tool install ruff
+          uv tool install ruff==0.12.7
 
       - name: Run ruff check
         run: |
@@ -54,40 +54,20 @@ jobs:
             python: '3.12'
           - os: ubuntu-22.04
             python: '3.13'
-          - os: macos-14
+          - os: macos-latest
             python: '3.9'
-          - os: macos-14
+          - os: macos-latest
             python: '3.10'
-          - os: macos-14
+          - os: macos-latest
             python: '3.11'
-          - os: macos-14
+          - os: macos-latest
             python: '3.12'
-          - os: macos-14
+          - os: macos-latest
             python: '3.13'
-          - os: macos-15
-            python: '3.9'
-          - os: macos-15
-            python: '3.10'
-          - os: macos-15
-            python: '3.11'
-          - os: macos-15
-            python: '3.12'
-          - os: macos-15
-            python: '3.13'
-          - os: macos-13
-            python: '3.9'
-          - os: macos-13
-            python: '3.10'
-          - os: macos-13
-            python: '3.11'
-          - os: macos-13
-            python: '3.12'
-          # Note: macos-13 + Python 3.13 excluded due to PyTorch compatibility
-          # (PyTorch 2.5+ supports Python 3.13 but not Intel Mac x86_64)
     runs-on: ${{ matrix.os }}
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.ref }}
           submodules: recursive
@@ -98,23 +78,21 @@ jobs:
           python-version: ${{ matrix.python }}
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v4
 
       - name: Install system dependencies (Ubuntu)
         if: runner.os == 'Linux'
         run: |
           sudo apt-get update
           sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \
-            pkg-config libabsl-dev libaio-dev libprotobuf-dev \
-            patchelf
+            pkg-config libopenblas-dev patchelf libabsl-dev libaio-dev libprotobuf-dev
 
           # Install Intel MKL for DiskANN
           wget -q https://registrationcenter-download.intel.com/akdlm/IRC_NAS/79153e0f-74d7-45af-b8c2-258941adf58a/intel-onemkl-2025.0.0.940.sh
           sudo sh intel-onemkl-2025.0.0.940.sh -a --components intel.oneapi.lin.mkl.devel --action install --eula accept -s
           source /opt/intel/oneapi/setvars.sh
           echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV
-          echo "LD_LIBRARY_PATH=/opt/intel/oneapi/compiler/latest/linux/compiler/lib/intel64_lin" >> $GITHUB_ENV
-          echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/intel/oneapi/mkl/latest/lib/intel64" >> $GITHUB_ENV
+          echo "LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH" >> $GITHUB_ENV
 
       - name: Install system dependencies (macOS)
         if: runner.os == 'macOS'
@@ -131,70 +109,41 @@ jobs:
             uv pip install --system delocate
           fi
 
-      - name: Set macOS environment variables
-        if: runner.os == 'macOS'
-        run: |
-          # Use brew --prefix to automatically detect Homebrew installation path
-          HOMEBREW_PREFIX=$(brew --prefix)
-          echo "HOMEBREW_PREFIX=${HOMEBREW_PREFIX}" >> $GITHUB_ENV
-          echo "OpenMP_ROOT=${HOMEBREW_PREFIX}/opt/libomp" >> $GITHUB_ENV
-
-          # Set CMAKE_PREFIX_PATH to let CMake find all packages automatically
-          echo "CMAKE_PREFIX_PATH=${HOMEBREW_PREFIX}" >> $GITHUB_ENV
-
-          # Set compiler flags for OpenMP (required for both backends)
-          echo "LDFLAGS=-L${HOMEBREW_PREFIX}/opt/libomp/lib" >> $GITHUB_ENV
-          echo "CPPFLAGS=-I${HOMEBREW_PREFIX}/opt/libomp/include" >> $GITHUB_ENV
-
       - name: Build packages
         run: |
-          # Build core (platform independent)
+          # Build core (platform independent) on all platforms for consistency
           cd packages/leann-core
           uv build
           cd ../..
 
           # Build HNSW backend
           cd packages/leann-backend-hnsw
-          if [[ "${{ matrix.os }}" == macos-* ]]; then
-            # Use system clang for better compatibility
+          if [ "${{ matrix.os }}" == "macos-latest" ]; then
+            # Use system clang instead of homebrew LLVM for better compatibility
             export CC=clang
             export CXX=clang++
-            # Homebrew libraries on each macOS version require matching minimum version
-            if [[ "${{ matrix.os }}" == "macos-13" ]]; then
-              export MACOSX_DEPLOYMENT_TARGET=13.0
-            elif [[ "${{ matrix.os }}" == "macos-14" ]]; then
-              export MACOSX_DEPLOYMENT_TARGET=14.0
-            elif [[ "${{ matrix.os }}" == "macos-15" ]]; then
-              export MACOSX_DEPLOYMENT_TARGET=15.0
-            fi
-            uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
+            export MACOSX_DEPLOYMENT_TARGET=11.0
+            uv build --wheel --python python
           else
-            uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
+            uv build --wheel --python python
           fi
           cd ../..
 
           # Build DiskANN backend
           cd packages/leann-backend-diskann
-          if [[ "${{ matrix.os }}" == macos-* ]]; then
-            # Use system clang for better compatibility
+          if [ "${{ matrix.os }}" == "macos-latest" ]; then
+            # Use system clang instead of homebrew LLVM for better compatibility
             export CC=clang
             export CXX=clang++
-            # DiskANN requires macOS 13.3+ for sgesdd_ LAPACK function
-            # But Homebrew libraries on each macOS version require matching minimum version
-            if [[ "${{ matrix.os }}" == "macos-13" ]]; then
-              export MACOSX_DEPLOYMENT_TARGET=13.3
-            elif [[ "${{ matrix.os }}" == "macos-14" ]]; then
-              export MACOSX_DEPLOYMENT_TARGET=14.0
-            elif [[ "${{ matrix.os }}" == "macos-15" ]]; then
-              export MACOSX_DEPLOYMENT_TARGET=15.0
-            fi
-            uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
+            # sgesdd_ is only available on macOS 13.3+
+            export MACOSX_DEPLOYMENT_TARGET=13.3
+            uv build --wheel --python python
           else
-            uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
+            uv build --wheel --python python
           fi
           cd ../..
 
-          # Build meta package (platform independent)
+          # Build meta package (platform independent) on all platforms
           cd packages/leann
           uv build
           cd ../..
@@ -211,10 +160,15 @@ jobs:
           fi
           cd ../..
 
-          # Repair DiskANN wheel
+          # Repair DiskANN wheel - use show first to debug
           cd packages/leann-backend-diskann
           if [ -d dist ]; then
+            echo "Checking DiskANN wheel contents before repair:"
+            unzip -l dist/*.whl | grep -E "\.so|\.pyd|_diskannpy" || echo "No .so files found"
+            auditwheel show dist/*.whl || echo "auditwheel show failed"
             auditwheel repair dist/*.whl -w dist_repaired
+            echo "Checking DiskANN wheel contents after repair:"
+            unzip -l dist_repaired/*.whl | grep -E "\.so|\.pyd|_diskannpy" || echo "No .so files found after repair"
             rm -rf dist
             mv dist_repaired dist
           fi
@@ -223,24 +177,10 @@ jobs:
       - name: Repair wheels (macOS)
         if: runner.os == 'macOS'
         run: |
-          # Determine deployment target based on runner OS
-          # Must match the Homebrew libraries for each macOS version
-          if [[ "${{ matrix.os }}" == "macos-13" ]]; then
-            HNSW_TARGET="13.0"
-            DISKANN_TARGET="13.3"
-          elif [[ "${{ matrix.os }}" == "macos-14" ]]; then
-            HNSW_TARGET="14.0"
-            DISKANN_TARGET="14.0"
-          elif [[ "${{ matrix.os }}" == "macos-15" ]]; then
-            HNSW_TARGET="15.0"
-            DISKANN_TARGET="15.0"
-          fi
-
           # Repair HNSW wheel
           cd packages/leann-backend-hnsw
           if [ -d dist ]; then
-            export MACOSX_DEPLOYMENT_TARGET=$HNSW_TARGET
-            delocate-wheel -w dist_repaired -v --require-target-macos-version $HNSW_TARGET dist/*.whl
+            delocate-wheel -w dist_repaired -v dist/*.whl
             rm -rf dist
             mv dist_repaired dist
           fi
@@ -249,8 +189,7 @@ jobs:
           # Repair DiskANN wheel
           cd packages/leann-backend-diskann
           if [ -d dist ]; then
-            export MACOSX_DEPLOYMENT_TARGET=$DISKANN_TARGET
-            delocate-wheel -w dist_repaired -v --require-target-macos-version $DISKANN_TARGET dist/*.whl
+            delocate-wheel -w dist_repaired -v dist/*.whl
             rm -rf dist
             mv dist_repaired dist
           fi
@@ -261,34 +200,44 @@ jobs:
           echo "📦 Built packages:"
           find packages/*/dist -name "*.whl" -o -name "*.tar.gz" | sort
 
-
       - name: Install built packages for testing
         run: |
           # Create a virtual environment with the correct Python version
-          uv venv --python ${{ matrix.python }}
+          uv venv --python python${{ matrix.python }}
           source .venv/bin/activate || source .venv/Scripts/activate
 
-          # Install packages using --find-links to prioritize local builds
-          uv pip install --find-links packages/leann-core/dist --find-links packages/leann-backend-hnsw/dist --find-links packages/leann-backend-diskann/dist packages/leann-core/dist/*.whl || uv pip install --find-links packages/leann-core/dist packages/leann-core/dist/*.tar.gz
-          uv pip install --find-links packages/leann-core/dist packages/leann-backend-hnsw/dist/*.whl
-          uv pip install --find-links packages/leann-core/dist packages/leann-backend-diskann/dist/*.whl
-          uv pip install packages/leann/dist/*.whl || uv pip install packages/leann/dist/*.tar.gz
+          # Install the built wheels directly to ensure we use locally built packages
+          # Use only locally built wheels on all platforms for full consistency
+          FIND_LINKS="--find-links packages/leann-core/dist --find-links packages/leann/dist"
+          FIND_LINKS="$FIND_LINKS --find-links packages/leann-backend-hnsw/dist --find-links packages/leann-backend-diskann/dist"
+
+          uv pip install leann-core leann leann-backend-hnsw leann-backend-diskann \
+            $FIND_LINKS --force-reinstall
 
           # Install test dependencies using extras
           uv pip install -e ".[test]"
 
+          # Debug: Check if _diskannpy module is installed correctly
+          echo "Checking installed DiskANN module structure:"
+          python -c "import leann_backend_diskann; print('leann_backend_diskann location:', leann_backend_diskann.__file__)" || echo "Failed to import leann_backend_diskann"
+          python -c "from leann_backend_diskann import _diskannpy; print('_diskannpy imported successfully')" || echo "Failed to import _diskannpy"
+          ls -la $(python -c "import leann_backend_diskann; import os; print(os.path.dirname(leann_backend_diskann.__file__))" 2>/dev/null) 2>/dev/null || echo "Failed to list module directory"
+
       - name: Run tests with pytest
         env:
-          CI: true
+          CI: true  # Mark as CI environment to skip memory-intensive tests
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           HF_HUB_DISABLE_SYMLINKS: 1
           TOKENIZERS_PARALLELISM: false
-          PYTORCH_ENABLE_MPS_FALLBACK: 0
-          OMP_NUM_THREADS: 1
-          MKL_NUM_THREADS: 1
+          PYTORCH_ENABLE_MPS_FALLBACK: 0  # Disable MPS on macOS CI to avoid memory issues
+          OMP_NUM_THREADS: 1  # Disable OpenMP parallelism to avoid libomp crashes
+          MKL_NUM_THREADS: 1  # Single thread for MKL operations
         run: |
+          # Activate virtual environment
           source .venv/bin/activate || source .venv/Scripts/activate
-          pytest tests/ -v --tb=short
+
+          # Run all tests
+          pytest tests/
 
       - name: Run sanity checks (optional)
         run: |
@@ -306,53 +255,3 @@ jobs:
         with:
           name: packages-${{ matrix.os }}-py${{ matrix.python }}
           path: packages/*/dist/
-
-
-  arch-smoke:
-    name: Arch Linux smoke test (install & import)
-    needs: build
-    runs-on: ubuntu-latest
-    container:
-      image: archlinux:latest
-
-    steps:
-      - name: Prepare system
-        run: |
-          pacman -Syu --noconfirm
-          pacman -S --noconfirm python python-pip gcc git zlib openssl
-
-      - name: Download ALL wheel artifacts from this run
-        uses: actions/download-artifact@v5
-        with:
-          # Don't specify name, download all artifacts
-          path: ./wheels
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v6
-
-      - name: Create virtual environment and install wheels
-        run: |
-          uv venv
-          source .venv/bin/activate || source .venv/Scripts/activate
-          uv pip install --find-links wheels leann-core
-          uv pip install --find-links wheels leann-backend-hnsw
-          uv pip install --find-links wheels leann-backend-diskann
-          uv pip install --find-links wheels leann
-
-      - name: Import & tiny runtime check
-        env:
-          OMP_NUM_THREADS: 1
-          MKL_NUM_THREADS: 1
-        run: |
-          source .venv/bin/activate || source .venv/Scripts/activate
-          python - <<'PY'
-          import leann
-          import leann_backend_hnsw as h
-          import leann_backend_diskann as d
-          from leann import LeannBuilder, LeannSearcher
-          b = LeannBuilder(backend_name="hnsw")
-          b.add_text("hello arch")
-          b.build_index("arch_demo.leann")
-          s = LeannSearcher("arch_demo.leann")
-          print("search:", s.search("hello", top_k=1))
-          PY

.github/workflows/link-check.yml

@@ -14,6 +14,6 @@ jobs:
       - uses: actions/checkout@v4
       - uses: lycheeverse/lychee-action@v2
         with:
-          args: --no-progress --insecure --user-agent 'curl/7.68.0' README.md docs/ apps/ examples/ benchmarks/
+          args: --no-progress --insecure README.md docs/ apps/ examples/ benchmarks/
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -18,7 +18,6 @@ demo/experiment_results/**/*.json
 *.eml
 *.emlx
 *.json
-!.vscode/*.json
 *.sh
 *.txt
 !CMakeLists.txt
@@ -93,5 +92,3 @@ packages/leann-backend-diskann/third_party/DiskANN/_deps/
 batchtest.py
 tests/__pytest_cache__/
 tests/__pycache__/
-
-benchmarks/data/

.vscode/extensions.json

@@ -1,5 +0,0 @@
-{
-    "recommendations": [
-        "charliermarsh.ruff",
-    ]
-}

.vscode/settings.json

@@ -1,22 +0,0 @@
-{
-  "python.defaultInterpreterPath": ".venv/bin/python",
-  "python.terminal.activateEnvironment": true,
-  "[python]": {
-    "editor.defaultFormatter": "charliermarsh.ruff",
-    "editor.formatOnSave": true,
-    "editor.codeActionsOnSave": {
-      "source.organizeImports": "explicit",
-      "source.fixAll": "explicit"
-    },
-    "editor.insertSpaces": true,
-    "editor.tabSize": 4
-  },
-  "ruff.enable": true,
-  "files.watcherExclude": {
-    "**/.venv/**": true,
-    "**/__pycache__/**": true,
-    "**/*.egg-info/**": true,
-    "**/build/**": true,
-    "**/dist/**": true
-  }
-}

README.md

@@ -3,11 +3,10 @@
 </p>
 
 <p align="center">
-  <img src="https://img.shields.io/badge/Python-3.9%20%7C%203.10%20%7C%203.11%20%7C%203.12%20%7C%203.13-blue.svg" alt="Python Versions">
-  <img src="https://github.com/yichuan-w/LEANN/actions/workflows/build-and-publish.yml/badge.svg" alt="CI Status">
-  <img src="https://img.shields.io/badge/Platform-Ubuntu%20%26%20Arch%20%26%20WSL%20%7C%20macOS%20(ARM64%2FIntel)-lightgrey" alt="Platform">
+  <img src="https://img.shields.io/badge/Python-3.9%2B-blue.svg" alt="Python 3.9+">
   <img src="https://img.shields.io/badge/License-MIT-green.svg" alt="MIT License">
-  <img src="https://img.shields.io/badge/MCP-Native%20Integration-blue" alt="MCP Integration">
+  <img src="https://img.shields.io/badge/Platform-Linux%20%7C%20macOS-lightgrey" alt="Platform">
+  <img src="https://img.shields.io/badge/MCP-Native%20Integration-blue?style=flat-square" alt="MCP Integration">
 </p>
 
 <h2 align="center" tabindex="-1" class="heading-element" dir="auto">
@@ -31,7 +30,7 @@ LEANN achieves this through *graph-based selective recomputation* with *high-deg
   <img src="assets/effects.png" alt="LEANN vs Traditional Vector DB Storage Comparison" width="70%">
 </p>
 
-> **The numbers speak for themselves:** Index 60 million text chunks in just 6GB instead of 201GB. From emails to browser history, everything fits on your laptop. [See detailed benchmarks for different applications below ↓](#-storage-comparison)
+> **The numbers speak for themselves:** Index 60 million text chunks in just 6GB instead of 201GB. From emails to browser history, everything fits on your laptop. [See detailed benchmarks for different applications below ↓](#storage-comparison)
 
 
 🔒 **Privacy:** Your data never leaves your laptop. No OpenAI, no cloud, no "terms of service".
@@ -70,8 +69,6 @@ uv venv
 source .venv/bin/activate
 uv pip install leann
 ```
-<!--
-> Low-resource? See “Low-resource setups” in the [Configuration Guide](docs/configuration-guide.md#low-resource-setups). -->
 
 <details>
 <summary>
@@ -87,65 +84,19 @@ git submodule update --init --recursive
 ```
 
 **macOS:**
-
-Note: DiskANN requires MacOS 13.3 or later.
-
 ```bash
-brew install libomp boost protobuf zeromq pkgconf
-uv sync --extra diskann
+brew install llvm libomp boost protobuf zeromq pkgconf
+CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv sync
 ```
 
-**Linux (Ubuntu/Debian):**
-
-Note: On Ubuntu 20.04, you may need to build a newer Abseil and pin Protobuf (e.g., v3.20.x) for building DiskANN. See [Issue #30](https://github.com/yichuan-w/LEANN/issues/30) for a step-by-step note.
-
-You can manually install [Intel oneAPI MKL](https://www.intel.com/content/www/us/en/developer/tools/oneapi/onemkl.html) instead of `libmkl-full-dev` for DiskANN. You can also use `libopenblas-dev` for building HNSW only, by removing `--extra diskann` in the command below.
-
+**Linux:**
 ```bash
-sudo apt-get update && sudo apt-get install -y \
-  libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \
-  pkg-config libabsl-dev libaio-dev libprotobuf-dev \
-  libmkl-full-dev
-
-uv sync --extra diskann
-```
-
-**Linux (Arch Linux):**
-
-```bash
-sudo pacman -Syu && sudo pacman -S --needed base-devel cmake pkgconf git gcc \
-  boost boost-libs protobuf abseil-cpp libaio zeromq
-
-# For MKL in DiskANN
-sudo pacman -S --needed base-devel git
-git clone https://aur.archlinux.org/paru-bin.git
-cd paru-bin && makepkg -si
-paru -S intel-oneapi-mkl intel-oneapi-compiler
-source /opt/intel/oneapi/setvars.sh
-
-uv sync --extra diskann
-```
-
-**Linux (RHEL / CentOS Stream / Oracle / Rocky / AlmaLinux):**
-
-See [Issue #50](https://github.com/yichuan-w/LEANN/issues/50) for more details.
-
-```bash
-sudo dnf groupinstall -y "Development Tools"
-sudo dnf install -y libomp-devel boost-devel protobuf-compiler protobuf-devel \
-  abseil-cpp-devel libaio-devel zeromq-devel pkgconf-pkg-config
-
-# For MKL in DiskANN
-sudo dnf install -y intel-oneapi-mkl intel-oneapi-mkl-devel \
-  intel-oneapi-openmp || sudo dnf install -y intel-oneapi-compiler
-source /opt/intel/oneapi/setvars.sh
-
-uv sync --extra diskann
+sudo apt-get install libomp-dev libboost-all-dev protobuf-compiler libabsl-dev libmkl-full-dev libaio-dev libzmq3-dev
+uv sync
 ```
 
 </details>
 
-
 ## Quick Start
 
 Our declarative API makes RAG as easy as writing a config file.
@@ -176,9 +127,6 @@ response = chat.ask("How much storage does LEANN save?", top_k=1)
 
 LEANN supports RAG on various data sources including documents (`.pdf`, `.txt`, `.md`), Apple Mail, Google Search History, WeChat, and more.
 
-**AST-Aware Code Chunking** - LEANN also features intelligent code chunking that preserves semantic boundaries (functions, classes, methods) for Python, Java, C#, and TypeScript files, providing improved code understanding compared to traditional text-based approaches.
-📖 Read the [AST Chunking Guide →](docs/ast_chunking_guide.md) to learn more.
-
 ### Generation Model Setup
 
 LEANN supports multiple LLM providers for text generation (OpenAI API, HuggingFace, Ollama).
@@ -234,34 +182,34 @@ All RAG examples share these common parameters. **Interactive mode** is availabl
 
 ```bash
 # Core Parameters (General preprocessing for all examples)
---index-dir DIR              # Directory to store the index (default: current directory)
---query "YOUR QUESTION"      # Single query mode. Omit for interactive chat (type 'quit' to exit), and now you can play with your index interactively
---max-items N                # Limit data preprocessing (default: -1, process all data)
---force-rebuild              # Force rebuild index even if it exists
+--index-dir DIR          # Directory to store the index (default: current directory)
+--query "YOUR QUESTION"  # Single query mode. Omit for interactive chat (type 'quit' to exit), and now you can play with your index interactively
+--max-items N           # Limit data preprocessing (default: -1, process all data)
+--force-rebuild         # Force rebuild index even if it exists
 
 # Embedding Parameters
---embedding-model MODEL      # e.g., facebook/contriever, text-embedding-3-small, mlx-community/Qwen3-Embedding-0.6B-8bit or nomic-embed-text
---embedding-mode MODE        # sentence-transformers, openai, mlx, or ollama
+--embedding-model MODEL  # e.g., facebook/contriever, text-embedding-3-small, nomic-embed-text, or mlx-community/multilingual-e5-base-mlx
+--embedding-mode MODE    # sentence-transformers, openai, mlx, or ollama
 
 # LLM Parameters (Text generation models)
---llm TYPE                   # LLM backend: openai, ollama, or hf (default: openai)
---llm-model MODEL            # Model name (default: gpt-4o) e.g., gpt-4o-mini, llama3.2:1b, Qwen/Qwen2.5-1.5B-Instruct
---thinking-budget LEVEL      # Thinking budget for reasoning models: low/medium/high (supported by o3, o3-mini, GPT-Oss:20b, and other reasoning models)
+--llm TYPE              # LLM backend: openai, ollama, or hf (default: openai)
+--llm-model MODEL       # Model name (default: gpt-4o) e.g., gpt-4o-mini, llama3.2:1b, Qwen/Qwen2.5-1.5B-Instruct
+--thinking-budget LEVEL # Thinking budget for reasoning models: low/medium/high (supported by o3, o3-mini, GPT-Oss:20b, and other reasoning models)
 
 # Search Parameters
---top-k N                    # Number of results to retrieve (default: 20)
---search-complexity N        # Search complexity for graph traversal (default: 32)
+--top-k N               # Number of results to retrieve (default: 20)
+--search-complexity N   # Search complexity for graph traversal (default: 32)
 
 # Chunking Parameters
---chunk-size N               # Size of text chunks (default varies by source: 256 for most, 192 for WeChat)
---chunk-overlap N            # Overlap between chunks (default varies: 25-128 depending on source)
+--chunk-size N          # Size of text chunks (default varies by source: 256 for most, 192 for WeChat)
+--chunk-overlap N       # Overlap between chunks (default varies: 25-128 depending on source)
 
 # Index Building Parameters
---backend-name NAME          # Backend to use: hnsw or diskann (default: hnsw)
---graph-degree N             # Graph degree for index construction (default: 32)
---build-complexity N         # Build complexity for index construction (default: 64)
---compact / --no-compact     # Use compact storage (default: true). Must be `no-compact` for `no-recompute` build.
---recompute / --no-recompute # Enable/disable embedding recomputation (default: enabled). Should not do a `no-recompute` search in a `recompute` build.
+--backend-name NAME     # Backend to use: hnsw or diskann (default: hnsw)
+--graph-degree N        # Graph degree for index construction (default: 32)
+--build-complexity N    # Build complexity for index construction (default: 64)
+--no-compact           # Disable compact index storage (compact storage IS enabled to save storage by default)
+--no-recompute         # Disable embedding recomputation (recomputation IS enabled to save storage by default)
 ```
 
 </details>
@@ -297,12 +245,6 @@ python -m apps.document_rag --data-dir "~/Documents/Papers" --chunk-size 1024
 
 # Filter only markdown and Python files with smaller chunks
 python -m apps.document_rag --data-dir "./docs" --chunk-size 256 --file-types .md .py
-
-# Enable AST-aware chunking for code files
-python -m apps.document_rag --enable-code-chunking --data-dir "./my_project"
-
-# Or use the specialized code RAG for better code understanding
-python -m apps.code_rag --repo-dir "./my_codebase" --query "How does authentication work?"
 ```
 
 </details>
@@ -480,22 +422,21 @@ Once the index is built, you can ask questions like:
 **The future of code assistance is here.** Transform your development workflow with LEANN's native MCP integration for Claude Code. Index your entire codebase and get intelligent code assistance directly in your IDE.
 
 **Key features:**
-- 🔍 **Semantic code search** across your entire project, fully local index and lightweight
-- 🧠 **AST-aware chunking** preserves code structure (functions, classes)
+- 🔍 **Semantic code search** across your entire project
 - 📚 **Context-aware assistance** for debugging and development
 - 🚀 **Zero-config setup** with automatic language detection
 
 ```bash
 # Install LEANN globally for MCP integration
-uv tool install leann-core --with leann
-claude mcp add --scope user leann-server -- leann_mcp
+uv tool install leann-core
+
 # Setup is automatic - just start using Claude Code!
 ```
 Try our fully agentic pipeline with auto query rewriting, semantic search planning, and more:
 
 ![LEANN MCP Integration](assets/mcp_leann.png)
 
-**🔥 Ready to supercharge your coding?** [Complete Setup Guide →](packages/leann-mcp/README.md)
+**Ready to supercharge your coding?** [Complete Setup Guide →](packages/leann-mcp/README.md)
 
 ## 🖥️ Command Line Interface
 
@@ -512,8 +453,7 @@ leann --help
 **To make it globally available:**
 ```bash
 # Install the LEANN CLI globally using uv tool
-uv tool install leann-core --with leann
-
+uv tool install leann-core
 
 # Now you can use leann from anywhere without activating venv
 leann --help
@@ -526,7 +466,7 @@ leann --help
 ### Usage Examples
 
 ```bash
-# build from a specific directory, and my_docs is the index name(Here you can also build from multiple dict or multiple files)
+# build from a specific directory, and my_docs is the index name
 leann build my-docs --docs ./your_documents
 
 # Search your documents
@@ -537,36 +477,30 @@ leann ask my-docs --interactive
 
 # List all your indexes
 leann list
-
-# Remove an index
-leann remove my-docs
 ```
 
 **Key CLI features:**
-- Auto-detects document formats (PDF, TXT, MD, DOCX, PPTX + code files)
-- **🧠 AST-aware chunking** for Python, Java, C#, TypeScript files
-- Smart text chunking with overlap for all other content
+- Auto-detects document formats (PDF, TXT, MD, DOCX)
+- Smart text chunking with overlap
 - Multiple LLM providers (Ollama, OpenAI, HuggingFace)
-- Organized index storage in `.leann/indexes/` (project-local)
+- Organized index storage in `~/.leann/indexes/`
 - Support for advanced search parameters
 
 <details>
 <summary><strong>📋 Click to expand: Complete CLI Reference</strong></summary>
 
-You can use `leann --help`, or `leann build --help`, `leann search --help`, `leann ask --help`, `leann list --help`, `leann remove --help` to get the complete CLI reference.
-
 **Build Command:**
 ```bash
-leann build INDEX_NAME --docs DIRECTORY|FILE [DIRECTORY|FILE ...] [OPTIONS]
+leann build INDEX_NAME --docs DIRECTORY [OPTIONS]
 
 Options:
   --backend {hnsw,diskann}     Backend to use (default: hnsw)
   --embedding-model MODEL      Embedding model (default: facebook/contriever)
-  --graph-degree N             Graph degree (default: 32)
-  --complexity N               Build complexity (default: 64)
-  --force                      Force rebuild existing index
-  --compact / --no-compact     Use compact storage (default: true). Must be `no-compact` for `no-recompute` build.
-  --recompute / --no-recompute Enable recomputation (default: true)
+  --graph-degree N            Graph degree (default: 32)
+  --complexity N              Build complexity (default: 64)
+  --force                     Force rebuild existing index
+  --compact                   Use compact storage (default: true)
+  --recompute                 Enable recomputation (default: true)
 ```
 
 **Search Command:**
@@ -574,9 +508,9 @@ Options:
 leann search INDEX_NAME QUERY [OPTIONS]
 
 Options:
-  --top-k N                     Number of results (default: 5)
-  --complexity N                Search complexity (default: 64)
-  --recompute / --no-recompute  Enable/disable embedding recomputation (default: enabled). Should not do a `no-recompute` search in a `recompute` build.
+  --top-k N                   Number of results (default: 5)
+  --complexity N              Search complexity (default: 64)
+  --recompute-embeddings      Use recomputation for highest accuracy
   --pruning-strategy {global,local,proportional}
 ```
 
@@ -591,31 +525,6 @@ Options:
   --top-k N                  Retrieval count (default: 20)
 ```
 
-**List Command:**
-```bash
-leann list
-
-# Lists all indexes across all projects with status indicators:
-# ✅ - Index is complete and ready to use
-# ❌ - Index is incomplete or corrupted
-# 📁 - CLI-created index (in .leann/indexes/)
-# 📄 - App-created index (*.leann.meta.json files)
-```
-
-**Remove Command:**
-```bash
-leann remove INDEX_NAME [OPTIONS]
-
-Options:
-  --force, -f    Force removal without confirmation
-
-# Smart removal: automatically finds and safely removes indexes
-# - Shows all matching indexes across projects
-# - Requires confirmation for cross-project removal
-# - Interactive selection when multiple matches found
-# - Supports both CLI and app-created indexes
-```
-
 </details>
 
 ## 🏗️ Architecture & How It Works
@@ -657,7 +566,6 @@ Options:
 ```bash
 uv pip install -e ".[dev]"  # Install dev dependencies
 python benchmarks/run_evaluation.py    # Will auto-download evaluation data and run benchmarks
-python benchmarks/run_evaluation.py benchmarks/data/indices/rpj_wiki/rpj_wiki --num-queries 2000    # After downloading data, you can run the benchmark with our biggest index
 ```
 
 The evaluation script downloads data automatically on first run. The last three results were tested with partial personal data, and you can reproduce them with your own data!
@@ -701,9 +609,8 @@ We welcome more contributors! Feel free to open issues or submit PRs.
 
 This work is done at [**Berkeley Sky Computing Lab**](https://sky.cs.berkeley.edu/).
 
-## Star History
+---
 
-[![Star History Chart](https://api.star-history.com/svg?repos=yichuan-w/LEANN&type=Date)](https://www.star-history.com/#yichuan-w/LEANN&Date)
 <p align="center">
   <strong>⭐ Star us on GitHub if Leann is useful for your research or applications!</strong>
 </p>

apps/base_rag_example.py

@@ -10,7 +10,7 @@ from typing import Any
 
 import dotenv
 from leann.api import LeannBuilder, LeannChat
-from leann.registry import register_project_directory
+from llama_index.core.node_parser import SentenceSplitter
 
 dotenv.load_dotenv()
 
@@ -69,14 +69,14 @@ class BaseRAGExample(ABC):
             "--embedding-model",
             type=str,
             default=embedding_model_default,
-            help=f"Embedding model to use (default: {embedding_model_default}), we provide facebook/contriever, text-embedding-3-small,mlx-community/Qwen3-Embedding-0.6B-8bit or nomic-embed-text",
+            help=f"Embedding model to use (default: {embedding_model_default})",
         )
         embedding_group.add_argument(
             "--embedding-mode",
             type=str,
             default="sentence-transformers",
             choices=["sentence-transformers", "openai", "mlx", "ollama"],
-            help="Embedding backend mode (default: sentence-transformers), we provide sentence-transformers, openai, mlx, or ollama",
+            help="Embedding backend mode (default: sentence-transformers)",
         )
 
         # LLM parameters
@@ -86,13 +86,13 @@ class BaseRAGExample(ABC):
             type=str,
             default="openai",
             choices=["openai", "ollama", "hf", "simulated"],
-            help="LLM backend: openai, ollama, or hf (default: openai)",
+            help="LLM backend to use (default: openai)",
         )
         llm_group.add_argument(
             "--llm-model",
             type=str,
             default=None,
-            help="Model name (default: gpt-4o) e.g., gpt-4o-mini, llama3.2:1b, Qwen/Qwen2.5-1.5B-Instruct",
+            help="LLM model name (default: gpt-4o for openai, llama3.2:1b for ollama)",
         )
         llm_group.add_argument(
             "--llm-host",
@@ -108,38 +108,6 @@ class BaseRAGExample(ABC):
             help="Thinking budget for reasoning models (low/medium/high). Supported by GPT-Oss:20b and other reasoning models.",
         )
 
-        # AST Chunking parameters
-        ast_group = parser.add_argument_group("AST Chunking Parameters")
-        ast_group.add_argument(
-            "--use-ast-chunking",
-            action="store_true",
-            help="Enable AST-aware chunking for code files (requires astchunk)",
-        )
-        ast_group.add_argument(
-            "--ast-chunk-size",
-            type=int,
-            default=512,
-            help="Maximum characters per AST chunk (default: 512)",
-        )
-        ast_group.add_argument(
-            "--ast-chunk-overlap",
-            type=int,
-            default=64,
-            help="Overlap between AST chunks (default: 64)",
-        )
-        ast_group.add_argument(
-            "--code-file-extensions",
-            nargs="+",
-            default=None,
-            help="Additional code file extensions to process with AST chunking (e.g., .py .java .cs .ts)",
-        )
-        ast_group.add_argument(
-            "--ast-fallback-traditional",
-            action="store_true",
-            default=True,
-            help="Fall back to traditional chunking if AST chunking fails (default: True)",
-        )
-
         # Search parameters
         search_group = parser.add_argument_group("Search Parameters")
         search_group.add_argument(
@@ -210,9 +178,6 @@ class BaseRAGExample(ABC):
             config["host"] = args.llm_host
         elif args.llm == "hf":
             config["model"] = args.llm_model or "Qwen/Qwen2.5-1.5B-Instruct"
-        elif args.llm == "simulated":
-            # Simulated LLM doesn't need additional configuration
-            pass
 
         return config
 
@@ -246,11 +211,6 @@ class BaseRAGExample(ABC):
         builder.build_index(index_path)
         print(f"Index saved to: {index_path}")
 
-        # Register project directory so leann list can discover this index
-        # The index is saved as args.index_dir/index_name.leann
-        # We want to register the current working directory where the app is run
-        register_project_directory(Path.cwd())
-
         return index_path
 
     async def run_interactive_chat(self, args, index_path: str):
@@ -341,3 +301,21 @@ class BaseRAGExample(ABC):
             await self.run_single_query(args, index_path, args.query)
         else:
             await self.run_interactive_chat(args, index_path)
+
+
+def create_text_chunks(documents, chunk_size=256, chunk_overlap=25) -> list[str]:
+    """Helper function to create text chunks from documents."""
+    node_parser = SentenceSplitter(
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+        separator=" ",
+        paragraph_separator="\n\n",
+    )
+
+    all_texts = []
+    for doc in documents:
+        nodes = node_parser.get_nodes_from_documents([doc])
+        if nodes:
+            all_texts.extend(node.get_content() for node in nodes)
+
+    return all_texts

apps/chunking/__init__.py

@@ -1,22 +0,0 @@
-"""
-Chunking utilities for LEANN RAG applications.
-Provides AST-aware and traditional text chunking functionality.
-"""
-
-from .utils import (
-    CODE_EXTENSIONS,
-    create_ast_chunks,
-    create_text_chunks,
-    create_traditional_chunks,
-    detect_code_files,
-    get_language_from_extension,
-)
-
-__all__ = [
-    "CODE_EXTENSIONS",
-    "create_ast_chunks",
-    "create_text_chunks",
-    "create_traditional_chunks",
-    "detect_code_files",
-    "get_language_from_extension",
-]

apps/chunking/utils.py

@@ -1,320 +0,0 @@
-"""
-Enhanced chunking utilities with AST-aware code chunking support.
-Provides unified interface for both traditional and AST-based text chunking.
-"""
-
-import logging
-from pathlib import Path
-from typing import Optional
-
-from llama_index.core.node_parser import SentenceSplitter
-
-logger = logging.getLogger(__name__)
-
-# Code file extensions supported by astchunk
-CODE_EXTENSIONS = {
-    ".py": "python",
-    ".java": "java",
-    ".cs": "csharp",
-    ".ts": "typescript",
-    ".tsx": "typescript",
-    ".js": "typescript",
-    ".jsx": "typescript",
-}
-
-# Default chunk parameters for different content types
-DEFAULT_CHUNK_PARAMS = {
-    "code": {
-        "max_chunk_size": 512,
-        "chunk_overlap": 64,
-    },
-    "text": {
-        "chunk_size": 256,
-        "chunk_overlap": 128,
-    },
-}
-
-
-def detect_code_files(documents, code_extensions=None) -> tuple[list, list]:
-    """
-    Separate documents into code files and regular text files.
-
-    Args:
-        documents: List of LlamaIndex Document objects
-        code_extensions: Dict mapping file extensions to languages (defaults to CODE_EXTENSIONS)
-
-    Returns:
-        Tuple of (code_documents, text_documents)
-    """
-    if code_extensions is None:
-        code_extensions = CODE_EXTENSIONS
-
-    code_docs = []
-    text_docs = []
-
-    for doc in documents:
-        # Get file path from metadata
-        file_path = doc.metadata.get("file_path", "")
-        if not file_path:
-            # Fallback to file_name
-            file_path = doc.metadata.get("file_name", "")
-
-        if file_path:
-            file_ext = Path(file_path).suffix.lower()
-            if file_ext in code_extensions:
-                # Add language info to metadata
-                doc.metadata["language"] = code_extensions[file_ext]
-                doc.metadata["is_code"] = True
-                code_docs.append(doc)
-            else:
-                doc.metadata["is_code"] = False
-                text_docs.append(doc)
-        else:
-            # If no file path, treat as text
-            doc.metadata["is_code"] = False
-            text_docs.append(doc)
-
-    logger.info(f"Detected {len(code_docs)} code files and {len(text_docs)} text files")
-    return code_docs, text_docs
-
-
-def get_language_from_extension(file_path: str) -> Optional[str]:
-    """Get the programming language from file extension."""
-    ext = Path(file_path).suffix.lower()
-    return CODE_EXTENSIONS.get(ext)
-
-
-def create_ast_chunks(
-    documents,
-    max_chunk_size: int = 512,
-    chunk_overlap: int = 64,
-    metadata_template: str = "default",
-) -> list[str]:
-    """
-    Create AST-aware chunks from code documents using astchunk.
-
-    Args:
-        documents: List of code documents
-        max_chunk_size: Maximum characters per chunk
-        chunk_overlap: Number of AST nodes to overlap between chunks
-        metadata_template: Template for chunk metadata
-
-    Returns:
-        List of text chunks with preserved code structure
-    """
-    try:
-        from astchunk import ASTChunkBuilder
-    except ImportError as e:
-        logger.error(f"astchunk not available: {e}")
-        logger.info("Falling back to traditional chunking for code files")
-        return create_traditional_chunks(documents, max_chunk_size, chunk_overlap)
-
-    all_chunks = []
-
-    for doc in documents:
-        # Get language from metadata (set by detect_code_files)
-        language = doc.metadata.get("language")
-        if not language:
-            logger.warning(
-                "No language detected for document, falling back to traditional chunking"
-            )
-            traditional_chunks = create_traditional_chunks([doc], max_chunk_size, chunk_overlap)
-            all_chunks.extend(traditional_chunks)
-            continue
-
-        try:
-            # Configure astchunk
-            configs = {
-                "max_chunk_size": max_chunk_size,
-                "language": language,
-                "metadata_template": metadata_template,
-                "chunk_overlap": chunk_overlap if chunk_overlap > 0 else 0,
-            }
-
-            # Add repository-level metadata if available
-            repo_metadata = {
-                "file_path": doc.metadata.get("file_path", ""),
-                "file_name": doc.metadata.get("file_name", ""),
-                "creation_date": doc.metadata.get("creation_date", ""),
-                "last_modified_date": doc.metadata.get("last_modified_date", ""),
-            }
-            configs["repo_level_metadata"] = repo_metadata
-
-            # Create chunk builder and process
-            chunk_builder = ASTChunkBuilder(**configs)
-            code_content = doc.get_content()
-
-            if not code_content or not code_content.strip():
-                logger.warning("Empty code content, skipping")
-                continue
-
-            chunks = chunk_builder.chunkify(code_content)
-
-            # Extract text content from chunks
-            for chunk in chunks:
-                if hasattr(chunk, "text"):
-                    chunk_text = chunk.text
-                elif isinstance(chunk, dict) and "text" in chunk:
-                    chunk_text = chunk["text"]
-                elif isinstance(chunk, str):
-                    chunk_text = chunk
-                else:
-                    # Try to convert to string
-                    chunk_text = str(chunk)
-
-                if chunk_text and chunk_text.strip():
-                    all_chunks.append(chunk_text.strip())
-
-            logger.info(
-                f"Created {len(chunks)} AST chunks from {language} file: {doc.metadata.get('file_name', 'unknown')}"
-            )
-
-        except Exception as e:
-            logger.warning(f"AST chunking failed for {language} file: {e}")
-            logger.info("Falling back to traditional chunking")
-            traditional_chunks = create_traditional_chunks([doc], max_chunk_size, chunk_overlap)
-            all_chunks.extend(traditional_chunks)
-
-    return all_chunks
-
-
-def create_traditional_chunks(
-    documents, chunk_size: int = 256, chunk_overlap: int = 128
-) -> list[str]:
-    """
-    Create traditional text chunks using LlamaIndex SentenceSplitter.
-
-    Args:
-        documents: List of documents to chunk
-        chunk_size: Size of each chunk in characters
-        chunk_overlap: Overlap between chunks
-
-    Returns:
-        List of text chunks
-    """
-    # Handle invalid chunk_size values
-    if chunk_size <= 0:
-        logger.warning(f"Invalid chunk_size={chunk_size}, using default value of 256")
-        chunk_size = 256
-
-    # Ensure chunk_overlap is not negative and not larger than chunk_size
-    if chunk_overlap < 0:
-        chunk_overlap = 0
-    if chunk_overlap >= chunk_size:
-        chunk_overlap = chunk_size // 2
-
-    node_parser = SentenceSplitter(
-        chunk_size=chunk_size,
-        chunk_overlap=chunk_overlap,
-        separator=" ",
-        paragraph_separator="\n\n",
-    )
-
-    all_texts = []
-    for doc in documents:
-        try:
-            nodes = node_parser.get_nodes_from_documents([doc])
-            if nodes:
-                chunk_texts = [node.get_content() for node in nodes]
-                all_texts.extend(chunk_texts)
-                logger.debug(f"Created {len(chunk_texts)} traditional chunks from document")
-        except Exception as e:
-            logger.error(f"Traditional chunking failed for document: {e}")
-            # As last resort, add the raw content
-            content = doc.get_content()
-            if content and content.strip():
-                all_texts.append(content.strip())
-
-    return all_texts
-
-
-def create_text_chunks(
-    documents,
-    chunk_size: int = 256,
-    chunk_overlap: int = 128,
-    use_ast_chunking: bool = False,
-    ast_chunk_size: int = 512,
-    ast_chunk_overlap: int = 64,
-    code_file_extensions: Optional[list[str]] = None,
-    ast_fallback_traditional: bool = True,
-) -> list[str]:
-    """
-    Create text chunks from documents with optional AST support for code files.
-
-    Args:
-        documents: List of LlamaIndex Document objects
-        chunk_size: Size for traditional text chunks
-        chunk_overlap: Overlap for traditional text chunks
-        use_ast_chunking: Whether to use AST chunking for code files
-        ast_chunk_size: Size for AST chunks
-        ast_chunk_overlap: Overlap for AST chunks
-        code_file_extensions: Custom list of code file extensions
-        ast_fallback_traditional: Fall back to traditional chunking on AST errors
-
-    Returns:
-        List of text chunks
-    """
-    if not documents:
-        logger.warning("No documents provided for chunking")
-        return []
-
-    # Create a local copy of supported extensions for this function call
-    local_code_extensions = CODE_EXTENSIONS.copy()
-
-    # Update supported extensions if provided
-    if code_file_extensions:
-        # Map extensions to languages (simplified mapping)
-        ext_mapping = {
-            ".py": "python",
-            ".java": "java",
-            ".cs": "c_sharp",
-            ".ts": "typescript",
-            ".tsx": "typescript",
-        }
-        for ext in code_file_extensions:
-            if ext.lower() not in local_code_extensions:
-                # Try to guess language from extension
-                if ext.lower() in ext_mapping:
-                    local_code_extensions[ext.lower()] = ext_mapping[ext.lower()]
-                else:
-                    logger.warning(f"Unsupported extension {ext}, will use traditional chunking")
-
-    all_chunks = []
-
-    if use_ast_chunking:
-        # Separate code and text documents using local extensions
-        code_docs, text_docs = detect_code_files(documents, local_code_extensions)
-
-        # Process code files with AST chunking
-        if code_docs:
-            logger.info(f"Processing {len(code_docs)} code files with AST chunking")
-            try:
-                ast_chunks = create_ast_chunks(
-                    code_docs, max_chunk_size=ast_chunk_size, chunk_overlap=ast_chunk_overlap
-                )
-                all_chunks.extend(ast_chunks)
-                logger.info(f"Created {len(ast_chunks)} AST chunks from code files")
-            except Exception as e:
-                logger.error(f"AST chunking failed: {e}")
-                if ast_fallback_traditional:
-                    logger.info("Falling back to traditional chunking for code files")
-                    traditional_code_chunks = create_traditional_chunks(
-                        code_docs, chunk_size, chunk_overlap
-                    )
-                    all_chunks.extend(traditional_code_chunks)
-                else:
-                    raise
-
-        # Process text files with traditional chunking
-        if text_docs:
-            logger.info(f"Processing {len(text_docs)} text files with traditional chunking")
-            text_chunks = create_traditional_chunks(text_docs, chunk_size, chunk_overlap)
-            all_chunks.extend(text_chunks)
-            logger.info(f"Created {len(text_chunks)} traditional chunks from text files")
-    else:
-        # Use traditional chunking for all files
-        logger.info(f"Processing {len(documents)} documents with traditional chunking")
-        all_chunks = create_traditional_chunks(documents, chunk_size, chunk_overlap)
-
-    logger.info(f"Total chunks created: {len(all_chunks)}")
-    return all_chunks

apps/code_rag.py

@@ -1,211 +0,0 @@
-"""
-Code RAG example using AST-aware chunking for optimal code understanding.
-Specialized for code repositories with automatic language detection and
-optimized chunking parameters.
-"""
-
-import sys
-from pathlib import Path
-
-# Add parent directory to path for imports
-sys.path.insert(0, str(Path(__file__).parent))
-
-from base_rag_example import BaseRAGExample
-from chunking import CODE_EXTENSIONS, create_text_chunks
-from llama_index.core import SimpleDirectoryReader
-
-
-class CodeRAG(BaseRAGExample):
-    """Specialized RAG example for code repositories with AST-aware chunking."""
-
-    def __init__(self):
-        super().__init__(
-            name="Code",
-            description="Process and query code repositories with AST-aware chunking",
-            default_index_name="code_index",
-        )
-        # Override defaults for code-specific usage
-        self.embedding_model_default = "facebook/contriever"  # Good for code
-        self.max_items_default = -1  # Process all code files by default
-
-    def _add_specific_arguments(self, parser):
-        """Add code-specific arguments."""
-        code_group = parser.add_argument_group("Code Repository Parameters")
-
-        code_group.add_argument(
-            "--repo-dir",
-            type=str,
-            default=".",
-            help="Code repository directory to index (default: current directory)",
-        )
-        code_group.add_argument(
-            "--include-extensions",
-            nargs="+",
-            default=list(CODE_EXTENSIONS.keys()),
-            help="File extensions to include (default: supported code extensions)",
-        )
-        code_group.add_argument(
-            "--exclude-dirs",
-            nargs="+",
-            default=[
-                ".git",
-                "__pycache__",
-                "node_modules",
-                "venv",
-                ".venv",
-                "build",
-                "dist",
-                "target",
-            ],
-            help="Directories to exclude from indexing",
-        )
-        code_group.add_argument(
-            "--max-file-size",
-            type=int,
-            default=1000000,  # 1MB
-            help="Maximum file size in bytes to process (default: 1MB)",
-        )
-        code_group.add_argument(
-            "--include-comments",
-            action="store_true",
-            help="Include comments in chunking (useful for documentation)",
-        )
-        code_group.add_argument(
-            "--preserve-imports",
-            action="store_true",
-            default=True,
-            help="Try to preserve import statements in chunks (default: True)",
-        )
-
-    async def load_data(self, args) -> list[str]:
-        """Load code files and convert to AST-aware chunks."""
-        print(f"🔍 Scanning code repository: {args.repo_dir}")
-        print(f"📁 Including extensions: {args.include_extensions}")
-        print(f"🚫 Excluding directories: {args.exclude_dirs}")
-
-        # Check if repository directory exists
-        repo_path = Path(args.repo_dir)
-        if not repo_path.exists():
-            raise ValueError(f"Repository directory not found: {args.repo_dir}")
-
-        # Load code files with filtering
-        reader_kwargs = {
-            "recursive": True,
-            "encoding": "utf-8",
-            "required_exts": args.include_extensions,
-            "exclude_hidden": True,
-        }
-
-        # Create exclusion filter
-        def file_filter(file_path: str) -> bool:
-            """Filter out unwanted files and directories."""
-            path = Path(file_path)
-
-            # Check file size
-            try:
-                if path.stat().st_size > args.max_file_size:
-                    print(f"⚠️ Skipping large file: {path.name} ({path.stat().st_size} bytes)")
-                    return False
-            except Exception:
-                return False
-
-            # Check if in excluded directory
-            for exclude_dir in args.exclude_dirs:
-                if exclude_dir in path.parts:
-                    return False
-
-            return True
-
-        try:
-            # Load documents with file filtering
-            documents = SimpleDirectoryReader(
-                args.repo_dir,
-                file_extractor=None,  # Use default extractors
-                **reader_kwargs,
-            ).load_data(show_progress=True)
-
-            # Apply custom filtering
-            filtered_docs = []
-            for doc in documents:
-                file_path = doc.metadata.get("file_path", "")
-                if file_filter(file_path):
-                    filtered_docs.append(doc)
-
-            documents = filtered_docs
-
-        except Exception as e:
-            print(f"❌ Error loading code files: {e}")
-            return []
-
-        if not documents:
-            print(
-                f"❌ No code files found in {args.repo_dir} with extensions {args.include_extensions}"
-            )
-            return []
-
-        print(f"✅ Loaded {len(documents)} code files")
-
-        # Show breakdown by language/extension
-        ext_counts = {}
-        for doc in documents:
-            file_path = doc.metadata.get("file_path", "")
-            if file_path:
-                ext = Path(file_path).suffix.lower()
-                ext_counts[ext] = ext_counts.get(ext, 0) + 1
-
-        print("📊 Files by extension:")
-        for ext, count in sorted(ext_counts.items()):
-            print(f"   {ext}: {count} files")
-
-        # Use AST-aware chunking by default for code
-        print(
-            f"🧠 Using AST-aware chunking (chunk_size: {args.ast_chunk_size}, overlap: {args.ast_chunk_overlap})"
-        )
-
-        all_texts = create_text_chunks(
-            documents,
-            chunk_size=256,  # Fallback for non-code files
-            chunk_overlap=64,
-            use_ast_chunking=True,  # Always use AST for code RAG
-            ast_chunk_size=args.ast_chunk_size,
-            ast_chunk_overlap=args.ast_chunk_overlap,
-            code_file_extensions=args.include_extensions,
-            ast_fallback_traditional=True,
-        )
-
-        # Apply max_items limit if specified
-        if args.max_items > 0 and len(all_texts) > args.max_items:
-            print(f"⏳ Limiting to {args.max_items} chunks (from {len(all_texts)})")
-            all_texts = all_texts[: args.max_items]
-
-        print(f"✅ Generated {len(all_texts)} code chunks")
-        return all_texts
-
-
-if __name__ == "__main__":
-    import asyncio
-
-    # Example queries for code RAG
-    print("\n💻 Code RAG Example")
-    print("=" * 50)
-    print("\nExample queries you can try:")
-    print("- 'How does the embedding computation work?'")
-    print("- 'What are the main classes in this codebase?'")
-    print("- 'Show me the search implementation'")
-    print("- 'How is error handling implemented?'")
-    print("- 'What design patterns are used?'")
-    print("- 'Explain the chunking logic'")
-    print("\n🚀 Features:")
-    print("- ✅ AST-aware chunking preserves code structure")
-    print("- ✅ Automatic language detection")
-    print("- ✅ Smart filtering of large files and common excludes")
-    print("- ✅ Optimized for code understanding")
-    print("\nUsage examples:")
-    print("  python -m apps.code_rag --repo-dir ./my_project")
-    print(
-        "  python -m apps.code_rag --include-extensions .py .js --query 'How does authentication work?'"
-    )
-    print("\nOr run without --query for interactive mode\n")
-
-    rag = CodeRAG()
-    asyncio.run(rag.run())

apps/document_rag.py

@@ -9,8 +9,7 @@ from pathlib import Path
 # Add parent directory to path for imports
 sys.path.insert(0, str(Path(__file__).parent))
 
-from base_rag_example import BaseRAGExample
-from chunking import create_text_chunks
+from base_rag_example import BaseRAGExample, create_text_chunks
 from llama_index.core import SimpleDirectoryReader
 
 
@@ -45,11 +44,6 @@ class DocumentRAG(BaseRAGExample):
         doc_group.add_argument(
             "--chunk-overlap", type=int, default=128, help="Text chunk overlap (default: 128)"
         )
-        doc_group.add_argument(
-            "--enable-code-chunking",
-            action="store_true",
-            help="Enable AST-aware chunking for code files in the data directory",
-        )
 
     async def load_data(self, args) -> list[str]:
         """Load documents and convert to text chunks."""
@@ -82,22 +76,9 @@ class DocumentRAG(BaseRAGExample):
 
         print(f"Loaded {len(documents)} documents")
 
-        # Determine chunking strategy
-        use_ast = args.enable_code_chunking or getattr(args, "use_ast_chunking", False)
-
-        if use_ast:
-            print("Using AST-aware chunking for code files")
-
-        # Convert to text chunks with optional AST support
+        # Convert to text chunks
         all_texts = create_text_chunks(
-            documents,
-            chunk_size=args.chunk_size,
-            chunk_overlap=args.chunk_overlap,
-            use_ast_chunking=use_ast,
-            ast_chunk_size=getattr(args, "ast_chunk_size", 512),
-            ast_chunk_overlap=getattr(args, "ast_chunk_overlap", 64),
-            code_file_extensions=getattr(args, "code_file_extensions", None),
-            ast_fallback_traditional=getattr(args, "ast_fallback_traditional", True),
+            documents, chunk_size=args.chunk_size, chunk_overlap=args.chunk_overlap
         )
 
         # Apply max_items limit if specified
@@ -121,10 +102,6 @@ if __name__ == "__main__":
     print(
         "- 'What is the problem of developing pan gu model Huawei meets? (盘古大模型开发中遇到什么问题?)'"
     )
-    print("\n🚀 NEW: Code-aware chunking available!")
-    print("- Use --enable-code-chunking to enable AST-aware chunking for code files")
-    print("- Supports Python, Java, C#, TypeScript files")
-    print("- Better semantic understanding of code structure")
     print("\nOr run without --query for interactive mode\n")
 
     rag = DocumentRAG()

benchmarks/benchmark_no_recompute.py

@@ -1,148 +0,0 @@
-import argparse
-import os
-import time
-from pathlib import Path
-
-from leann import LeannBuilder, LeannSearcher
-
-
-def _meta_exists(index_path: str) -> bool:
-    p = Path(index_path)
-    return (p.parent / f"{p.stem}.meta.json").exists()
-
-
-def ensure_index(index_path: str, backend_name: str, num_docs: int, is_recompute: bool) -> None:
-    # if _meta_exists(index_path):
-    #     return
-    kwargs = {}
-    if backend_name == "hnsw":
-        kwargs["is_compact"] = is_recompute
-    builder = LeannBuilder(
-        backend_name=backend_name,
-        embedding_model=os.getenv("LEANN_EMBED_MODEL", "facebook/contriever"),
-        embedding_mode=os.getenv("LEANN_EMBED_MODE", "sentence-transformers"),
-        graph_degree=32,
-        complexity=64,
-        is_recompute=is_recompute,
-        num_threads=4,
-        **kwargs,
-    )
-    for i in range(num_docs):
-        builder.add_text(
-            f"This is a test document number {i}. It contains some repeated text for benchmarking."
-        )
-    builder.build_index(index_path)
-
-
-def _bench_group(
-    index_path: str,
-    recompute: bool,
-    query: str,
-    repeats: int,
-    complexity: int = 32,
-    top_k: int = 10,
-) -> float:
-    # Independent searcher per group; fixed port when recompute
-    searcher = LeannSearcher(index_path=index_path)
-
-    # Warm-up once
-    _ = searcher.search(
-        query,
-        top_k=top_k,
-        complexity=complexity,
-        recompute_embeddings=recompute,
-    )
-
-    def _once() -> float:
-        t0 = time.time()
-        _ = searcher.search(
-            query,
-            top_k=top_k,
-            complexity=complexity,
-            recompute_embeddings=recompute,
-        )
-        return time.time() - t0
-
-    if repeats <= 1:
-        t = _once()
-    else:
-        vals = [_once() for _ in range(repeats)]
-        vals.sort()
-        t = vals[len(vals) // 2]
-
-    searcher.cleanup()
-    return t
-
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--num-docs", type=int, default=5000)
-    parser.add_argument("--repeats", type=int, default=3)
-    parser.add_argument("--complexity", type=int, default=32)
-    args = parser.parse_args()
-
-    base = Path.cwd() / ".leann" / "indexes" / f"bench_n{args.num_docs}"
-    base.parent.mkdir(parents=True, exist_ok=True)
-    # ---------- Build HNSW variants ----------
-    hnsw_r = str(base / f"hnsw_recompute_n{args.num_docs}.leann")
-    hnsw_nr = str(base / f"hnsw_norecompute_n{args.num_docs}.leann")
-    ensure_index(hnsw_r, "hnsw", args.num_docs, True)
-    ensure_index(hnsw_nr, "hnsw", args.num_docs, False)
-
-    # ---------- Build DiskANN variants ----------
-    diskann_r = str(base / "diskann_r.leann")
-    diskann_nr = str(base / "diskann_nr.leann")
-    ensure_index(diskann_r, "diskann", args.num_docs, True)
-    ensure_index(diskann_nr, "diskann", args.num_docs, False)
-
-    # ---------- Helpers ----------
-    def _size_for(prefix: str) -> int:
-        p = Path(prefix)
-        base_dir = p.parent
-        stem = p.stem
-        total = 0
-        for f in base_dir.iterdir():
-            if f.is_file() and f.name.startswith(stem):
-                total += f.stat().st_size
-        return total
-
-    # ---------- HNSW benchmark ----------
-    t_hnsw_r = _bench_group(
-        hnsw_r, True, "test document number 42", repeats=args.repeats, complexity=args.complexity
-    )
-    t_hnsw_nr = _bench_group(
-        hnsw_nr, False, "test document number 42", repeats=args.repeats, complexity=args.complexity
-    )
-    size_hnsw_r = _size_for(hnsw_r)
-    size_hnsw_nr = _size_for(hnsw_nr)
-
-    print("Benchmark results (HNSW):")
-    print(f"  recompute=True:  search_time={t_hnsw_r:.3f}s, size={size_hnsw_r / 1024 / 1024:.1f}MB")
-    print(
-        f"  recompute=False: search_time={t_hnsw_nr:.3f}s, size={size_hnsw_nr / 1024 / 1024:.1f}MB"
-    )
-    print("  Expectation: no-recompute should be faster but larger on disk.")
-
-    # ---------- DiskANN benchmark ----------
-    t_diskann_r = _bench_group(
-        diskann_r, True, "DiskANN R test doc 123", repeats=args.repeats, complexity=args.complexity
-    )
-    t_diskann_nr = _bench_group(
-        diskann_nr,
-        False,
-        "DiskANN NR test doc 123",
-        repeats=args.repeats,
-        complexity=args.complexity,
-    )
-    size_diskann_r = _size_for(diskann_r)
-    size_diskann_nr = _size_for(diskann_nr)
-
-    print("\nBenchmark results (DiskANN):")
-    print(f"  build(recompute=True, partition): size={size_diskann_r / 1024 / 1024:.1f}MB")
-    print(f"  build(recompute=False):          size={size_diskann_nr / 1024 / 1024:.1f}MB")
-    print(f"  search recompute=True (final rerank): {t_diskann_r:.3f}s")
-    print(f"  search recompute=False (PQ only):     {t_diskann_nr:.3f}s")
-
-
-if __name__ == "__main__":
-    main()

benchmarks/data/.gitattributes

@@ -0,0 +1,82 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.lz4 filter=lfs diff=lfs merge=lfs -text
+*.mds filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Audio files - uncompressed
+*.pcm filter=lfs diff=lfs merge=lfs -text
+*.sam filter=lfs diff=lfs merge=lfs -text
+*.raw filter=lfs diff=lfs merge=lfs -text
+# Audio files - compressed
+*.aac filter=lfs diff=lfs merge=lfs -text
+*.flac filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.ogg filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+# Image files - uncompressed
+*.bmp filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.tiff filter=lfs diff=lfs merge=lfs -text
+# Image files - compressed
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text
+# Video files - compressed
+*.mp4 filter=lfs diff=lfs merge=lfs -text
+*.webm filter=lfs diff=lfs merge=lfs -text
+ground_truth/dpr/id_map.json filter=lfs diff=lfs merge=lfs -text
+indices/dpr/dpr_diskann.passages.idx filter=lfs diff=lfs merge=lfs -text
+indices/dpr/dpr_diskann.passages.jsonl filter=lfs diff=lfs merge=lfs -text
+indices/dpr/dpr_diskann_disk.index filter=lfs diff=lfs merge=lfs -text
+indices/dpr/leann.labels.map filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/leann.labels.map filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.index filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.0.idx filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.0.jsonl filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.1.idx filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.1.jsonl filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.2.idx filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.2.jsonl filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.3.idx filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.3.jsonl filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.4.idx filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.4.jsonl filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.5.idx filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.5.jsonl filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.6.idx filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.6.jsonl filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.7.idx filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.7.jsonl filter=lfs diff=lfs merge=lfs -text

benchmarks/diskann_vs_hnsw_speed_comparison.py

@@ -10,7 +10,6 @@ This benchmark compares search performance between DiskANN and HNSW backends:
 """
 
 import gc
-import multiprocessing as mp
 import tempfile
 import time
 from pathlib import Path
@@ -18,12 +17,6 @@ from typing import Any
 
 import numpy as np
 
-# Prefer 'fork' start method to avoid POSIX semaphore leaks on macOS
-try:
-    mp.set_start_method("fork", force=True)
-except Exception:
-    pass
-
 
 def create_test_texts(n_docs: int) -> list[str]:
     """Create synthetic test documents for benchmarking."""
@@ -120,10 +113,10 @@ def benchmark_backend(
         ]
         score_validity_rate = len(valid_scores) / len(all_scores) if all_scores else 0
 
-        # Clean up (ensure embedding server shutdown and object GC)
+        # Clean up
         try:
-            if hasattr(searcher, "cleanup"):
-                searcher.cleanup()
+            if hasattr(searcher, "__del__"):
+                searcher.__del__()
             del searcher
             del builder
             gc.collect()
@@ -266,21 +259,10 @@ if __name__ == "__main__":
         print(f"\n❌ Benchmark failed: {e}")
         sys.exit(1)
     finally:
-        # Ensure clean exit (forceful to prevent rare hangs from atexit/threads)
+        # Ensure clean exit
         try:
             gc.collect()
             print("\n🧹 Cleanup completed")
-            # Flush stdio to ensure message is visible before hard-exit
-            try:
-                import sys as _sys
-
-                _sys.stdout.flush()
-                _sys.stderr.flush()
-            except Exception:
-                pass
         except Exception:
             pass
-        # Use os._exit to bypass atexit handlers that may hang in rare cases
-        import os as _os
-
-        _os._exit(0)
+        sys.exit(0)

benchmarks/simple_mac_tpt_test.py

@@ -183,9 +183,6 @@ class Benchmark:
         start_time = time.time()
         with torch.no_grad():
             self.model(input_ids=input_ids, attention_mask=attention_mask)
-        # mps sync
-        if torch.backends.mps.is_available():
-            torch.mps.synchronize()
         end_time = time.time()
 
         return end_time - start_time

docs/ast_chunking_guide.md

@@ -1,128 +0,0 @@
-# AST-Aware Code chunking guide
-
-## Overview
-
-This guide covers best practices for using AST-aware code chunking in LEANN. AST chunking provides better semantic understanding of code structure compared to traditional text-based chunking.
-
-## Quick Start
-
-### Basic Usage
-
-```bash
-# Enable AST chunking for mixed content (code + docs)
-python -m apps.document_rag --enable-code-chunking --data-dir ./my_project
-
-# Specialized code repository indexing
-python -m apps.code_rag --repo-dir ./my_codebase
-
-# Global CLI with AST support
-leann build my-code-index --docs ./src --use-ast-chunking
-```
-
-### Installation
-
-```bash
-# Install LEANN with AST chunking support
-uv pip install -e "."
-```
-
-## Best Practices
-
-### When to Use AST Chunking
-
-✅ **Recommended for:**
-- Code repositories with multiple languages
-- Mixed documentation and code content
-- Complex codebases with deep function/class hierarchies
-- When working with Claude Code for code assistance
-
-❌ **Not recommended for:**
-- Pure text documents
-- Very large files (>1MB)
-- Languages not supported by tree-sitter
-
-### Optimal Configuration
-
-```bash
-# Recommended settings for most codebases
-python -m apps.code_rag \
-    --repo-dir ./src \
-    --ast-chunk-size 768 \
-    --ast-chunk-overlap 96 \
-    --exclude-dirs .git __pycache__ node_modules build dist
-```
-
-### Supported Languages
-
-| Extension | Language | Status |
-|-----------|----------|--------|
-| `.py` | Python | ✅ Full support |
-| `.java` | Java | ✅ Full support |
-| `.cs` | C# | ✅ Full support |
-| `.ts`, `.tsx` | TypeScript | ✅ Full support |
-| `.js`, `.jsx` | JavaScript | ✅ Via TypeScript parser |
-
-## Integration Examples
-
-### Document RAG with Code Support
-
-```python
-# Enable code chunking in document RAG
-python -m apps.document_rag \
-    --enable-code-chunking \
-    --data-dir ./project \
-    --query "How does authentication work in the codebase?"
-```
-
-### Claude Code Integration
-
-When using with Claude Code MCP server, AST chunking provides better context for:
-- Code completion and suggestions
-- Bug analysis and debugging
-- Architecture understanding
-- Refactoring assistance
-
-## Troubleshooting
-
-### Common Issues
-
-1. **Fallback to Traditional Chunking**
-   - Normal behavior for unsupported languages
-   - Check logs for specific language support
-
-2. **Performance with Large Files**
-   - Adjust `--max-file-size` parameter
-   - Use `--exclude-dirs` to skip unnecessary directories
-
-3. **Quality Issues**
-   - Try different `--ast-chunk-size` values (512, 768, 1024)
-   - Adjust overlap for better context preservation
-
-### Debug Mode
-
-```bash
-export LEANN_LOG_LEVEL=DEBUG
-python -m apps.code_rag --repo-dir ./my_code
-```
-
-## Migration from Traditional Chunking
-
-Existing workflows continue to work without changes. To enable AST chunking:
-
-```bash
-# Before
-python -m apps.document_rag --chunk-size 256
-
-# After (maintains traditional chunking for non-code files)
-python -m apps.document_rag --enable-code-chunking --chunk-size 256 --ast-chunk-size 768
-```
-
-## References
-
-- [astchunk GitHub Repository](https://github.com/yilinjz/astchunk)
-- [LEANN MCP Integration](../packages/leann-mcp/README.md)
-- [Research Paper](https://arxiv.org/html/2506.15655v1)
-
----
-
-**Note**: AST chunking maintains full backward compatibility while enhancing code understanding capabilities.

docs/configuration-guide.md

@@ -52,7 +52,7 @@ Based on our experience developing LEANN, embedding models fall into three categ
 ### Quick Start: Cloud and Local Embedding Options
 
 **OpenAI Embeddings (Fastest Setup)**
-For immediate testing without local model downloads(also if you [do not have GPU](https://github.com/yichuan-w/LEANN/issues/43) and do not care that much about your document leak, you should use this, we compute the embedding and recompute using openai API):
+For immediate testing without local model downloads:
 ```bash
 # Set OpenAI embeddings (requires OPENAI_API_KEY)
 --embedding-mode openai --embedding-model text-embedding-3-small
@@ -97,23 +97,29 @@ ollama pull nomic-embed-text
 ```
 
 ### DiskANN
-**Best for**: Large datasets, especially when you want `recompute=True`.
+**Best for**: Performance-critical applications and large datasets - **Production-ready with automatic graph partitioning**
 
-**Key advantages:**
-- **Faster search** on large datasets (3x+ speedup vs HNSW in many cases)
-- **Smart storage**: `recompute=True` enables automatic graph partitioning for smaller indexes
-- **Better scaling**: Designed for 100k+ documents
+**How it works:**
+- **Product Quantization (PQ) + Real-time Reranking**: Uses compressed PQ codes for fast graph traversal, then recomputes exact embeddings for final candidates
+- **Automatic Graph Partitioning**: When `is_recompute=True`, automatically partitions large indices and safely removes redundant files to save storage
+- **Superior Speed-Accuracy Trade-off**: Faster search than HNSW while maintaining high accuracy
 
-**Recompute behavior:**
-- `recompute=True` (recommended): Pure PQ traversal + final reranking - faster and enables partitioning
-- `recompute=False`: PQ + partial real distances during traversal - slower but higher accuracy
+**Trade-offs compared to HNSW:**
+- ✅ **Faster search latency** (typically 2-8x speedup)
+- ✅ **Better scaling** for large datasets
+- ✅ **Smart storage management** with automatic partitioning
+- ✅ **Better graph locality** with `--ldg-times` parameter for SSD optimization
+- ⚠️ **Slightly larger index size** due to PQ tables and graph metadata
 
 ```bash
 # Recommended for most use cases
 --backend-name diskann --graph-degree 32 --build-complexity 64
+
+# For large-scale deployments
+--backend-name diskann --graph-degree 64 --build-complexity 128
 ```
 
-**Performance Benchmark**: Run `uv run benchmarks/diskann_vs_hnsw_speed_comparison.py` to compare DiskANN and HNSW on your system.
+**Performance Benchmark**: Run `python benchmarks/diskann_vs_hnsw_speed_comparison.py` to compare DiskANN and HNSW on your system.
 
 ## LLM Selection: Engine and Model Comparison
 
@@ -230,15 +236,9 @@ python apps/document_rag.py --query "What are the main techniques LEANN explores
 
 3. **Use MLX on Apple Silicon** (optional optimization):
    ```bash
-   --embedding-mode mlx --embedding-model mlx-community/Qwen3-Embedding-0.6B-8bit
+   --embedding-mode mlx --embedding-model mlx-community/multilingual-e5-base-mlx
    ```
-    MLX might not be the best choice, as we tested and found that it only offers 1.3x acceleration compared to HF, so maybe using ollama is a better choice for embedding generation
 
-4. **Use Ollama**
-   ```bash
-   --embedding-mode ollama --embedding-model nomic-embed-text
-   ```
-   To discover additional embedding models in ollama, check out https://ollama.com/search?c=embedding or read more about embedding models at https://ollama.com/blog/embedding-models, please do check the model size that works best for you
 ### If Search Quality is Poor
 
 1. **Increase retrieval count**:
@@ -267,114 +267,24 @@ Every configuration choice involves trade-offs:
 
 The key is finding the right balance for your specific use case. Start small and simple, measure performance, then scale up only where needed.
 
-## Low-resource setups
+## Deep Dive: Critical Configuration Decisions
 
-If you don’t have a local GPU or builds/searches are too slow, use one or more of the options below.
+### When to Disable Recomputation
 
-### 1) Use OpenAI embeddings (no local compute)
-
-Fastest path with zero local GPU requirements. Set your API key and use OpenAI embeddings during build and search:
+LEANN's recomputation feature provides exact distance calculations but can be disabled for extreme QPS requirements:
 
 ```bash
-export OPENAI_API_KEY=sk-...
-
-# Build with OpenAI embeddings
-leann build my-index \
-  --embedding-mode openai \
-  --embedding-model text-embedding-3-small
-
-# Search with OpenAI embeddings (recompute at query time)
-leann search my-index "your query" \
-  --recompute
+--no-recompute  # Disable selective recomputation
 ```
 
-### 2) Run remote builds with SkyPilot (cloud GPU)
-
-Offload embedding generation and index building to a GPU VM using [SkyPilot](https://skypilot.readthedocs.io/en/latest/). A template is provided at `sky/leann-build.yaml`.
-
-```bash
-# One-time: install and configure SkyPilot
-pip install skypilot
-
-# Launch with defaults (L4:1) and mount ./data to ~/leann-data; the build runs automatically
-sky launch -c leann-gpu sky/leann-build.yaml
-
-# Override parameters via -e key=value (optional)
-sky launch -c leann-gpu sky/leann-build.yaml \
-  -e index_name=my-index \
-  -e backend=hnsw \
-  -e embedding_mode=sentence-transformers \
-  -e embedding_model=Qwen/Qwen3-Embedding-0.6B
-
-# Copy the built index back to your local .leann (use rsync)
-rsync -Pavz leann-gpu:~/.leann/indexes/my-index ./.leann/indexes/
-```
-
-### 3) Disable recomputation to trade storage for speed
-
-If you need lower latency and have more storage/memory, disable recomputation. This stores full embeddings and avoids recomputing at search time.
-
-```bash
-# Build without recomputation (HNSW requires non-compact in this mode)
-leann build my-index --no-recompute --no-compact
-
-# Search without recomputation
-leann search my-index "your query" --no-recompute
-```
-
-When to use:
-- Extreme low latency requirements (high QPS, interactive assistants)
-- Read-heavy workloads where storage is cheaper than latency
-- No always-available GPU
-
-Constraints:
-- HNSW: when `--no-recompute` is set, LEANN automatically disables compact mode during build
-- DiskANN: supported; `--no-recompute` skips selective recompute during search
-
-Storage impact:
-- Storing N embeddings of dimension D with float32 requires approximately N × D × 4 bytes
-- Example: 1,000,000 chunks × 768 dims × 4 bytes ≈ 2.86 GB (plus graph/metadata)
-
-Converting an existing index (rebuild required):
-```bash
-# Rebuild in-place (ensure you still have original docs or can regenerate chunks)
-leann build my-index --force --no-recompute --no-compact
-```
-
-Python API usage:
-```python
-from leann import LeannSearcher
-
-searcher = LeannSearcher("/path/to/my-index.leann")
-results = searcher.search("your query", top_k=10, recompute_embeddings=False)
-```
-
-Trade-offs:
-- Lower latency and fewer network hops at query time
-- Significantly higher storage (10–100× vs selective recomputation)
-- Slightly larger memory footprint during build and search
-
-Quick benchmark results (`benchmarks/benchmark_no_recompute.py` with 5k texts, complexity=32):
-
-- HNSW
-
-  ```text
-  recompute=True:  search_time=0.818s, size=1.1MB
-  recompute=False: search_time=0.012s, size=16.6MB
-  ```
-
-- DiskANN
-
-  ```text
-  recompute=True:  search_time=0.041s, size=5.9MB
-  recompute=False: search_time=0.013s, size=24.6MB
-  ```
-
-Conclusion:
-- **HNSW**: `no-recompute` is significantly faster (no embedding recomputation) but requires much more storage (stores all embeddings)
-- **DiskANN**: `no-recompute` uses PQ + partial real distances during traversal (slower but higher accuracy), while `recompute=True` uses pure PQ traversal + final reranking (faster traversal, enables build-time partitioning for smaller storage)
-
+**Trade-offs**:
+- **With recomputation** (default): Exact distances, best quality, higher latency, minimal storage (only stores metadata, recomputes embeddings on-demand)
+- **Without recomputation**: Must store full embeddings, significantly higher memory and storage usage (10-100x more), but faster search
 
+**Disable when**:
+- You have abundant storage and memory
+- Need extremely low latency (< 100ms)
+- Running a read-heavy workload where storage cost is acceptable
 
 ## Further Reading
 

docs/features.md

@@ -3,7 +3,6 @@
 ## 🔥 Core Features
 
 - **🔄 Real-time Embeddings** - Eliminate heavy embedding storage with dynamic computation using optimized ZMQ servers and highly optimized search paradigm (overlapping and batching) with highly optimized embedding engine
-- **🧠 AST-Aware Code Chunking** - Intelligent code chunking that preserves semantic boundaries (functions, classes, methods) for Python, Java, C#, and TypeScript files
 - **📈 Scalable Architecture** - Handles millions of documents on consumer hardware; the larger your dataset, the more LEANN can save
 - **🎯 Graph Pruning** - Advanced techniques to minimize the storage overhead of vector search to a limited footprint
 - **🏗️ Pluggable Backends** - HNSW/FAISS (default), with optional DiskANN for large-scale deployments

packages/leann-backend-diskann/CMakeLists.txt

@@ -0,0 +1,8 @@
+# packages/leann-backend-diskann/CMakeLists.txt (simplified version)
+
+cmake_minimum_required(VERSION 3.20)
+project(leann_backend_diskann_wrapper)
+
+# Tell CMake to directly enter the DiskANN submodule and execute its own CMakeLists.txt
+# DiskANN will handle everything itself, including compiling Python bindings
+add_subdirectory(src/third_party/DiskANN)

packages/leann-backend-diskann/leann_backend_diskann/diskann_backend.py

@@ -22,11 +22,6 @@ logger = logging.getLogger(__name__)
 @contextlib.contextmanager
 def suppress_cpp_output_if_needed():
     """Suppress C++ stdout/stderr based on LEANN_LOG_LEVEL"""
-    # In CI we avoid fiddling with low-level file descriptors to prevent aborts
-    if os.getenv("CI") == "true":
-        yield
-        return
-
     log_level = os.getenv("LEANN_LOG_LEVEL", "WARNING").upper()
 
     # Only suppress if log level is WARNING or higher (ERROR, CRITICAL)
@@ -441,14 +436,9 @@ class DiskannSearcher(BaseSearcher):
         else:  # "global"
             use_global_pruning = True
 
-        # Strategy:
-        # - Traversal always uses PQ distances
-        # - If recompute_embeddings=True, do a single final rerank via deferred fetch
-        #   (fetch embeddings for the final candidate set only)
-        # - Do not recompute neighbor distances along the path
-        use_deferred_fetch = True if recompute_embeddings else False
-        recompute_neighors = False  # Expected typo. For backward compatibility.
-
+        # Perform search with suppressed C++ output based on log level
+        use_deferred_fetch = kwargs.get("USE_DEFERRED_FETCH", True)
+        recompute_neighors = False
         with suppress_cpp_output_if_needed():
             labels, distances = self._index.batch_search(
                 query,
@@ -469,3 +459,25 @@ class DiskannSearcher(BaseSearcher):
         string_labels = [[str(int_label) for int_label in batch_labels] for batch_labels in labels]
 
         return {"labels": string_labels, "distances": distances}
+
+    def cleanup(self):
+        """Cleanup DiskANN-specific resources including C++ index."""
+        # Call parent cleanup first
+        super().cleanup()
+
+        # Delete the C++ index to trigger destructors
+        try:
+            if hasattr(self, "_index") and self._index is not None:
+                del self._index
+                self._index = None
+                self._current_zmq_port = None
+        except Exception:
+            pass
+
+        # Force garbage collection to ensure C++ objects are destroyed
+        try:
+            import gc
+
+            gc.collect()
+        except Exception:
+            pass

packages/leann-backend-diskann/leann_backend_diskann/diskann_embedding_server.py

@@ -83,7 +83,9 @@ def create_diskann_embedding_server(
 
     logger.info(f"Loading PassageManager with metadata_file_path: {passages_file}")
     passages = PassageManager(meta["passage_sources"], metadata_file_path=passages_file)
-    logger.info(f"Loaded PassageManager with {len(passages)} passages from metadata")
+    logger.info(
+        f"Loaded PassageManager with {len(passages.global_offset_map)} passages from metadata"
+    )
 
     # Import protobuf after ensuring the path is correct
     try:
@@ -101,9 +103,8 @@ def create_diskann_embedding_server(
         socket.bind(f"tcp://*:{zmq_port}")
         logger.info(f"DiskANN ZMQ REP server listening on port {zmq_port}")
 
-        socket.setsockopt(zmq.RCVTIMEO, 1000)
-        socket.setsockopt(zmq.SNDTIMEO, 1000)
-        socket.setsockopt(zmq.LINGER, 0)
+        socket.setsockopt(zmq.RCVTIMEO, 300000)
+        socket.setsockopt(zmq.SNDTIMEO, 300000)
 
         while True:
             try:
@@ -220,217 +221,30 @@ def create_diskann_embedding_server(
                 traceback.print_exc()
                 raise
 
-    def zmq_server_thread_with_shutdown(shutdown_event):
-        """ZMQ server thread that respects shutdown signal.
-
-        This creates its own REP socket, binds to zmq_port, and periodically
-        checks shutdown_event using recv timeouts to exit cleanly.
-        """
-        logger.info("DiskANN ZMQ server thread started with shutdown support")
-
-        context = zmq.Context()
-        rep_socket = context.socket(zmq.REP)
-        rep_socket.bind(f"tcp://*:{zmq_port}")
-        logger.info(f"DiskANN ZMQ REP server listening on port {zmq_port}")
-
-        # Set receive timeout so we can check shutdown_event periodically
-        rep_socket.setsockopt(zmq.RCVTIMEO, 1000)  # 1 second timeout
-        rep_socket.setsockopt(zmq.SNDTIMEO, 1000)
-        rep_socket.setsockopt(zmq.LINGER, 0)
-
-        try:
-            while not shutdown_event.is_set():
-                try:
-                    e2e_start = time.time()
-                    # REP socket receives single-part messages
-                    message = rep_socket.recv()
-
-                    # Check for empty messages - REP socket requires response to every request
-                    if not message:
-                        logger.warning("Received empty message, sending empty response")
-                        rep_socket.send(b"")
-                        continue
-
-                    # Try protobuf first (same logic as original)
-                    texts = []
-                    is_text_request = False
-
-                    try:
-                        req_proto = embedding_pb2.NodeEmbeddingRequest()
-                        req_proto.ParseFromString(message)
-                        node_ids = list(req_proto.node_ids)
-
-                        # Look up texts by node IDs
-                        for nid in node_ids:
-                            try:
-                                passage_data = passages.get_passage(str(nid))
-                                txt = passage_data["text"]
-                                if not txt:
-                                    raise RuntimeError(f"FATAL: Empty text for passage ID {nid}")
-                                texts.append(txt)
-                            except KeyError:
-                                raise RuntimeError(f"FATAL: Passage with ID {nid} not found")
-
-                        logger.info(f"ZMQ received protobuf request for {len(node_ids)} node IDs")
-                    except Exception:
-                        # Fallback to msgpack for text requests
-                        try:
-                            import msgpack
-
-                            request = msgpack.unpackb(message)
-                            if isinstance(request, list) and all(
-                                isinstance(item, str) for item in request
-                            ):
-                                texts = request
-                                is_text_request = True
-                                logger.info(
-                                    f"ZMQ received msgpack text request for {len(texts)} texts"
-                                )
-                            else:
-                                raise ValueError("Not a valid msgpack text request")
-                        except Exception:
-                            logger.error("Both protobuf and msgpack parsing failed!")
-                            # Send error response
-                            resp_proto = embedding_pb2.NodeEmbeddingResponse()
-                            rep_socket.send(resp_proto.SerializeToString())
-                            continue
-
-                    # Process the request
-                    embeddings = compute_embeddings(texts, model_name, mode=embedding_mode)
-                    logger.info(f"Computed embeddings shape: {embeddings.shape}")
-
-                    # Validation
-                    if np.isnan(embeddings).any() or np.isinf(embeddings).any():
-                        logger.error("NaN or Inf detected in embeddings!")
-                        # Send error response
-                        if is_text_request:
-                            import msgpack
-
-                            response_data = msgpack.packb([])
-                        else:
-                            resp_proto = embedding_pb2.NodeEmbeddingResponse()
-                            response_data = resp_proto.SerializeToString()
-                        rep_socket.send(response_data)
-                        continue
-
-                    # Prepare response based on request type
-                    if is_text_request:
-                        # For direct text requests, return msgpack
-                        import msgpack
-
-                        response_data = msgpack.packb(embeddings.tolist())
-                    else:
-                        # For protobuf requests, return protobuf
-                        resp_proto = embedding_pb2.NodeEmbeddingResponse()
-                        hidden_contiguous = np.ascontiguousarray(embeddings, dtype=np.float32)
-
-                        resp_proto.embeddings_data = hidden_contiguous.tobytes()
-                        resp_proto.dimensions.append(hidden_contiguous.shape[0])
-                        resp_proto.dimensions.append(hidden_contiguous.shape[1])
-
-                        response_data = resp_proto.SerializeToString()
-
-                    # Send response back to the client
-                    rep_socket.send(response_data)
-
-                    e2e_end = time.time()
-                    logger.info(f"⏱️  ZMQ E2E time: {e2e_end - e2e_start:.6f}s")
-
-                except zmq.Again:
-                    # Timeout - check shutdown_event and continue
-                    continue
-                except Exception as e:
-                    if not shutdown_event.is_set():
-                        logger.error(f"Error in ZMQ server loop: {e}")
-                        try:
-                            # Send error response for REP socket
-                            resp_proto = embedding_pb2.NodeEmbeddingResponse()
-                            rep_socket.send(resp_proto.SerializeToString())
-                        except Exception:
-                            pass
-                    else:
-                        logger.info("Shutdown in progress, ignoring ZMQ error")
-                        break
-        finally:
-            try:
-                rep_socket.close(0)
-            except Exception:
-                pass
-            try:
-                context.term()
-            except Exception:
-                pass
-
-        logger.info("DiskANN ZMQ server thread exiting gracefully")
-
-    # Add shutdown coordination
-    shutdown_event = threading.Event()
-
-    def shutdown_zmq_server():
-        """Gracefully shutdown ZMQ server."""
-        logger.info("Initiating graceful shutdown...")
-        shutdown_event.set()
-
-        if zmq_thread.is_alive():
-            logger.info("Waiting for ZMQ thread to finish...")
-            zmq_thread.join(timeout=5)
-            if zmq_thread.is_alive():
-                logger.warning("ZMQ thread did not finish in time")
-
-        # Clean up ZMQ resources
-        try:
-            # Note: socket and context are cleaned up by thread exit
-            logger.info("ZMQ resources cleaned up")
-        except Exception as e:
-            logger.warning(f"Error cleaning ZMQ resources: {e}")
-
-        # Clean up other resources
-        try:
-            import gc
-
-            gc.collect()
-            logger.info("Additional resources cleaned up")
-        except Exception as e:
-            logger.warning(f"Error cleaning additional resources: {e}")
-
-        logger.info("Graceful shutdown completed")
-        sys.exit(0)
-
-    # Register signal handlers within this function scope
-    import signal
-
-    def signal_handler(sig, frame):
-        logger.info(f"Received signal {sig}, shutting down gracefully...")
-        shutdown_zmq_server()
-
-    signal.signal(signal.SIGTERM, signal_handler)
-    signal.signal(signal.SIGINT, signal_handler)
-
-    # Start ZMQ thread (NOT daemon!)
-    zmq_thread = threading.Thread(
-        target=lambda: zmq_server_thread_with_shutdown(shutdown_event),
-        daemon=False,  # Not daemon - we want to wait for it
-    )
+    zmq_thread = threading.Thread(target=zmq_server_thread, daemon=True)
     zmq_thread.start()
     logger.info(f"Started DiskANN ZMQ server thread on port {zmq_port}")
 
     # Keep the main thread alive
     try:
-        while not shutdown_event.is_set():
-            time.sleep(0.1)  # Check shutdown more frequently
+        while True:
+            time.sleep(1)
     except KeyboardInterrupt:
         logger.info("DiskANN Server shutting down...")
-        shutdown_zmq_server()
         return
 
-    # If we reach here, shutdown was triggered by signal
-    logger.info("Main loop exited, process should be shutting down")
-
 
 if __name__ == "__main__":
+    import signal
     import sys
 
-    # Signal handlers are now registered within create_diskann_embedding_server
+    def signal_handler(sig, frame):
+        logger.info(f"Received signal {sig}, shutting down gracefully...")
+        sys.exit(0)
+
+    # Register signal handlers for graceful shutdown
+    signal.signal(signal.SIGTERM, signal_handler)
+    signal.signal(signal.SIGINT, signal_handler)
 
     parser = argparse.ArgumentParser(description="DiskANN Embedding service")
     parser.add_argument("--zmq-port", type=int, default=5555, help="ZMQ port to run on")

packages/leann-backend-diskann/leann_backend_diskann/graph_partition_simple.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+"""
+Simplified Graph Partition Module for LEANN DiskANN Backend
+
+This module provides a simple Python interface for graph partitioning
+that directly calls the existing executables.
+"""
+
+import os
+import subprocess
+import tempfile
+from pathlib import Path
+from typing import Optional
+
+
+def partition_graph_simple(
+    index_prefix_path: str, output_dir: Optional[str] = None, **kwargs
+) -> tuple[str, str]:
+    """
+    Simple function to partition a graph index.
+
+    Args:
+        index_prefix_path: Path to the index prefix (e.g., "/path/to/index")
+        output_dir: Output directory (defaults to parent of index_prefix_path)
+        **kwargs: Additional parameters for graph partitioning
+
+    Returns:
+        Tuple of (disk_graph_index_path, partition_bin_path)
+    """
+    # Set default parameters
+    params = {
+        "gp_times": 10,
+        "lock_nums": 10,
+        "cut": 100,
+        "scale_factor": 1,
+        "data_type": "float",
+        "thread_nums": 10,
+        **kwargs,
+    }
+
+    # Determine output directory
+    if output_dir is None:
+        output_dir = str(Path(index_prefix_path).parent)
+
+    # Find the graph_partition directory
+    current_file = Path(__file__)
+    graph_partition_dir = current_file.parent.parent / "third_party" / "DiskANN" / "graph_partition"
+
+    if not graph_partition_dir.exists():
+        raise RuntimeError(f"Graph partition directory not found: {graph_partition_dir}")
+
+    # Find input index file
+    old_index_file = f"{index_prefix_path}_disk_beam_search.index"
+    if not os.path.exists(old_index_file):
+        old_index_file = f"{index_prefix_path}_disk.index"
+
+    if not os.path.exists(old_index_file):
+        raise RuntimeError(f"Index file not found: {old_index_file}")
+
+    # Create temporary directory for processing
+    with tempfile.TemporaryDirectory() as temp_dir:
+        temp_data_dir = Path(temp_dir) / "data"
+        temp_data_dir.mkdir(parents=True, exist_ok=True)
+
+        # Set up paths for temporary files
+        graph_path = temp_data_dir / "starling" / "_M_R_L_B" / "GRAPH"
+        graph_gp_path = (
+            graph_path
+            / f"GP_TIMES_{params['gp_times']}_LOCK_{params['lock_nums']}_GP_USE_FREQ0_CUT{params['cut']}_SCALE{params['scale_factor']}"
+        )
+        graph_gp_path.mkdir(parents=True, exist_ok=True)
+
+        # Run the build script with our parameters
+        cmd = [str(graph_partition_dir / "build.sh"), "release", "split_graph", index_prefix_path]
+
+        # Set environment variables for parameters
+        env = os.environ.copy()
+        env.update(
+            {
+                "GP_TIMES": str(params["gp_times"]),
+                "GP_LOCK_NUMS": str(params["lock_nums"]),
+                "GP_CUT": str(params["cut"]),
+                "GP_SCALE_F": str(params["scale_factor"]),
+                "DATA_TYPE": params["data_type"],
+                "GP_T": str(params["thread_nums"]),
+            }
+        )
+
+        print(f"Running graph partition with command: {' '.join(cmd)}")
+        print(f"Working directory: {graph_partition_dir}")
+
+        # Run the command
+        result = subprocess.run(
+            cmd, env=env, capture_output=True, text=True, cwd=graph_partition_dir
+        )
+
+        if result.returncode != 0:
+            print(f"Command failed with return code {result.returncode}")
+            print(f"stdout: {result.stdout}")
+            print(f"stderr: {result.stderr}")
+            raise RuntimeError(
+                f"Graph partitioning failed with return code {result.returncode}.\n"
+                f"stdout: {result.stdout}\n"
+                f"stderr: {result.stderr}"
+            )
+
+        # Check if output files were created
+        disk_graph_path = Path(output_dir) / "_disk_graph.index"
+        partition_bin_path = Path(output_dir) / "_partition.bin"
+
+        if not disk_graph_path.exists():
+            raise RuntimeError(f"Expected output file not found: {disk_graph_path}")
+
+        if not partition_bin_path.exists():
+            raise RuntimeError(f"Expected output file not found: {partition_bin_path}")
+
+        print("✅ Partitioning completed successfully!")
+        print(f"   Disk graph index: {disk_graph_path}")
+        print(f"   Partition binary: {partition_bin_path}")
+
+        return str(disk_graph_path), str(partition_bin_path)
+
+
+# Example usage
+if __name__ == "__main__":
+    try:
+        disk_graph_path, partition_bin_path = partition_graph_simple(
+            "/Users/yichuan/Desktop/release2/leann/diskannbuild/test_doc_files",
+            gp_times=5,
+            lock_nums=5,
+            cut=50,
+        )
+        print("Success! Output files:")
+        print(f"  - {disk_graph_path}")
+        print(f"  - {partition_bin_path}")
+    except Exception as e:
+        print(f"Error: {e}")

packages/leann-backend-diskann/pyproject.toml

@@ -4,8 +4,8 @@ build-backend = "scikit_build_core.build"
 
 [project]
 name = "leann-backend-diskann"
-version = "0.3.2"
-dependencies = ["leann-core==0.3.2", "numpy", "protobuf>=3.19.0"]
+version = "0.2.5"
+dependencies = ["leann-core==0.2.5", "numpy", "protobuf>=3.19.0"]
 
 [tool.scikit-build]
 # Key: simplified CMake path
@@ -17,5 +17,3 @@ editable.mode = "redirect"
 cmake.build-type = "Release"
 build.verbose = true
 build.tool-args = ["-j8"]
-# Let CMake find packages via Homebrew prefix
-cmake.define = {CMAKE_PREFIX_PATH = {env = "CMAKE_PREFIX_PATH"}, OpenMP_ROOT = {env = "OpenMP_ROOT"}}

packages/leann-backend-hnsw/CMakeLists.txt

@@ -5,20 +5,11 @@ set(CMAKE_CXX_COMPILER_WORKS 1)
 
 # Set OpenMP path for macOS
 if(APPLE)
-    # Detect Homebrew installation path (Apple Silicon vs Intel)
-    if(EXISTS "/opt/homebrew/opt/libomp")
-        set(HOMEBREW_PREFIX "/opt/homebrew")
-    elseif(EXISTS "/usr/local/opt/libomp")
-        set(HOMEBREW_PREFIX "/usr/local")
-    else()
-        message(FATAL_ERROR "Could not find libomp installation. Please install with: brew install libomp")
-    endif()
-
-    set(OpenMP_C_FLAGS "-Xpreprocessor -fopenmp -I${HOMEBREW_PREFIX}/opt/libomp/include")
-    set(OpenMP_CXX_FLAGS "-Xpreprocessor -fopenmp -I${HOMEBREW_PREFIX}/opt/libomp/include")
+    set(OpenMP_C_FLAGS "-Xpreprocessor -fopenmp -I/opt/homebrew/opt/libomp/include")
+    set(OpenMP_CXX_FLAGS "-Xpreprocessor -fopenmp -I/opt/homebrew/opt/libomp/include")
     set(OpenMP_C_LIB_NAMES "omp")
     set(OpenMP_CXX_LIB_NAMES "omp")
-    set(OpenMP_omp_LIBRARY "${HOMEBREW_PREFIX}/opt/libomp/lib/libomp.dylib")
+    set(OpenMP_omp_LIBRARY "/opt/homebrew/opt/libomp/lib/libomp.dylib")
 
     # Force use of system libc++ to avoid version mismatch
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")

packages/leann-backend-hnsw/leann_backend_hnsw/convert_to_csr.py

@@ -1,6 +1,5 @@
 import argparse
 import gc  # Import garbage collector interface
-import logging
 import os
 import struct
 import sys
@@ -8,12 +7,6 @@ import time
 
 import numpy as np
 
-# Set up logging to avoid print buffer issues
-logger = logging.getLogger(__name__)
-LOG_LEVEL = os.getenv("LEANN_LOG_LEVEL", "WARNING").upper()
-log_level = getattr(logging, LOG_LEVEL, logging.WARNING)
-logger.setLevel(log_level)
-
 # --- FourCCs (add more if needed) ---
 INDEX_HNSW_FLAT_FOURCC = int.from_bytes(b"IHNf", "little")
 # Add other HNSW fourccs if you expect different storage types inside HNSW
@@ -250,8 +243,6 @@ def convert_hnsw_graph_to_csr(input_filename, output_filename, prune_embeddings=
         output_filename: Output CSR index file
         prune_embeddings: Whether to prune embedding storage (write NULL storage marker)
     """
-    # Keep prints simple; rely on CI runner to flush output as needed
-
     print(f"Starting conversion: {input_filename} -> {output_filename}")
     start_time = time.time()
     original_hnsw_data = {}

packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_backend.py

@@ -54,13 +54,12 @@ class HNSWBuilder(LeannBackendBuilderInterface):
         self.efConstruction = self.build_params.setdefault("efConstruction", 200)
         self.distance_metric = self.build_params.setdefault("distance_metric", "mips")
         self.dimensions = self.build_params.get("dimensions")
-        if not self.is_recompute and self.is_compact:
-            # Auto-correct: non-recompute requires non-compact storage for HNSW
-            logger.warning(
-                "is_recompute=False requires non-compact HNSW. Forcing is_compact=False."
-            )
-            self.is_compact = False
-            self.build_params["is_compact"] = False
+        if not self.is_recompute:
+            if self.is_compact:
+                # TODO: support this case @andy
+                raise ValueError(
+                    "is_recompute is False, but is_compact is True. This is not compatible now. change is compact to False and you can use the original HNSW index."
+                )
 
     def build(self, data: np.ndarray, ids: list[str], index_path: str, **kwargs):
         from . import faiss  # type: ignore
@@ -185,11 +184,9 @@ class HNSWSearcher(BaseSearcher):
         """
         from . import faiss  # type: ignore
 
-        if not recompute_embeddings and self.is_pruned:
-            raise RuntimeError(
-                "Recompute is required for pruned/compact HNSW index. "
-                "Re-run search with --recompute, or rebuild with --no-recompute and --no-compact."
-            )
+        if not recompute_embeddings:
+            if self.is_pruned:
+                raise RuntimeError("Recompute is required for pruned index.")
         if recompute_embeddings:
             if zmq_port is None:
                 raise ValueError("zmq_port must be provided if recompute_embeddings is True")

packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_embedding_server.py

@@ -82,315 +82,188 @@ def create_hnsw_embedding_server(
     with open(passages_file) as f:
         meta = json.load(f)
 
-    # Let PassageManager handle path resolution uniformly. It supports fallback order:
-    # 1) path/index_path; 2) *_relative; 3) standard siblings next to meta
+    # Let PassageManager handle path resolution uniformly
     passages = PassageManager(meta["passage_sources"], metadata_file_path=passages_file)
-    # Dimension from metadata for shaping responses
-    try:
-        embedding_dim: int = int(meta.get("dimensions", 0))
-    except Exception:
-        embedding_dim = 0
-    logger.info(f"Loaded PassageManager with {len(passages)} passages from metadata")
-
-    # (legacy ZMQ thread removed; using shutdown-capable server only)
-
-    def zmq_server_thread_with_shutdown(shutdown_event):
-        """ZMQ server thread that respects shutdown signal.
-
-        Creates its own REP socket bound to zmq_port and polls with timeouts
-        to allow graceful shutdown.
-        """
-        logger.info("ZMQ server thread started with shutdown support")
+    logger.info(
+        f"Loaded PassageManager with {len(passages.global_offset_map)} passages from metadata"
+    )
 
+    def zmq_server_thread():
+        """ZMQ server thread"""
         context = zmq.Context()
-        rep_socket = context.socket(zmq.REP)
-        rep_socket.bind(f"tcp://*:{zmq_port}")
-        logger.info(f"HNSW ZMQ REP server listening on port {zmq_port}")
-        rep_socket.setsockopt(zmq.RCVTIMEO, 1000)
-        # Keep sends from blocking during shutdown; fail fast and drop on close
-        rep_socket.setsockopt(zmq.SNDTIMEO, 1000)
-        rep_socket.setsockopt(zmq.LINGER, 0)
+        socket = context.socket(zmq.REP)
+        socket.bind(f"tcp://*:{zmq_port}")
+        logger.info(f"HNSW ZMQ server listening on port {zmq_port}")
 
-        # Track last request type/length for shape-correct fallbacks
-        last_request_type = "unknown"  # 'text' | 'distance' | 'embedding' | 'unknown'
-        last_request_length = 0
+        socket.setsockopt(zmq.RCVTIMEO, 300000)
+        socket.setsockopt(zmq.SNDTIMEO, 300000)
 
-        try:
-            while not shutdown_event.is_set():
-                try:
-                    e2e_start = time.time()
-                    logger.debug("🔍 Waiting for ZMQ message...")
-                    request_bytes = rep_socket.recv()
+        while True:
+            try:
+                message_bytes = socket.recv()
+                logger.debug(f"Received ZMQ request of size {len(message_bytes)} bytes")
 
-                    # Rest of the processing logic (same as original)
-                    request = msgpack.unpackb(request_bytes)
+                e2e_start = time.time()
+                request_payload = msgpack.unpackb(message_bytes)
 
-                    if len(request) == 1 and request[0] == "__QUERY_MODEL__":
-                        response_bytes = msgpack.packb([model_name])
-                        rep_socket.send(response_bytes)
-                        continue
+                # Handle direct text embedding request
+                if isinstance(request_payload, list) and len(request_payload) > 0:
+                    # Check if this is a direct text request (list of strings)
+                    if all(isinstance(item, str) for item in request_payload):
+                        logger.info(
+                            f"Processing direct text embedding request for {len(request_payload)} texts in {embedding_mode} mode"
+                        )
 
-                    # Handle direct text embedding request
-                    if (
-                        isinstance(request, list)
-                        and request
-                        and all(isinstance(item, str) for item in request)
-                    ):
-                        last_request_type = "text"
-                        last_request_length = len(request)
-                        embeddings = compute_embeddings(request, model_name, mode=embedding_mode)
-                        rep_socket.send(msgpack.packb(embeddings.tolist()))
+                        # Use unified embedding computation (now with model caching)
+                        embeddings = compute_embeddings(
+                            request_payload, model_name, mode=embedding_mode
+                        )
+
+                        response = embeddings.tolist()
+                        socket.send(msgpack.packb(response))
                         e2e_end = time.time()
                         logger.info(f"⏱️  Text embedding E2E time: {e2e_end - e2e_start:.6f}s")
                         continue
 
-                    # Handle distance calculation request: [[ids], [query_vector]]
-                    if (
-                        isinstance(request, list)
-                        and len(request) == 2
-                        and isinstance(request[0], list)
-                        and isinstance(request[1], list)
-                    ):
-                        node_ids = request[0]
-                        # Handle nested [[ids]] shape defensively
-                        if len(node_ids) == 1 and isinstance(node_ids[0], list):
-                            node_ids = node_ids[0]
-                        query_vector = np.array(request[1], dtype=np.float32)
-                        last_request_type = "distance"
-                        last_request_length = len(node_ids)
+                # Handle distance calculation requests
+                if (
+                    isinstance(request_payload, list)
+                    and len(request_payload) == 2
+                    and isinstance(request_payload[0], list)
+                    and isinstance(request_payload[1], list)
+                ):
+                    node_ids = request_payload[0]
+                    query_vector = np.array(request_payload[1], dtype=np.float32)
 
-                        logger.debug("Distance calculation request received")
-                        logger.debug(f"    Node IDs: {node_ids}")
-                        logger.debug(f"    Query vector dim: {len(query_vector)}")
+                    logger.debug("Distance calculation request received")
+                    logger.debug(f"    Node IDs: {node_ids}")
+                    logger.debug(f"    Query vector dim: {len(query_vector)}")
 
-                        # Gather texts for found ids
-                        texts: list[str] = []
-                        found_indices: list[int] = []
-                        for idx, nid in enumerate(node_ids):
-                            try:
-                                passage_data = passages.get_passage(str(nid))
-                                txt = passage_data.get("text", "")
-                                if isinstance(txt, str) and len(txt) > 0:
-                                    texts.append(txt)
-                                    found_indices.append(idx)
-                                else:
-                                    logger.error(f"Empty text for passage ID {nid}")
-                            except KeyError:
-                                logger.error(f"Passage ID {nid} not found")
-                            except Exception as e:
-                                logger.error(f"Exception looking up passage ID {nid}: {e}")
-
-                        # Prepare full-length response with large sentinel values
-                        large_distance = 1e9
-                        response_distances = [large_distance] * len(node_ids)
-
-                        if texts:
-                            try:
-                                embeddings = compute_embeddings(
-                                    texts, model_name, mode=embedding_mode
-                                )
-                                logger.info(
-                                    f"Computed embeddings for {len(texts)} texts, shape: {embeddings.shape}"
-                                )
-                                if distance_metric == "l2":
-                                    partial = np.sum(
-                                        np.square(embeddings - query_vector.reshape(1, -1)), axis=1
-                                    )
-                                else:  # mips or cosine
-                                    partial = -np.dot(embeddings, query_vector)
-
-                                for pos, dval in zip(found_indices, partial.flatten().tolist()):
-                                    response_distances[pos] = float(dval)
-                            except Exception as e:
-                                logger.error(f"Distance computation error, using sentinels: {e}")
-
-                        # Send response in expected shape [[distances]]
-                        rep_socket.send(msgpack.packb([response_distances], use_single_float=True))
-                        e2e_end = time.time()
-                        logger.info(f"⏱️  Distance calculation E2E time: {e2e_end - e2e_start:.6f}s")
-                        continue
-
-                    # Fallback: treat as embedding-by-id request
-                    if (
-                        isinstance(request, list)
-                        and len(request) == 1
-                        and isinstance(request[0], list)
-                    ):
-                        node_ids = request[0]
-                    elif isinstance(request, list):
-                        node_ids = request
-                    else:
-                        node_ids = []
-                    last_request_type = "embedding"
-                    last_request_length = len(node_ids)
-                    logger.info(f"ZMQ received {len(node_ids)} node IDs for embedding fetch")
-
-                    # Preallocate zero-filled flat data for robustness
-                    if embedding_dim <= 0:
-                        dims = [0, 0]
-                        flat_data: list[float] = []
-                    else:
-                        dims = [len(node_ids), embedding_dim]
-                        flat_data = [0.0] * (dims[0] * dims[1])
-
-                    # Collect texts for found ids
-                    texts: list[str] = []
-                    found_indices: list[int] = []
-                    for idx, nid in enumerate(node_ids):
+                    # Get embeddings for node IDs
+                    texts = []
+                    for nid in node_ids:
                         try:
                             passage_data = passages.get_passage(str(nid))
-                            txt = passage_data.get("text", "")
-                            if isinstance(txt, str) and len(txt) > 0:
-                                texts.append(txt)
-                                found_indices.append(idx)
-                            else:
-                                logger.error(f"Empty text for passage ID {nid}")
+                            txt = passage_data["text"]
+                            texts.append(txt)
                         except KeyError:
-                            logger.error(f"Passage with ID {nid} not found")
+                            logger.error(f"Passage ID {nid} not found")
+                            raise RuntimeError(f"FATAL: Passage with ID {nid} not found")
                         except Exception as e:
                             logger.error(f"Exception looking up passage ID {nid}: {e}")
+                            raise
 
-                    if texts:
-                        try:
-                            embeddings = compute_embeddings(texts, model_name, mode=embedding_mode)
-                            logger.info(
-                                f"Computed embeddings for {len(texts)} texts, shape: {embeddings.shape}"
-                            )
+                    # Process embeddings
+                    embeddings = compute_embeddings(texts, model_name, mode=embedding_mode)
+                    logger.info(
+                        f"Computed embeddings for {len(texts)} texts, shape: {embeddings.shape}"
+                    )
 
-                            if np.isnan(embeddings).any() or np.isinf(embeddings).any():
-                                logger.error(
-                                    f"NaN or Inf detected in embeddings! Requested IDs: {node_ids[:5]}..."
-                                )
-                                dims = [0, embedding_dim]
-                                flat_data = []
-                            else:
-                                emb_f32 = np.ascontiguousarray(embeddings, dtype=np.float32)
-                                flat = emb_f32.flatten().tolist()
-                                for j, pos in enumerate(found_indices):
-                                    start = pos * embedding_dim
-                                    end = start + embedding_dim
-                                    if end <= len(flat_data):
-                                        flat_data[start:end] = flat[
-                                            j * embedding_dim : (j + 1) * embedding_dim
-                                        ]
-                        except Exception as e:
-                            logger.error(f"Embedding computation error, returning zeros: {e}")
+                    # Calculate distances
+                    if distance_metric == "l2":
+                        distances = np.sum(
+                            np.square(embeddings - query_vector.reshape(1, -1)), axis=1
+                        )
+                    else:  # mips or cosine
+                        distances = -np.dot(embeddings, query_vector)
 
-                    response_payload = [dims, flat_data]
-                    response_bytes = msgpack.packb(response_payload, use_single_float=True)
+                    response_payload = distances.flatten().tolist()
+                    response_bytes = msgpack.packb([response_payload], use_single_float=True)
+                    logger.debug(f"Sending distance response with {len(distances)} distances")
 
-                    rep_socket.send(response_bytes)
+                    socket.send(response_bytes)
                     e2e_end = time.time()
-                    logger.info(f"⏱️  ZMQ E2E time: {e2e_end - e2e_start:.6f}s")
-
-                except zmq.Again:
-                    # Timeout - check shutdown_event and continue
+                    logger.info(f"⏱️  Distance calculation E2E time: {e2e_end - e2e_start:.6f}s")
                     continue
-                except Exception as e:
-                    if not shutdown_event.is_set():
-                        logger.error(f"Error in ZMQ server loop: {e}")
-                        # Shape-correct fallback
-                        try:
-                            if last_request_type == "distance":
-                                large_distance = 1e9
-                                fallback_len = max(0, int(last_request_length))
-                                safe = [[large_distance] * fallback_len]
-                            elif last_request_type == "embedding":
-                                bsz = max(0, int(last_request_length))
-                                dim = max(0, int(embedding_dim))
-                                safe = (
-                                    [[bsz, dim], [0.0] * (bsz * dim)] if dim > 0 else [[0, 0], []]
-                                )
-                            elif last_request_type == "text":
-                                safe = []  # direct text embeddings expectation is a flat list
-                            else:
-                                safe = [[0, int(embedding_dim) if embedding_dim > 0 else 0], []]
-                            rep_socket.send(msgpack.packb(safe, use_single_float=True))
-                        except Exception:
-                            pass
-                    else:
-                        logger.info("Shutdown in progress, ignoring ZMQ error")
-                        break
-        finally:
-            try:
-                rep_socket.close(0)
-            except Exception:
-                pass
-            try:
-                context.term()
-            except Exception:
-                pass
 
-        logger.info("ZMQ server thread exiting gracefully")
+                # Standard embedding request (passage ID lookup)
+                if (
+                    not isinstance(request_payload, list)
+                    or len(request_payload) != 1
+                    or not isinstance(request_payload[0], list)
+                ):
+                    logger.error(
+                        f"Invalid MessagePack request format. Expected [[ids...]] or [texts...], got: {type(request_payload)}"
+                    )
+                    socket.send(msgpack.packb([[], []]))
+                    continue
 
-    # Add shutdown coordination
-    shutdown_event = threading.Event()
+                node_ids = request_payload[0]
+                logger.debug(f"Request for {len(node_ids)} node embeddings")
 
-    def shutdown_zmq_server():
-        """Gracefully shutdown ZMQ server."""
-        logger.info("Initiating graceful shutdown...")
-        shutdown_event.set()
+                # Look up texts by node IDs
+                texts = []
+                for nid in node_ids:
+                    try:
+                        passage_data = passages.get_passage(str(nid))
+                        txt = passage_data["text"]
+                        if not txt:
+                            raise RuntimeError(f"FATAL: Empty text for passage ID {nid}")
+                        texts.append(txt)
+                    except KeyError:
+                        raise RuntimeError(f"FATAL: Passage with ID {nid} not found")
+                    except Exception as e:
+                        logger.error(f"Exception looking up passage ID {nid}: {e}")
+                        raise
 
-        if zmq_thread.is_alive():
-            logger.info("Waiting for ZMQ thread to finish...")
-            zmq_thread.join(timeout=5)
-            if zmq_thread.is_alive():
-                logger.warning("ZMQ thread did not finish in time")
+                # Process embeddings
+                embeddings = compute_embeddings(texts, model_name, mode=embedding_mode)
+                logger.info(
+                    f"Computed embeddings for {len(texts)} texts, shape: {embeddings.shape}"
+                )
 
-        # Clean up ZMQ resources
-        try:
-            # Note: socket and context are cleaned up by thread exit
-            logger.info("ZMQ resources cleaned up")
-        except Exception as e:
-            logger.warning(f"Error cleaning ZMQ resources: {e}")
+                # Serialization and response
+                if np.isnan(embeddings).any() or np.isinf(embeddings).any():
+                    logger.error(
+                        f"NaN or Inf detected in embeddings! Requested IDs: {node_ids[:5]}..."
+                    )
+                    raise AssertionError()
 
-        # Clean up other resources
-        try:
-            import gc
+                hidden_contiguous_f32 = np.ascontiguousarray(embeddings, dtype=np.float32)
+                response_payload = [
+                    list(hidden_contiguous_f32.shape),
+                    hidden_contiguous_f32.flatten().tolist(),
+                ]
+                response_bytes = msgpack.packb(response_payload, use_single_float=True)
 
-            gc.collect()
-            logger.info("Additional resources cleaned up")
-        except Exception as e:
-            logger.warning(f"Error cleaning additional resources: {e}")
+                socket.send(response_bytes)
+                e2e_end = time.time()
+                logger.info(f"⏱️  ZMQ E2E time: {e2e_end - e2e_start:.6f}s")
 
-        logger.info("Graceful shutdown completed")
-        sys.exit(0)
+            except zmq.Again:
+                logger.debug("ZMQ socket timeout, continuing to listen")
+                continue
+            except Exception as e:
+                logger.error(f"Error in ZMQ server loop: {e}")
+                import traceback
 
-    # Register signal handlers within this function scope
-    import signal
+                traceback.print_exc()
+                socket.send(msgpack.packb([[], []]))
 
-    def signal_handler(sig, frame):
-        logger.info(f"Received signal {sig}, shutting down gracefully...")
-        shutdown_zmq_server()
-
-    signal.signal(signal.SIGTERM, signal_handler)
-    signal.signal(signal.SIGINT, signal_handler)
-
-    # Pass shutdown_event to ZMQ thread
-    zmq_thread = threading.Thread(
-        target=lambda: zmq_server_thread_with_shutdown(shutdown_event),
-        daemon=False,  # Not daemon - we want to wait for it
-    )
+    zmq_thread = threading.Thread(target=zmq_server_thread, daemon=True)
     zmq_thread.start()
     logger.info(f"Started HNSW ZMQ server thread on port {zmq_port}")
 
     # Keep the main thread alive
     try:
-        while not shutdown_event.is_set():
-            time.sleep(0.1)  # Check shutdown more frequently
+        while True:
+            time.sleep(1)
     except KeyboardInterrupt:
         logger.info("HNSW Server shutting down...")
-        shutdown_zmq_server()
         return
 
-    # If we reach here, shutdown was triggered by signal
-    logger.info("Main loop exited, process should be shutting down")
-
 
 if __name__ == "__main__":
+    import signal
     import sys
 
-    # Signal handlers are now registered within create_hnsw_embedding_server
+    def signal_handler(sig, frame):
+        logger.info(f"Received signal {sig}, shutting down gracefully...")
+        sys.exit(0)
+
+    # Register signal handlers for graceful shutdown
+    signal.signal(signal.SIGTERM, signal_handler)
+    signal.signal(signal.SIGINT, signal_handler)
 
     parser = argparse.ArgumentParser(description="HNSW Embedding service")
     parser.add_argument("--zmq-port", type=int, default=5555, help="ZMQ port to run on")

packages/leann-backend-hnsw/pyproject.toml

@@ -6,10 +6,10 @@ build-backend = "scikit_build_core.build"
 
 [project]
 name = "leann-backend-hnsw"
-version = "0.3.2"
+version = "0.2.5"
 description = "Custom-built HNSW (Faiss) backend for the Leann toolkit."
 dependencies = [
-    "leann-core==0.3.2",
+    "leann-core==0.2.5",
     "numpy",
     "pyzmq>=23.0.0",
     "msgpack>=1.0.0",
@@ -22,8 +22,6 @@ cmake.build-type = "Release"
 build.verbose = true
 build.tool-args = ["-j8"]
 
-# CMake definitions to optimize compilation and find Homebrew packages
+# CMake definitions to optimize compilation
 [tool.scikit-build.cmake.define]
 CMAKE_BUILD_PARALLEL_LEVEL = "8"
-CMAKE_PREFIX_PATH = {env = "CMAKE_PREFIX_PATH"}
-OpenMP_ROOT = {env = "OpenMP_ROOT"}

packages/leann-core/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "leann-core"
-version = "0.3.2"
+version = "0.2.5"
 description = "Core API and plugin system for LEANN"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -31,10 +31,8 @@ dependencies = [
     "PyPDF2>=3.0.0",
     "pymupdf>=1.23.0",
     "pdfplumber>=0.10.0",
-    "nbconvert>=7.0.0",  # For .ipynb file support
-    "gitignore-parser>=0.1.12",  # For proper .gitignore handling
-    "mlx>=0.26.3; sys_platform == 'darwin' and platform_machine == 'arm64'",
-    "mlx-lm>=0.26.0; sys_platform == 'darwin' and platform_machine == 'arm64'",
+    "mlx>=0.26.3; sys_platform == 'darwin'",
+    "mlx-lm>=0.26.0; sys_platform == 'darwin'",
 ]
 
 [project.optional-dependencies]

packages/leann-core/src/leann/api.py

@@ -46,7 +46,6 @@ def compute_embeddings(
             - "sentence-transformers": Use sentence-transformers library (default)
             - "mlx": Use MLX backend for Apple Silicon
             - "openai": Use OpenAI embedding API
-            - "gemini": Use Google Gemini embedding API
         use_server: Whether to use embedding server (True for search, False for build)
 
     Returns:
@@ -88,21 +87,26 @@ def compute_embeddings_via_server(chunks: list[str], model_name: str, port: int)
     # Connect to embedding server
     context = zmq.Context()
     socket = context.socket(zmq.REQ)
+    socket.setsockopt(zmq.LINGER, 0)  # Don't block on close
+    socket.setsockopt(zmq.RCVTIMEO, 300000)
+    socket.setsockopt(zmq.SNDTIMEO, 300000)
+    socket.setsockopt(zmq.IMMEDIATE, 1)
     socket.connect(f"tcp://localhost:{port}")
 
-    # Send chunks to server for embedding computation
-    request = chunks
-    socket.send(msgpack.packb(request))
+    try:
+        # Send chunks to server for embedding computation
+        request = chunks
+        socket.send(msgpack.packb(request))
 
-    # Receive embeddings from server
-    response = socket.recv()
-    embeddings_list = msgpack.unpackb(response)
+        # Receive embeddings from server
+        response = socket.recv()
+        embeddings_list = msgpack.unpackb(response)
 
-    # Convert back to numpy array
-    embeddings = np.array(embeddings_list, dtype=np.float32)
-
-    socket.close()
-    context.term()
+        # Convert back to numpy array
+        embeddings = np.array(embeddings_list, dtype=np.float32)
+    finally:
+        socket.close()
+        # Don't call context.term() - this was causing hangs
 
     return embeddings
 
@@ -119,102 +123,57 @@ class PassageManager:
     def __init__(
         self, passage_sources: list[dict[str, Any]], metadata_file_path: Optional[str] = None
     ):
-        self.offset_maps: dict[str, dict[str, int]] = {}
-        self.passage_files: dict[str, str] = {}
-        # Avoid materializing a single gigantic global map to reduce memory
-        # footprint on very large corpora (e.g., 60M+ passages). Instead, keep
-        # per-shard maps and do a lightweight per-shard lookup on demand.
-        self._total_count: int = 0
-
-        # Derive index base name for standard sibling fallbacks, e.g., <index_name>.passages.*
-        index_name_base = None
-        if metadata_file_path:
-            meta_name = Path(metadata_file_path).name
-            if meta_name.endswith(".meta.json"):
-                index_name_base = meta_name[: -len(".meta.json")]
+        self.offset_maps = {}
+        self.passage_files = {}
+        self.global_offset_map = {}  # Combined map for fast lookup
 
         for source in passage_sources:
             assert source["type"] == "jsonl", "only jsonl is supported"
-            passage_file = source.get("path", "")
-            index_file = source.get("index_path", "")  # .idx file
+            passage_file = source["path"]
+            index_file = source["index_path"]  # .idx file
 
             # Fix path resolution - relative paths should be relative to metadata file directory
-            def _resolve_candidates(
-                primary: str,
-                relative_key: str,
-                default_name: Optional[str],
-                source_dict: dict[str, Any],
-            ) -> list[Path]:
-                """
-                Build an ordered list of candidate paths. For relative paths specified in
-                metadata, prefer resolution relative to the metadata file directory first,
-                then fall back to CWD-based resolution, and finally to conventional
-                sibling defaults (e.g., <index_base>.passages.idx / .jsonl).
-                """
-                candidates: list[Path] = []
-                # 1) Primary path
-                if primary:
-                    p = Path(primary)
-                    if p.is_absolute():
-                        candidates.append(p)
-                    else:
-                        # Prefer metadata-relative resolution for relative paths
-                        if metadata_file_path:
-                            candidates.append(Path(metadata_file_path).parent / p)
-                        # Also consider CWD-relative as a fallback for legacy layouts
-                        candidates.append(Path.cwd() / p)
-                # 2) metadata-relative explicit relative key (if present)
-                if metadata_file_path and source_dict.get(relative_key):
-                    candidates.append(Path(metadata_file_path).parent / source_dict[relative_key])
-                # 3) metadata-relative standard sibling filename
-                if metadata_file_path and default_name:
-                    candidates.append(Path(metadata_file_path).parent / default_name)
-                return candidates
-
-            # Build candidate lists and pick first existing; otherwise keep last candidate for error message
-            idx_default = f"{index_name_base}.passages.idx" if index_name_base else None
-            idx_candidates = _resolve_candidates(
-                index_file, "index_path_relative", idx_default, source
-            )
-            pas_default = f"{index_name_base}.passages.jsonl" if index_name_base else None
-            pas_candidates = _resolve_candidates(passage_file, "path_relative", pas_default, source)
-
-            def _pick_existing(cands: list[Path]) -> str:
-                for c in cands:
-                    if c.exists():
-                        return str(c.resolve())
-                # Fallback to last candidate (best guess) even if not exists; will error below
-                return str(cands[-1].resolve()) if cands else ""
-
-            index_file = _pick_existing(idx_candidates)
-            passage_file = _pick_existing(pas_candidates)
+            if not Path(index_file).is_absolute():
+                if metadata_file_path:
+                    # Resolve relative to metadata file directory
+                    metadata_dir = Path(metadata_file_path).parent
+                    logger.debug(
+                        f"PassageManager: Resolving relative paths from metadata_dir: {metadata_dir}"
+                    )
+                    index_file = str((metadata_dir / index_file).resolve())
+                    passage_file = str((metadata_dir / passage_file).resolve())
+                    logger.debug(f"PassageManager: Resolved index_file: {index_file}")
+                else:
+                    # Fallback to current directory resolution (legacy behavior)
+                    logger.warning(
+                        "PassageManager: No metadata_file_path provided, using fallback resolution from cwd"
+                    )
+                    logger.debug(f"PassageManager: Current working directory: {Path.cwd()}")
+                    index_file = str(Path(index_file).resolve())
+                    passage_file = str(Path(passage_file).resolve())
+                    logger.debug(f"PassageManager: Fallback resolved index_file: {index_file}")
 
             if not Path(index_file).exists():
                 raise FileNotFoundError(f"Passage index file not found: {index_file}")
 
             with open(index_file, "rb") as f:
-                offset_map: dict[str, int] = pickle.load(f)
+                offset_map = pickle.load(f)
                 self.offset_maps[passage_file] = offset_map
                 self.passage_files[passage_file] = passage_file
-                self._total_count += len(offset_map)
+
+                # Build global map for O(1) lookup
+                for passage_id, offset in offset_map.items():
+                    self.global_offset_map[passage_id] = (passage_file, offset)
 
     def get_passage(self, passage_id: str) -> dict[str, Any]:
-        # Fast path: check each shard map (there are typically few shards).
-        # This avoids building a massive combined dict while keeping lookups
-        # bounded by the number of shards.
-        for passage_file, offset_map in self.offset_maps.items():
-            try:
-                offset = offset_map[passage_id]
-                with open(passage_file, encoding="utf-8") as f:
-                    f.seek(offset)
-                    return json.loads(f.readline())
-            except KeyError:
-                continue
+        if passage_id in self.global_offset_map:
+            passage_file, offset = self.global_offset_map[passage_id]
+            # Lazy file opening - only open when needed
+            with open(passage_file, encoding="utf-8") as f:
+                f.seek(offset)
+                return json.loads(f.readline())
         raise KeyError(f"Passage ID not found: {passage_id}")
 
-    def __len__(self) -> int:
-        return self._total_count
-
 
 class LeannBuilder:
     def __init__(
@@ -226,18 +185,6 @@ class LeannBuilder:
         **backend_kwargs,
     ):
         self.backend_name = backend_name
-        # Normalize incompatible combinations early (for consistent metadata)
-        if backend_name == "hnsw":
-            is_recompute = backend_kwargs.get("is_recompute", True)
-            is_compact = backend_kwargs.get("is_compact", True)
-            if is_recompute is False and is_compact is True:
-                warnings.warn(
-                    "HNSW with is_recompute=False requires non-compact storage. Forcing is_compact=False.",
-                    UserWarning,
-                    stacklevel=2,
-                )
-                backend_kwargs["is_compact"] = False
-
         backend_factory: Optional[LeannBackendFactoryInterface] = BACKEND_REGISTRY.get(backend_name)
         if backend_factory is None:
             raise ValueError(f"Backend '{backend_name}' not found or not registered.")
@@ -328,23 +275,6 @@ class LeannBuilder:
     def build_index(self, index_path: str):
         if not self.chunks:
             raise ValueError("No chunks added.")
-
-        # Filter out invalid/empty text chunks early to keep passage and embedding counts aligned
-        valid_chunks: list[dict[str, Any]] = []
-        skipped = 0
-        for chunk in self.chunks:
-            text = chunk.get("text", "")
-            if isinstance(text, str) and text.strip():
-                valid_chunks.append(chunk)
-            else:
-                skipped += 1
-        if skipped > 0:
-            print(
-                f"Warning: Skipping {skipped} empty/invalid text chunk(s). Processing {len(valid_chunks)} valid chunks"
-            )
-            self.chunks = valid_chunks
-            if not self.chunks:
-                raise ValueError("All provided chunks are empty or invalid. Nothing to index.")
         if self.dimensions is None:
             self.dimensions = len(
                 compute_embeddings(
@@ -407,12 +337,8 @@ class LeannBuilder:
             "passage_sources": [
                 {
                     "type": "jsonl",
-                    # Preserve existing relative file names (backward-compatible)
-                    "path": passages_file.name,
-                    "index_path": offset_file.name,
-                    # Add optional redundant relative keys for remote build portability (non-breaking)
-                    "path_relative": passages_file.name,
-                    "index_path_relative": offset_file.name,
+                    "path": passages_file.name,  # Use relative path (just filename)
+                    "index_path": offset_file.name,  # Use relative path (just filename)
                 }
             ],
         }
@@ -527,12 +453,8 @@ class LeannBuilder:
             "passage_sources": [
                 {
                     "type": "jsonl",
-                    # Preserve existing relative file names (backward-compatible)
-                    "path": passages_file.name,
-                    "index_path": offset_file.name,
-                    # Add optional redundant relative keys for remote build portability (non-breaking)
-                    "path_relative": passages_file.name,
-                    "index_path_relative": offset_file.name,
+                    "path": passages_file.name,  # Use relative path (just filename)
+                    "index_path": offset_file.name,  # Use relative path (just filename)
                 }
             ],
             "built_from_precomputed_embeddings": True,
@@ -574,7 +496,6 @@ class LeannSearcher:
         self.embedding_model = self.meta_data["embedding_model"]
         # Support both old and new format
         self.embedding_mode = self.meta_data.get("embedding_mode", "sentence-transformers")
-        # Delegate portability handling to PassageManager
         self.passage_manager = PassageManager(
             self.meta_data.get("passage_sources", []), metadata_file_path=self.meta_path_str
         )
@@ -605,9 +526,7 @@ class LeannSearcher:
         logger.info(f"  Additional kwargs: {kwargs}")
 
         # Smart top_k detection and adjustment
-        # Use PassageManager length (sum of shard sizes) to avoid
-        # depending on a massive combined map
-        total_docs = len(self.passage_manager)
+        total_docs = len(self.passage_manager.global_offset_map)
         original_top_k = top_k
         if top_k > total_docs:
             top_k = total_docs
@@ -637,7 +556,7 @@ class LeannSearcher:
             zmq_port=zmq_port,
         )
         # logger.info(f"  Generated embedding shape: {query_embedding.shape}")
-        # time.time() - start_time
+        time.time() - start_time
         # logger.info(f"  Embedding time: {embedding_time} seconds")
 
         start_time = time.time()
@@ -658,7 +577,6 @@ class LeannSearcher:
         enriched_results = []
         if "labels" in results and "distances" in results:
             logger.info(f"  Processing {len(results['labels'][0])} passage IDs:")
-            # Python 3.9 does not support zip(strict=...); lengths are expected to match
             for i, (string_id, dist) in enumerate(
                 zip(results["labels"][0], results["distances"][0])
             ):
@@ -686,43 +604,17 @@ class LeannSearcher:
                     )
                 except KeyError:
                     RED = "\033[91m"
-                    RESET = "\033[0m"
                     logger.error(
                         f"   {RED}✗{RESET} [{i + 1:2d}] ID: '{string_id}' -> {RED}ERROR: Passage not found!{RESET}"
                     )
 
-        # Define color codes outside the loop for final message
-        GREEN = "\033[92m"
-        RESET = "\033[0m"
         logger.info(f"  {GREEN}✓ Final enriched results: {len(enriched_results)} passages{RESET}")
         return enriched_results
 
     def cleanup(self):
-        """Explicitly cleanup embedding server resources.
-
-        This method should be called after you're done using the searcher,
-        especially in test environments or batch processing scenarios.
-        """
-        backend = getattr(self.backend_impl, "embedding_server_manager", None)
-        if backend is not None:
-            backend.stop_server()
-
-    # Enable automatic cleanup patterns
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exc_type, exc, tb):
-        try:
-            self.cleanup()
-        except Exception:
-            pass
-
-    def __del__(self):
-        try:
-            self.cleanup()
-        except Exception:
-            # Avoid noisy errors during interpreter shutdown
-            pass
+        """Cleanup embedding server and other resources."""
+        if hasattr(self.backend_impl, "cleanup"):
+            self.backend_impl.cleanup()
 
 
 class LeannChat:
@@ -793,28 +685,3 @@ class LeannChat:
             except (KeyboardInterrupt, EOFError):
                 print("\nGoodbye!")
                 break
-
-    def cleanup(self):
-        """Explicitly cleanup embedding server resources.
-
-        This method should be called after you're done using the chat interface,
-        especially in test environments or batch processing scenarios.
-        """
-        if hasattr(self.searcher, "cleanup"):
-            self.searcher.cleanup()
-
-    # Enable automatic cleanup patterns
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exc_type, exc, tb):
-        try:
-            self.cleanup()
-        except Exception:
-            pass
-
-    def __del__(self):
-        try:
-            self.cleanup()
-        except Exception:
-            pass

packages/leann-core/src/leann/chat.py

@@ -422,6 +422,7 @@ class LLMInterface(ABC):
                 top_k=10,
                 complexity=64,
                 beam_width=8,
+                USE_DEFERRED_FETCH=True,
                 skip_search_reorder=True,
                 recompute_beighbor_embeddings=True,
                 dedup_node_dis=True,
@@ -433,6 +434,7 @@ class LLMInterface(ABC):
         Supported kwargs:
             - complexity (int): Search complexity parameter (default: 32)
             - beam_width (int): Beam width for search (default: 4)
+            - USE_DEFERRED_FETCH (bool): Enable deferred fetch mode (default: False)
             - skip_search_reorder (bool): Skip search reorder step (default: False)
             - recompute_beighbor_embeddings (bool): Enable ZMQ embedding server for neighbor recomputation (default: False)
             - dedup_node_dis (bool): Deduplicate nodes by distance (default: False)
@@ -680,60 +682,6 @@ class HFChat(LLMInterface):
         return response.strip()
 
 
-class GeminiChat(LLMInterface):
-    """LLM interface for Google Gemini models."""
-
-    def __init__(self, model: str = "gemini-2.5-flash", api_key: Optional[str] = None):
-        self.model = model
-        self.api_key = api_key or os.getenv("GEMINI_API_KEY")
-
-        if not self.api_key:
-            raise ValueError(
-                "Gemini API key is required. Set GEMINI_API_KEY environment variable or pass api_key parameter."
-            )
-
-        logger.info(f"Initializing Gemini Chat with model='{model}'")
-
-        try:
-            import google.genai as genai
-
-            self.client = genai.Client(api_key=self.api_key)
-        except ImportError:
-            raise ImportError(
-                "The 'google-genai' library is required for Gemini models. Please install it with 'uv pip install google-genai'."
-            )
-
-    def ask(self, prompt: str, **kwargs) -> str:
-        logger.info(f"Sending request to Gemini with model {self.model}")
-
-        try:
-            from google.genai.types import GenerateContentConfig
-
-            generation_config = GenerateContentConfig(
-                temperature=kwargs.get("temperature", 0.7),
-                max_output_tokens=kwargs.get("max_tokens", 1000),
-            )
-
-            # Handle top_p parameter
-            if "top_p" in kwargs:
-                generation_config.top_p = kwargs["top_p"]
-
-            response = self.client.models.generate_content(
-                model=self.model,
-                contents=prompt,
-                config=generation_config,
-            )
-            # Handle potential None response text
-            response_text = response.text
-            if response_text is None:
-                logger.warning("Gemini returned None response text")
-                return ""
-            return response_text.strip()
-        except Exception as e:
-            logger.error(f"Error communicating with Gemini: {e}")
-            return f"Error: Could not get a response from Gemini. Details: {e}"
-
-
 class OpenAIChat(LLMInterface):
     """LLM interface for OpenAI models."""
 
@@ -847,8 +795,6 @@ def get_llm(llm_config: Optional[dict[str, Any]] = None) -> LLMInterface:
         return HFChat(model_name=model or "deepseek-ai/deepseek-llm-7b-chat")
     elif llm_type == "openai":
         return OpenAIChat(model=model or "gpt-4o", api_key=llm_config.get("api_key"))
-    elif llm_type == "gemini":
-        return GeminiChat(model=model or "gemini-2.5-flash", api_key=llm_config.get("api_key"))
     elif llm_type == "simulated":
         return SimulatedChat()
     else:

packages/leann-core/src/leann/embedding_compute.py

@@ -6,6 +6,7 @@ Preserves all optimization parameters to ensure performance
 
 import logging
 import os
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from typing import Any
 
 import numpy as np
@@ -57,8 +58,6 @@ def compute_embeddings(
         return compute_embeddings_mlx(texts, model_name)
     elif mode == "ollama":
         return compute_embeddings_ollama(texts, model_name, is_build=is_build)
-    elif mode == "gemini":
-        return compute_embeddings_gemini(texts, model_name, is_build=is_build)
     else:
         raise ValueError(f"Unsupported embedding mode: {mode}")
 
@@ -246,16 +245,6 @@ def compute_embeddings_openai(texts: list[str], model_name: str) -> np.ndarray:
     except ImportError as e:
         raise ImportError(f"OpenAI package not installed: {e}")
 
-    # Validate input list
-    if not texts:
-        raise ValueError("Cannot compute embeddings for empty text list")
-    # Extra validation: abort early if any item is empty/whitespace
-    invalid_count = sum(1 for t in texts if not isinstance(t, str) or not t.strip())
-    if invalid_count > 0:
-        raise ValueError(
-            f"Found {invalid_count} empty/invalid text(s) in input. Upstream should filter before calling OpenAI."
-        )
-
     api_key = os.getenv("OPENAI_API_KEY")
     if not api_key:
         raise RuntimeError("OPENAI_API_KEY environment variable not set")
@@ -275,16 +264,8 @@ def compute_embeddings_openai(texts: list[str], model_name: str) -> np.ndarray:
     print(f"len of texts: {len(texts)}")
 
     # OpenAI has limits on batch size and input length
-    max_batch_size = 800  # Conservative batch size because the token limit is 300K
+    max_batch_size = 1000  # Conservative batch size
     all_embeddings = []
-    # get the avg len of texts
-    avg_len = sum(len(text) for text in texts) / len(texts)
-    print(f"avg len of texts: {avg_len}")
-    # if avg len is less than 1000, use the max batch size
-    if avg_len > 300:
-        max_batch_size = 500
-
-    # if avg len is less than 1000, use the max batch size
 
     try:
         from tqdm import tqdm
@@ -393,9 +374,7 @@ def compute_embeddings_ollama(
     texts: list[str], model_name: str, is_build: bool = False, host: str = "http://localhost:11434"
 ) -> np.ndarray:
     """
-    Compute embeddings using Ollama API with simplified batch processing.
-
-    Uses batch size of 32 for MPS/CPU and 128 for CUDA to optimize performance.
+    Compute embeddings using Ollama API.
 
     Args:
         texts: List of texts to compute embeddings for
@@ -459,19 +438,12 @@ def compute_embeddings_ollama(
             if any(emb in base_name for emb in ["embed", "bge", "minilm", "e5"]):
                 embedding_models.append(model)
 
-        # Check if model exists (handle versioned names) and resolve to full name
-        resolved_model_name = None
-        for name in model_names:
-            # Exact match
-            if model_name == name:
-                resolved_model_name = name
-                break
-            # Match without version tag (use the versioned name)
-            elif model_name == name.split(":")[0]:
-                resolved_model_name = name
-                break
+        # Check if model exists (handle versioned names)
+        model_found = any(
+            model_name == name.split(":")[0] or model_name == name for name in model_names
+        )
 
-        if not resolved_model_name:
+        if not model_found:
             error_msg = f"❌ Model '{model_name}' not found in local Ollama.\n\n"
 
             # Suggest pulling the model
@@ -493,11 +465,6 @@ def compute_embeddings_ollama(
             error_msg += "\n📚 Browse more: https://ollama.com/library"
             raise ValueError(error_msg)
 
-        # Use the resolved model name for all subsequent operations
-        if resolved_model_name != model_name:
-            logger.info(f"Resolved model name '{model_name}' to '{resolved_model_name}'")
-        model_name = resolved_model_name
-
         # Verify the model supports embeddings by testing it
         try:
             test_response = requests.post(
@@ -518,148 +485,138 @@ def compute_embeddings_ollama(
     except requests.exceptions.RequestException as e:
         logger.warning(f"Could not verify model existence: {e}")
 
-    # Determine batch size based on device availability
-    # Check for CUDA/MPS availability using torch if available
-    batch_size = 32  # Default for MPS/CPU
-    try:
-        import torch
+    # Process embeddings with optimized concurrent processing
+    import requests
 
-        if torch.cuda.is_available():
-            batch_size = 128  # CUDA gets larger batch size
-        elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
-            batch_size = 32  # MPS gets smaller batch size
-    except ImportError:
-        # If torch is not available, use conservative batch size
-        batch_size = 32
+    def get_single_embedding(text_idx_tuple):
+        """Helper function to get embedding for a single text."""
+        text, idx = text_idx_tuple
+        max_retries = 3
+        retry_count = 0
 
-    logger.info(f"Using batch size: {batch_size}")
+        # Truncate very long texts to avoid API issues
+        truncated_text = text[:8000] if len(text) > 8000 else text
 
-    def get_batch_embeddings(batch_texts):
-        """Get embeddings for a batch of texts."""
-        all_embeddings = []
-        failed_indices = []
+        while retry_count < max_retries:
+            try:
+                response = requests.post(
+                    f"{host}/api/embeddings",
+                    json={"model": model_name, "prompt": truncated_text},
+                    timeout=30,
+                )
+                response.raise_for_status()
 
-        for i, text in enumerate(batch_texts):
-            max_retries = 3
-            retry_count = 0
+                result = response.json()
+                embedding = result.get("embedding")
 
-            # Truncate very long texts to avoid API issues
-            truncated_text = text[:8000] if len(text) > 8000 else text
-            while retry_count < max_retries:
-                try:
-                    response = requests.post(
-                        f"{host}/api/embeddings",
-                        json={"model": model_name, "prompt": truncated_text},
-                        timeout=30,
+                if embedding is None:
+                    raise ValueError(f"No embedding returned for text {idx}")
+
+                return idx, embedding
+
+            except requests.exceptions.Timeout:
+                retry_count += 1
+                if retry_count >= max_retries:
+                    logger.warning(f"Timeout for text {idx} after {max_retries} retries")
+                    return idx, None
+
+            except Exception as e:
+                if retry_count >= max_retries - 1:
+                    logger.error(f"Failed to get embedding for text {idx}: {e}")
+                    return idx, None
+                retry_count += 1
+
+        return idx, None
+
+    # Determine if we should use concurrent processing
+    use_concurrent = (
+        len(texts) > 5 and not is_build
+    )  # Don't use concurrent in build mode to avoid overwhelming
+    max_workers = min(4, len(texts))  # Limit concurrent requests to avoid overwhelming Ollama
+
+    all_embeddings = [None] * len(texts)  # Pre-allocate list to maintain order
+    failed_indices = []
+
+    if use_concurrent:
+        logger.info(
+            f"Using concurrent processing with {max_workers} workers for {len(texts)} texts"
+        )
+
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            # Submit all tasks
+            future_to_idx = {
+                executor.submit(get_single_embedding, (text, idx)): idx
+                for idx, text in enumerate(texts)
+            }
+
+            # Add progress bar for concurrent processing
+            try:
+                if is_build or len(texts) > 10:
+                    from tqdm import tqdm
+
+                    futures_iterator = tqdm(
+                        as_completed(future_to_idx),
+                        total=len(texts),
+                        desc="Computing Ollama embeddings",
                     )
-                    response.raise_for_status()
-
-                    result = response.json()
-                    embedding = result.get("embedding")
-
-                    if embedding is None:
-                        raise ValueError(f"No embedding returned for text {i}")
-
-                    if not isinstance(embedding, list) or len(embedding) == 0:
-                        raise ValueError(f"Invalid embedding format for text {i}")
-
-                    all_embeddings.append(embedding)
-                    break
-
-                except requests.exceptions.Timeout:
-                    retry_count += 1
-                    if retry_count >= max_retries:
-                        logger.warning(f"Timeout for text {i} after {max_retries} retries")
-                        failed_indices.append(i)
-                        all_embeddings.append(None)
-                        break
+                else:
+                    futures_iterator = as_completed(future_to_idx)
+            except ImportError:
+                futures_iterator = as_completed(future_to_idx)
 
+            # Collect results as they complete
+            for future in futures_iterator:
+                try:
+                    idx, embedding = future.result()
+                    if embedding is not None:
+                        all_embeddings[idx] = embedding
+                    else:
+                        failed_indices.append(idx)
                 except Exception as e:
-                    retry_count += 1
-                    if retry_count >= max_retries:
-                        logger.error(f"Failed to get embedding for text {i}: {e}")
-                        failed_indices.append(i)
-                        all_embeddings.append(None)
-                        break
-        return all_embeddings, failed_indices
+                    idx = future_to_idx[future]
+                    logger.error(f"Exception for text {idx}: {e}")
+                    failed_indices.append(idx)
 
-    # Process texts in batches
-    all_embeddings = []
-    all_failed_indices = []
-
-    # Setup progress bar if needed
-    show_progress = is_build or len(texts) > 10
-    try:
-        if show_progress:
-            from tqdm import tqdm
-    except ImportError:
-        show_progress = False
-
-    # Process batches
-    num_batches = (len(texts) + batch_size - 1) // batch_size
-
-    if show_progress:
-        batch_iterator = tqdm(range(num_batches), desc="Computing Ollama embeddings")
     else:
-        batch_iterator = range(num_batches)
+        # Sequential processing with progress bar
+        show_progress = is_build or len(texts) > 10
 
-    for batch_idx in batch_iterator:
-        start_idx = batch_idx * batch_size
-        end_idx = min(start_idx + batch_size, len(texts))
-        batch_texts = texts[start_idx:end_idx]
+        try:
+            if show_progress:
+                from tqdm import tqdm
 
-        batch_embeddings, batch_failed = get_batch_embeddings(batch_texts)
+                iterator = tqdm(
+                    enumerate(texts), total=len(texts), desc="Computing Ollama embeddings"
+                )
+            else:
+                iterator = enumerate(texts)
+        except ImportError:
+            iterator = enumerate(texts)
 
-        # Adjust failed indices to global indices
-        global_failed = [start_idx + idx for idx in batch_failed]
-        all_failed_indices.extend(global_failed)
-        all_embeddings.extend(batch_embeddings)
+        for idx, text in iterator:
+            result_idx, embedding = get_single_embedding((text, idx))
+            if embedding is not None:
+                all_embeddings[idx] = embedding
+            else:
+                failed_indices.append(idx)
 
     # Handle failed embeddings
-    if all_failed_indices:
-        if len(all_failed_indices) == len(texts):
+    if failed_indices:
+        if len(failed_indices) == len(texts):
             raise RuntimeError("Failed to compute any embeddings")
 
-        logger.warning(
-            f"Failed to compute embeddings for {len(all_failed_indices)}/{len(texts)} texts"
-        )
+        logger.warning(f"Failed to compute embeddings for {len(failed_indices)}/{len(texts)} texts")
 
         # Use zero embeddings as fallback for failed ones
         valid_embedding = next((e for e in all_embeddings if e is not None), None)
         if valid_embedding:
             embedding_dim = len(valid_embedding)
-            for i, embedding in enumerate(all_embeddings):
-                if embedding is None:
-                    all_embeddings[i] = [0.0] * embedding_dim
+            for idx in failed_indices:
+                all_embeddings[idx] = [0.0] * embedding_dim
 
-    # Remove None values
+    # Remove None values and convert to numpy array
     all_embeddings = [e for e in all_embeddings if e is not None]
 
-    if not all_embeddings:
-        raise RuntimeError("No valid embeddings were computed")
-
-    # Validate embedding dimensions
-    expected_dim = len(all_embeddings[0])
-    inconsistent_dims = []
-    for i, embedding in enumerate(all_embeddings):
-        if len(embedding) != expected_dim:
-            inconsistent_dims.append((i, len(embedding)))
-
-    if inconsistent_dims:
-        error_msg = f"Ollama returned inconsistent embedding dimensions. Expected {expected_dim}, but got:\n"
-        for idx, dim in inconsistent_dims[:10]:  # Show first 10 inconsistent ones
-            error_msg += f"  - Text {idx}: {dim} dimensions\n"
-        if len(inconsistent_dims) > 10:
-            error_msg += f"  ... and {len(inconsistent_dims) - 10} more\n"
-        error_msg += f"\nThis is likely an Ollama API bug with model '{model_name}'. Please try:\n"
-        error_msg += "1. Restart Ollama service: 'ollama serve'\n"
-        error_msg += f"2. Re-pull the model: 'ollama pull {model_name}'\n"
-        error_msg += (
-            "3. Use sentence-transformers instead: --embedding-mode sentence-transformers\n"
-        )
-        error_msg += "4. Report this issue to Ollama: https://github.com/ollama/ollama/issues"
-        raise ValueError(error_msg)
-
     # Convert to numpy array and normalize
     embeddings = np.array(all_embeddings, dtype=np.float32)
 
@@ -670,83 +627,3 @@ def compute_embeddings_ollama(
     logger.info(f"Generated {len(embeddings)} embeddings, dimension: {embeddings.shape[1]}")
 
     return embeddings
-
-
-def compute_embeddings_gemini(
-    texts: list[str], model_name: str = "text-embedding-004", is_build: bool = False
-) -> np.ndarray:
-    """
-    Compute embeddings using Google Gemini API.
-
-    Args:
-        texts: List of texts to compute embeddings for
-        model_name: Gemini model name (default: "text-embedding-004")
-        is_build: Whether this is a build operation (shows progress bar)
-
-    Returns:
-        Embeddings array, shape: (len(texts), embedding_dim)
-    """
-    try:
-        import os
-
-        import google.genai as genai
-    except ImportError as e:
-        raise ImportError(f"Google GenAI package not installed: {e}")
-
-    api_key = os.getenv("GEMINI_API_KEY")
-    if not api_key:
-        raise RuntimeError("GEMINI_API_KEY environment variable not set")
-
-    # Cache Gemini client
-    cache_key = "gemini_client"
-    if cache_key in _model_cache:
-        client = _model_cache[cache_key]
-    else:
-        client = genai.Client(api_key=api_key)
-        _model_cache[cache_key] = client
-        logger.info("Gemini client cached")
-
-    logger.info(
-        f"Computing embeddings for {len(texts)} texts using Gemini API, model: '{model_name}'"
-    )
-
-    # Gemini supports batch embedding
-    max_batch_size = 100  # Conservative batch size for Gemini
-    all_embeddings = []
-
-    try:
-        from tqdm import tqdm
-
-        total_batches = (len(texts) + max_batch_size - 1) // max_batch_size
-        batch_range = range(0, len(texts), max_batch_size)
-        batch_iterator = tqdm(
-            batch_range, desc="Computing embeddings", unit="batch", total=total_batches
-        )
-    except ImportError:
-        # Fallback when tqdm is not available
-        batch_iterator = range(0, len(texts), max_batch_size)
-
-    for i in batch_iterator:
-        batch_texts = texts[i : i + max_batch_size]
-
-        try:
-            # Use the embed_content method from the new Google GenAI SDK
-            response = client.models.embed_content(
-                model=model_name,
-                contents=batch_texts,
-                config=genai.types.EmbedContentConfig(
-                    task_type="RETRIEVAL_DOCUMENT"  # For document embedding
-                ),
-            )
-
-            # Extract embeddings from response
-            for embedding_data in response.embeddings:
-                all_embeddings.append(embedding_data.values)
-        except Exception as e:
-            logger.error(f"Batch {i} failed: {e}")
-            raise
-
-    embeddings = np.array(all_embeddings, dtype=np.float32)
-    logger.info(f"Generated {len(embeddings)} embeddings, dimension: {embeddings.shape[1]}")
-
-    return embeddings

packages/leann-core/src/leann/embedding_server_manager.py

@@ -1,6 +1,7 @@
 import atexit
 import logging
 import os
+import signal
 import socket
 import subprocess
 import sys
@@ -8,7 +9,7 @@ import time
 from pathlib import Path
 from typing import Optional
 
-# Lightweight, self-contained server manager with no cross-process inspection
+import psutil
 
 # Set up logging based on environment variable
 LOG_LEVEL = os.getenv("LEANN_LOG_LEVEL", "WARNING").upper()
@@ -43,7 +44,130 @@ def _check_port(port: int) -> bool:
         return s.connect_ex(("localhost", port)) == 0
 
 
-# Note: All cross-process scanning helpers removed for simplicity
+def _check_process_matches_config(
+    port: int, expected_model: str, expected_passages_file: str
+) -> bool:
+    """
+    Check if the process using the port matches our expected model and passages file.
+    Returns True if matches, False otherwise.
+    """
+    try:
+        for proc in psutil.process_iter(["pid", "cmdline"]):
+            if not _is_process_listening_on_port(proc, port):
+                continue
+
+            cmdline = proc.info["cmdline"]
+            if not cmdline:
+                continue
+
+            return _check_cmdline_matches_config(
+                cmdline, port, expected_model, expected_passages_file
+            )
+
+        logger.debug(f"No process found listening on port {port}")
+        return False
+
+    except Exception as e:
+        logger.warning(f"Could not check process on port {port}: {e}")
+        return False
+
+
+def _is_process_listening_on_port(proc, port: int) -> bool:
+    """Check if a process is listening on the given port."""
+    try:
+        connections = proc.net_connections()
+        for conn in connections:
+            if conn.laddr.port == port and conn.status == psutil.CONN_LISTEN:
+                return True
+        return False
+    except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
+        return False
+
+
+def _check_cmdline_matches_config(
+    cmdline: list, port: int, expected_model: str, expected_passages_file: str
+) -> bool:
+    """Check if command line matches our expected configuration."""
+    cmdline_str = " ".join(cmdline)
+    logger.debug(f"Found process on port {port}: {cmdline_str}")
+
+    # Check if it's our embedding server
+    is_embedding_server = any(
+        server_type in cmdline_str
+        for server_type in [
+            "embedding_server",
+            "leann_backend_diskann.embedding_server",
+            "leann_backend_hnsw.hnsw_embedding_server",
+        ]
+    )
+
+    if not is_embedding_server:
+        logger.debug(f"Process on port {port} is not our embedding server")
+        return False
+
+    # Check model name
+    model_matches = _check_model_in_cmdline(cmdline, expected_model)
+
+    # Check passages file if provided
+    passages_matches = _check_passages_in_cmdline(cmdline, expected_passages_file)
+
+    result = model_matches and passages_matches
+    logger.debug(
+        f"model_matches: {model_matches}, passages_matches: {passages_matches}, overall: {result}"
+    )
+    return result
+
+
+def _check_model_in_cmdline(cmdline: list, expected_model: str) -> bool:
+    """Check if the command line contains the expected model."""
+    if "--model-name" not in cmdline:
+        return False
+
+    model_idx = cmdline.index("--model-name")
+    if model_idx + 1 >= len(cmdline):
+        return False
+
+    actual_model = cmdline[model_idx + 1]
+    return actual_model == expected_model
+
+
+def _check_passages_in_cmdline(cmdline: list, expected_passages_file: str) -> bool:
+    """Check if the command line contains the expected passages file."""
+    if "--passages-file" not in cmdline:
+        return False  # Expected but not found
+
+    passages_idx = cmdline.index("--passages-file")
+    if passages_idx + 1 >= len(cmdline):
+        return False
+
+    actual_passages = cmdline[passages_idx + 1]
+    expected_path = Path(expected_passages_file).resolve()
+    actual_path = Path(actual_passages).resolve()
+    return actual_path == expected_path
+
+
+def _find_compatible_port_or_next_available(
+    start_port: int, model_name: str, passages_file: str, max_attempts: int = 100
+) -> tuple[int, bool]:
+    """
+    Find a port that either has a compatible server or is available.
+    Returns (port, is_compatible) where is_compatible indicates if we found a matching server.
+    """
+    for port in range(start_port, start_port + max_attempts):
+        if not _check_port(port):
+            # Port is available
+            return port, False
+
+        # Port is in use, check if it's compatible
+        if _check_process_matches_config(port, model_name, passages_file):
+            logger.info(f"Found compatible server on port {port}")
+            return port, True
+        else:
+            logger.info(f"Port {port} has incompatible server, trying next port...")
+
+    raise RuntimeError(
+        f"Could not find compatible or available port in range {start_port}-{start_port + max_attempts}"
+    )
 
 
 class EmbeddingServerManager:
@@ -62,16 +186,7 @@ class EmbeddingServerManager:
         self.backend_module_name = backend_module_name
         self.server_process: Optional[subprocess.Popen] = None
         self.server_port: Optional[int] = None
-        # Track last-started config for in-process reuse only
-        self._server_config: Optional[dict] = None
         self._atexit_registered = False
-        # Also register a weakref finalizer to ensure cleanup when manager is GC'ed
-        try:
-            import weakref
-
-            self._finalizer = weakref.finalize(self, self._finalize_process)
-        except Exception:
-            self._finalizer = None
 
     def start_server(
         self,
@@ -81,24 +196,26 @@ class EmbeddingServerManager:
         **kwargs,
     ) -> tuple[bool, int]:
         """Start the embedding server."""
-        # passages_file may be present in kwargs for server CLI, but we don't need it here
+        passages_file = kwargs.get("passages_file")
 
-        # If this manager already has a live server, just reuse it
-        if self.server_process and self.server_process.poll() is None and self.server_port:
-            logger.info("Reusing in-process server")
-            return True, self.server_port
+        # Check if we have a compatible server already running
+        if self._has_compatible_running_server(model_name, passages_file):
+            logger.info("Found compatible running server!")
+            return True, port
 
         # For Colab environment, use a different strategy
         if _is_colab_environment():
             logger.info("Detected Colab environment, using alternative startup strategy")
             return self._start_server_colab(port, model_name, embedding_mode, **kwargs)
 
-        # Always pick a fresh available port
-        try:
-            actual_port = _get_available_port(port)
-        except RuntimeError:
-            logger.error("No available ports found")
-            return False, port
+        # Find a compatible port or next available
+        actual_port, is_compatible = _find_compatible_port_or_next_available(
+            port, model_name, passages_file
+        )
+
+        if is_compatible:
+            logger.info(f"Found compatible server on port {actual_port}")
+            return True, actual_port
 
         # Start a new server
         return self._start_new_server(actual_port, model_name, embedding_mode, **kwargs)
@@ -131,7 +248,17 @@ class EmbeddingServerManager:
             logger.error(f"Failed to start embedding server in Colab: {e}")
             return False, actual_port
 
-    # Note: No compatibility check needed; manager is per-searcher and configs are stable per instance
+    def _has_compatible_running_server(self, model_name: str, passages_file: str) -> bool:
+        """Check if we have a compatible running server."""
+        if not (self.server_process and self.server_process.poll() is None and self.server_port):
+            return False
+
+        if _check_process_matches_config(self.server_port, model_name, passages_file):
+            logger.info(f"Existing server process (PID {self.server_process.pid}) is compatible")
+            return True
+
+        logger.info("Existing server process is incompatible. Should start a new server.")
+        return False
 
     def _start_new_server(
         self, port: int, model_name: str, embedding_mode: str, **kwargs
@@ -178,62 +305,23 @@ class EmbeddingServerManager:
         project_root = Path(__file__).parent.parent.parent.parent.parent
         logger.info(f"Command: {' '.join(command)}")
 
-        # In CI environment, redirect stdout to avoid buffer deadlock but keep stderr for debugging
-        # Embedding servers use many print statements that can fill stdout buffers
-        is_ci = os.environ.get("CI") == "true"
-        if is_ci:
-            stdout_target = subprocess.DEVNULL
-            stderr_target = None  # Keep stderr for error debugging in CI
-            logger.info(
-                "CI environment detected, redirecting embedding server stdout to DEVNULL, keeping stderr"
-            )
-        else:
-            stdout_target = None  # Direct to console for visible logs
-            stderr_target = None  # Direct to console for visible logs
-
-        # Start embedding server subprocess
-        logger.info(f"Starting server process with command: {' '.join(command)}")
+        # Let server output go directly to console
+        # The server will respect LEANN_LOG_LEVEL environment variable
         self.server_process = subprocess.Popen(
             command,
             cwd=project_root,
-            stdout=stdout_target,
-            stderr=stderr_target,
+            stdout=None,  # Direct to console
+            stderr=None,  # Direct to console
+            start_new_session=True,  # Create new process group for better cleanup
         )
         self.server_port = port
-        # Record config for in-process reuse
-        try:
-            self._server_config = {
-                "model_name": command[command.index("--model-name") + 1]
-                if "--model-name" in command
-                else "",
-                "passages_file": command[command.index("--passages-file") + 1]
-                if "--passages-file" in command
-                else "",
-                "embedding_mode": command[command.index("--embedding-mode") + 1]
-                if "--embedding-mode" in command
-                else "sentence-transformers",
-            }
-        except Exception:
-            self._server_config = {
-                "model_name": "",
-                "passages_file": "",
-                "embedding_mode": "sentence-transformers",
-            }
         logger.info(f"Server process started with PID: {self.server_process.pid}")
 
         # Register atexit callback only when we actually start a process
         if not self._atexit_registered:
-            # Always attempt best-effort finalize at interpreter exit
-            atexit.register(self._finalize_process)
+            # Use a lambda to avoid issues with bound methods
+            atexit.register(lambda: self.stop_server() if self.server_process else None)
             self._atexit_registered = True
-        # Touch finalizer so it knows there is a live process
-        if getattr(self, "_finalizer", None) is not None and not self._finalizer.alive:
-            try:
-                import weakref
-
-                self._finalizer = weakref.finalize(self, self._finalize_process)
-            except Exception:
-                pass
 
     def _wait_for_server_ready(self, port: int) -> tuple[bool, int]:
         """Wait for the server to be ready."""
@@ -258,35 +346,37 @@ class EmbeddingServerManager:
         if not self.server_process:
             return
 
-        if self.server_process and self.server_process.poll() is not None:
+        if self.server_process.poll() is not None:
             # Process already terminated
             self.server_process = None
-            self.server_port = None
-            self._server_config = None
             return
 
         logger.info(
             f"Terminating server process (PID: {self.server_process.pid}) for backend {self.backend_module_name}..."
         )
 
-        # Use simple termination first; if the server installed signal handlers,
-        # it will exit cleanly. Otherwise escalate to kill after a short wait.
+        # Try terminating the whole process group first
         try:
-            self.server_process.terminate()
+            pgid = os.getpgid(self.server_process.pid)
+            os.killpg(pgid, signal.SIGTERM)
         except Exception:
-            pass
+            # Fallback to terminating just the process
+            self.server_process.terminate()
 
         try:
-            self.server_process.wait(timeout=5)  # Give more time for graceful shutdown
-            logger.info(f"Server process {self.server_process.pid} terminated gracefully.")
+            self.server_process.wait(timeout=3)
+            logger.info(f"Server process {self.server_process.pid} terminated.")
         except subprocess.TimeoutExpired:
             logger.warning(
-                f"Server process {self.server_process.pid} did not terminate within 5 seconds, force killing..."
+                f"Server process {self.server_process.pid} did not terminate gracefully within 3 seconds, killing it."
             )
+            # Try killing the whole process group
             try:
-                self.server_process.kill()
+                pgid = os.getpgid(self.server_process.pid)
+                os.killpg(pgid, signal.SIGKILL)
             except Exception:
-                pass
+                # Fallback to killing just the process
+                self.server_process.kill()
             try:
                 self.server_process.wait(timeout=2)
                 logger.info(f"Server process {self.server_process.pid} killed successfully.")
@@ -294,33 +384,20 @@ class EmbeddingServerManager:
                 logger.error(
                     f"Failed to kill server process {self.server_process.pid} - it may be hung"
                 )
+                # Don't hang indefinitely
 
-        # Clean up process resources with timeout to avoid CI hang
+        # Clean up process resources to prevent resource tracker warnings
         try:
-            # Use shorter timeout in CI environments
-            is_ci = os.environ.get("CI") == "true"
-            timeout = 3 if is_ci else 10
-            self.server_process.wait(timeout=timeout)
-            logger.info(f"Server process {self.server_process.pid} cleanup completed")
+            self.server_process.wait(timeout=1)  # Give it one final chance with timeout
         except subprocess.TimeoutExpired:
-            logger.warning(f"Process cleanup timeout after {timeout}s, proceeding anyway")
-        except Exception as e:
-            logger.warning(f"Error during process cleanup: {e}")
-        finally:
-            self.server_process = None
-            self.server_port = None
-            self._server_config = None
-
-    def _finalize_process(self) -> None:
-        """Best-effort cleanup used by weakref.finalize/atexit."""
-        try:
-            self.stop_server()
+            logger.warning(
+                f"Process {self.server_process.pid} still hanging after all kill attempts"
+            )
+            # Don't wait indefinitely - just abandon it
         except Exception:
             pass
 
-    def _adopt_existing_server(self, *args, **kwargs) -> None:
-        # Removed: cross-process adoption no longer supported
-        return
+        self.server_process = None
 
     def _launch_server_process_colab(self, command: list, port: int) -> None:
         """Launch the server process with Colab-specific settings."""
@@ -336,16 +413,10 @@ class EmbeddingServerManager:
         self.server_port = port
         logger.info(f"Colab server process started with PID: {self.server_process.pid}")
 
-        # Register atexit callback (unified)
+        # Register atexit callback
         if not self._atexit_registered:
-            atexit.register(self._finalize_process)
+            atexit.register(lambda: self.stop_server() if self.server_process else None)
             self._atexit_registered = True
-        # Record config for in-process reuse is best-effort in Colab mode
-        self._server_config = {
-            "model_name": "",
-            "passages_file": "",
-            "embedding_mode": "sentence-transformers",
-        }
 
     def _wait_for_server_ready_colab(self, port: int) -> tuple[bool, int]:
         """Wait for the server to be ready with Colab-specific timeout."""

packages/leann-core/src/leann/mcp.py

@@ -25,48 +25,32 @@ def handle_request(request):
                 "tools": [
                     {
                         "name": "leann_search",
-                        "description": """🔍 Search code using natural language - like having a coding assistant who knows your entire codebase!
-
-🎯 **Perfect for**:
-- "How does authentication work?" → finds auth-related code
-- "Error handling patterns" → locates try-catch blocks and error logic
-- "Database connection setup" → finds DB initialization code
-- "API endpoint definitions" → locates route handlers
-- "Configuration management" → finds config files and usage
-
-💡 **Pro tip**: Use this before making any changes to understand existing patterns and conventions.""",
+                        "description": "Search LEANN index",
                         "inputSchema": {
                             "type": "object",
                             "properties": {
-                                "index_name": {
-                                    "type": "string",
-                                    "description": "Name of the LEANN index to search. Use 'leann_list' first to see available indexes.",
-                                },
-                                "query": {
-                                    "type": "string",
-                                    "description": "Search query - can be natural language (e.g., 'how to handle errors') or technical terms (e.g., 'async function definition')",
-                                },
-                                "top_k": {
-                                    "type": "integer",
-                                    "default": 5,
-                                    "minimum": 1,
-                                    "maximum": 20,
-                                    "description": "Number of search results to return. Use 5-10 for focused results, 15-20 for comprehensive exploration.",
-                                },
-                                "complexity": {
-                                    "type": "integer",
-                                    "default": 32,
-                                    "minimum": 16,
-                                    "maximum": 128,
-                                    "description": "Search complexity level. Use 16-32 for fast searches (recommended), 64+ for higher precision when needed.",
-                                },
+                                "index_name": {"type": "string"},
+                                "query": {"type": "string"},
+                                "top_k": {"type": "integer", "default": 5},
                             },
                             "required": ["index_name", "query"],
                         },
                     },
+                    {
+                        "name": "leann_ask",
+                        "description": "Ask question using LEANN RAG",
+                        "inputSchema": {
+                            "type": "object",
+                            "properties": {
+                                "index_name": {"type": "string"},
+                                "question": {"type": "string"},
+                            },
+                            "required": ["index_name", "question"],
+                        },
+                    },
                     {
                         "name": "leann_list",
-                        "description": "📋 Show all your indexed codebases - your personal code library! Use this to see what's available for search.",
+                        "description": "List all LEANN indexes",
                         "inputSchema": {"type": "object", "properties": {}},
                     },
                 ]
@@ -79,33 +63,20 @@ def handle_request(request):
 
         try:
             if tool_name == "leann_search":
-                # Validate required parameters
-                if not args.get("index_name") or not args.get("query"):
-                    return {
-                        "jsonrpc": "2.0",
-                        "id": request.get("id"),
-                        "result": {
-                            "content": [
-                                {
-                                    "type": "text",
-                                    "text": "Error: Both index_name and query are required",
-                                }
-                            ]
-                        },
-                    }
-
-                # Build simplified command with non-interactive flag for MCP compatibility
                 cmd = [
                     "leann",
                     "search",
                     args["index_name"],
                     args["query"],
+                    "--recompute-embeddings",
                     f"--top-k={args.get('top_k', 5)}",
-                    f"--complexity={args.get('complexity', 32)}",
-                    "--non-interactive",
                 ]
                 result = subprocess.run(cmd, capture_output=True, text=True)
 
+            elif tool_name == "leann_ask":
+                cmd = f'echo "{args["question"]}" | leann ask {args["index_name"]} --recompute-embeddings --llm ollama --model qwen3:8b'
+                result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
+
             elif tool_name == "leann_list":
                 result = subprocess.run(["leann", "list"], capture_output=True, text=True)
 

packages/leann-core/src/leann/registry.py

@@ -2,17 +2,11 @@
 
 import importlib
 import importlib.metadata
-import json
-import logging
-from pathlib import Path
-from typing import TYPE_CHECKING, Optional, Union
+from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from leann.interface import LeannBackendFactoryInterface
 
-# Set up logger for this module
-logger = logging.getLogger(__name__)
-
 BACKEND_REGISTRY: dict[str, "LeannBackendFactoryInterface"] = {}
 
 
@@ -20,7 +14,7 @@ def register_backend(name: str):
     """A decorator to register a new backend class."""
 
     def decorator(cls):
-        logger.debug(f"Registering backend '{name}'")
+        print(f"INFO: Registering backend '{name}'")
         BACKEND_REGISTRY[name] = cls
         return cls
 
@@ -45,54 +39,3 @@ def autodiscover_backends():
             # print(f"WARN: Could not import backend module '{backend_module_name}': {e}")
             pass
     # print("INFO: Backend auto-discovery finished.")
-
-
-def register_project_directory(project_dir: Optional[Union[str, Path]] = None):
-    """
-    Register a project directory in the global LEANN registry.
-
-    This allows `leann list` to discover indexes created by apps or other tools.
-
-    Args:
-        project_dir: Directory to register. If None, uses current working directory.
-    """
-    if project_dir is None:
-        project_dir = Path.cwd()
-    else:
-        project_dir = Path(project_dir)
-
-    # Only register directories that have some kind of LEANN content
-    # Either .leann/indexes/ (CLI format) or *.leann.meta.json files (apps format)
-    has_cli_indexes = (project_dir / ".leann" / "indexes").exists()
-    has_app_indexes = any(project_dir.rglob("*.leann.meta.json"))
-
-    if not (has_cli_indexes or has_app_indexes):
-        # Don't register if there are no LEANN indexes
-        return
-
-    global_registry = Path.home() / ".leann" / "projects.json"
-    global_registry.parent.mkdir(exist_ok=True)
-
-    project_str = str(project_dir.resolve())
-
-    # Load existing registry
-    projects = []
-    if global_registry.exists():
-        try:
-            with open(global_registry) as f:
-                projects = json.load(f)
-        except Exception:
-            logger.debug("Could not load existing project registry")
-            projects = []
-
-    # Add project if not already present
-    if project_str not in projects:
-        projects.append(project_str)
-
-        # Save updated registry
-        try:
-            with open(global_registry, "w") as f:
-                json.dump(projects, f, indent=2)
-            logger.debug(f"Registered project directory: {project_str}")
-        except Exception as e:
-            logger.warning(f"Could not save project registry: {e}")

packages/leann-core/src/leann/searcher_base.py

@@ -132,10 +132,15 @@ class BaseSearcher(LeannBackendSearcherInterface, ABC):
         import msgpack
         import zmq
 
+        context = None
+        socket = None
         try:
             context = zmq.Context()
             socket = context.socket(zmq.REQ)
-            socket.setsockopt(zmq.RCVTIMEO, 30000)  # 30 second timeout
+            socket.setsockopt(zmq.LINGER, 0)  # Don't block on close
+            socket.setsockopt(zmq.RCVTIMEO, 300000)
+            socket.setsockopt(zmq.SNDTIMEO, 300000)
+            socket.setsockopt(zmq.IMMEDIATE, 1)
             socket.connect(f"tcp://localhost:{zmq_port}")
 
             # Send embedding request
@@ -147,9 +152,6 @@ class BaseSearcher(LeannBackendSearcherInterface, ABC):
             response_bytes = socket.recv()
             response = msgpack.unpackb(response_bytes)
 
-            socket.close()
-            context.term()
-
             # Convert response to numpy array
             if isinstance(response, list) and len(response) > 0:
                 return np.array(response, dtype=np.float32)
@@ -158,6 +160,10 @@ class BaseSearcher(LeannBackendSearcherInterface, ABC):
 
         except Exception as e:
             raise RuntimeError(f"Failed to compute embeddings via server: {e}")
+        finally:
+            if socket:
+                socket.close()
+            # Don't call context.term() - this was causing hangs
 
     @abstractmethod
     def search(
@@ -191,7 +197,15 @@ class BaseSearcher(LeannBackendSearcherInterface, ABC):
         """
         pass
 
-    def __del__(self):
-        """Ensures the embedding server is stopped when the searcher is destroyed."""
+    def cleanup(self):
+        """Cleanup resources including embedding server."""
         if hasattr(self, "embedding_server_manager"):
             self.embedding_server_manager.stop_server()
+
+    def __del__(self):
+        """Ensures resources are cleaned up when the searcher is destroyed."""
+        try:
+            self.cleanup()
+        except Exception:
+            # Ignore errors during destruction
+            pass

packages/leann-mcp/README.md

@@ -4,29 +4,27 @@ Transform your development workflow with intelligent code assistance using LEANN
 
 ## Prerequisites
 
-Install LEANN globally for MCP integration (with default backend):
+**Step 1:** First, complete the basic LEANN installation following the [📦 Installation guide](../../README.md#installation) in the root README:
 
 ```bash
-uv tool install leann-core --with leann
+uv venv
+source .venv/bin/activate
+uv pip install leann
 ```
-This installs the `leann` CLI into an isolated tool environment and includes both backends so `leann build` works out-of-the-box.
+
+**Step 2:** Install LEANN globally for MCP integration:
+```bash
+uv tool install leann-core
+```
+
+This makes the `leann` command available system-wide, which `leann_mcp` requires.
 
 ## 🚀 Quick Setup
 
-Add the LEANN MCP server to Claude Code. Choose the scope based on how widely you want it available. Below is the command to install it globally; if you prefer a local install, skip this step:
+Add the LEANN MCP server to Claude Code:
 
 ```bash
-# Global (recommended): available in all projects for your user
-claude mcp add --scope user leann-server -- leann_mcp
-```
-
-- `leann-server`: the display name of the MCP server in Claude Code (you can change it).
-- `leann_mcp`: the Python entry point installed with LEANN that starts the MCP server.
-
-Verify it is registered globally:
-
-```bash
-claude mcp list | cat
+claude mcp add leann-server -- leann_mcp
 ```
 
 ## 🛠️ Available Tools
@@ -35,64 +33,19 @@ Once connected, you'll have access to these powerful semantic search tools in Cl
 
 - **`leann_list`** - List all available indexes across your projects
 - **`leann_search`** - Perform semantic searches across code and documents
-
+- **`leann_ask`** - Ask natural language questions and get AI-powered answers from your codebase
 
 ## 🎯 Quick Start Example
 
 ```bash
-# Add locally if you did not add it globally (current folder only; default if --scope is omitted)
-claude mcp add leann-server -- leann_mcp
-
 # Build an index for your project (change to your actual path)
-# See the advanced examples below for more ways to configure indexing
-# Set the index name (replace 'my-project' with your own)
-leann build my-project --docs $(git ls-files)
+leann build my-project --docs ./
 
 # Start Claude Code
 claude
 ```
 
-## 🚀 Advanced Usage Examples to build the index
-
-### Index Entire Git Repository
-```bash
-# Index all tracked files in your Git repository.
-# Note: submodules are currently skipped; we can add them back if needed.
-leann build my-repo --docs $(git ls-files) --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw
-
-# Index only tracked Python files from Git.
-leann build my-python-code --docs $(git ls-files "*.py") --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw
-
-# If you encounter empty requests caused by empty files (e.g., __init__.py), exclude zero-byte files. Thanks @ww2283 for pointing [that](https://github.com/yichuan-w/LEANN/issues/48) out
-leann build leann-prospec-lig --docs $(find ./src -name "*.py" -not -empty) --embedding-mode openai --embedding-model text-embedding-3-small
-```
-
-### Multiple Directories and Files
-```bash
-# Index multiple directories
-leann build my-codebase --docs ./src ./tests ./docs ./config --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw
-
-# Mix files and directories
-leann build my-project --docs ./README.md ./src/ ./package.json ./docs/ --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw
-
-# Specific files only
-leann build my-configs --docs ./tsconfig.json ./package.json ./webpack.config.js --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw
-```
-
-### Advanced Git Integration
-```bash
-# Index recently modified files
-leann build recent-changes --docs $(git diff --name-only HEAD~10..HEAD) --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw
-
-# Index files matching pattern
-leann build frontend --docs $(git ls-files "*.tsx" "*.ts" "*.jsx" "*.js") --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw
-
-# Index documentation and config files
-leann build docs-and-configs --docs $(git ls-files "*.md" "*.yml" "*.yaml" "*.json" "*.toml") --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw
-```
-
-
-## **Try this in Claude Code:**
+**Try this in Claude Code:**
 ```
 Help me understand this codebase. List available indexes and search for authentication patterns.
 ```
@@ -101,7 +54,6 @@ Help me understand this codebase. List available indexes and search for authenti
   <img src="../../assets/claude_code_leann.png" alt="LEANN in Claude Code" width="80%">
 </p>
 
-If you see a prompt asking whether to proceed with LEANN, you can now use it in your chat!
 
 ## 🧠 How It Works
 
@@ -137,11 +89,3 @@ To remove LEANN
 ```
 uv pip uninstall leann leann-backend-hnsw leann-core
 ```
-
-To globally remove LEANN (for version update)
-```
-uv tool list | cat
-uv tool uninstall leann-core
-command -v leann || echo "leann gone"
-command -v leann_mcp || echo "leann_mcp gone"
-```

packages/leann/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "leann"
-version = "0.3.2"
+version = "0.2.5"
 description = "LEANN - The smallest vector index in the world. RAG Everything with LEANN!"
 readme = "README.md"
 requires-python = ">=3.9"

packages/wechat-exporter/__init__.py

@@ -1 +0,0 @@
-__all__ = []

packages/wechat-exporter/main.py

@@ -136,9 +136,5 @@ def export_sqlite(
     connection.commit()
 
 
-def main():
-    app()
-
-
 if __name__ == "__main__":
-    main()
+    app()

pyproject.toml

@@ -10,10 +10,11 @@ requires-python = ">=3.9"
 dependencies = [
     "leann-core",
     "leann-backend-hnsw",
-    "typer>=0.12.3",
     "numpy>=1.26.0",
     "torch",
     "tqdm",
+    "flask",
+    "flask_compress",
     "datasets>=2.15.0",
     "evaluate",
     "colorama",
@@ -39,20 +40,10 @@ dependencies = [
     # Other dependencies
     "ipykernel==6.29.5",
     "msgpack>=1.1.1",
-    "mlx>=0.26.3; sys_platform == 'darwin' and platform_machine == 'arm64'",
-    "mlx-lm>=0.26.0; sys_platform == 'darwin' and platform_machine == 'arm64'",
+    "mlx>=0.26.3; sys_platform == 'darwin'",
+    "mlx-lm>=0.26.0; sys_platform == 'darwin'",
     "psutil>=5.8.0",
     "pybind11>=3.0.0",
-    "pathspec>=0.12.1",
-    "nbconvert>=7.16.6",
-    "gitignore-parser>=0.1.12",
-    # AST-aware code chunking dependencies
-    "astchunk>=0.1.0",
-    "tree-sitter>=0.20.0",
-    "tree-sitter-python>=0.20.0",
-    "tree-sitter-java>=0.20.0",
-    "tree-sitter-c-sharp>=0.20.0",
-    "tree-sitter-typescript>=0.20.0",
 ]
 
 [project.optional-dependencies]
@@ -69,9 +60,11 @@ dev = [
 
 test = [
     "pytest>=7.0",
-    "pytest-timeout>=2.0",
+    "pytest-timeout>=2.0",  # Simple timeout protection for CI
     "llama-index-core>=0.12.0",
+    "llama-index-readers-file>=0.4.0",
     "python-dotenv>=1.0.0",
+    "sentence-transformers>=2.2.0",
 ]
 
 diskann = [
@@ -88,11 +81,6 @@ documents = [
 
 [tool.setuptools]
 py-modules = []
-packages = ["wechat_exporter"]
-package-dir = { "wechat_exporter" = "packages/wechat-exporter" }
-
-[project.scripts]
-wechat-exporter = "wechat_exporter.main:main"
 
 
 [tool.uv.sources]
@@ -103,8 +91,13 @@ leann-backend-hnsw = { path = "packages/leann-backend-hnsw", editable = true }
 [tool.ruff]
 target-version = "py39"
 line-length = 100
-extend-exclude = ["third_party"]
-
+extend-exclude = [
+    "third_party",
+    "*.egg-info",
+    "__pycache__",
+    ".git",
+    ".venv",
+]
 
 [tool.ruff.lint]
 select = [
@@ -127,12 +120,21 @@ ignore = [
     "RUF012", # mutable class attributes should be annotated with typing.ClassVar
 ]
 
+[tool.ruff.lint.per-file-ignores]
+"test/**/*.py" = ["E402"]      # module level import not at top of file (common in tests)
+"examples/**/*.py" = ["E402"]  # module level import not at top of file (common in examples)
+
 [tool.ruff.format]
 quote-style = "double"
 indent-style = "space"
 skip-magic-trailing-comma = false
 line-ending = "auto"
 
+[dependency-groups]
+dev = [
+    "ruff>=0.12.4",
+]
+
 [tool.lychee]
 accept = ["200", "403", "429", "503"]
 timeout = 20
@@ -150,7 +152,7 @@ markers = [
     "slow: marks tests as slow (deselect with '-m \"not slow\"')",
     "openai: marks tests that require OpenAI API key",
 ]
-timeout = 300  # Reduced from 600s (10min) to 300s (5min) for CI safety
+timeout = 300  # Simple timeout for CI safety (5 minutes)
 addopts = [
     "-v",
     "--tb=short",

sky/leann-build.yaml

@@ -1,76 +0,0 @@
-name: leann-build
-
-resources:
-  # Choose a GPU for fast embeddings (examples: L4, A10G, A100). CPU also works but is slower.
-  accelerators: L4:1
-  # Optionally pin a cloud, otherwise SkyPilot will auto-select
-  # cloud: aws
-  disk_size: 100
-
-envs:
-  # Build parameters (override with: sky launch -c leann-gpu sky/leann-build.yaml -e key=value)
-  index_name: my-index
-  docs: ./data
-  backend: hnsw               # hnsw | diskann
-  complexity: 64
-  graph_degree: 32
-  num_threads: 8
-  # Embedding selection
-  embedding_mode: sentence-transformers   # sentence-transformers | openai | mlx | ollama
-  embedding_model: facebook/contriever
-  # Storage/latency knobs
-  recompute: true             # true => selective recomputation (recommended)
-  compact: true               # for HNSW only
-  # Optional pass-through
-  extra_args: ""
-  # Rebuild control
-  force: true
-
-# Sync local paths to the remote VM. Adjust as needed.
-file_mounts:
-  # Example: mount your local data directory used for building
-  ~/leann-data: ${docs}
-
-setup: |
-  set -e
-  # Install uv (package manager)
-  curl -LsSf https://astral.sh/uv/install.sh | sh
-  export PATH="$HOME/.local/bin:$PATH"
-
-  # Ensure modern libstdc++ for FAISS (GLIBCXX >= 3.4.30)
-  sudo apt-get update -y
-  sudo apt-get install -y libstdc++6 libgomp1
-  # Also upgrade conda's libstdc++ in base env (Skypilot images include conda)
-  if command -v conda >/dev/null 2>&1; then
-    conda install -y -n base -c conda-forge libstdcxx-ng
-  fi
-
-  # Install LEANN CLI and backends into the user environment
-  uv pip install --upgrade pip
-  uv pip install leann-core leann-backend-hnsw leann-backend-diskann
-
-run: |
-  export PATH="$HOME/.local/bin:$PATH"
-  # Derive flags from env
-  recompute_flag=""
-  if [ "${recompute}" = "false" ] || [ "${recompute}" = "0" ]; then
-    recompute_flag="--no-recompute"
-  fi
-  force_flag=""
-  if [ "${force}" = "true" ] || [ "${force}" = "1" ]; then
-    force_flag="--force"
-  fi
-
-  # Build command
-  python -m leann.cli build ${index_name} \
-    --docs ~/leann-data \
-    --backend ${backend} \
-    --complexity ${complexity} \
-    --graph-degree ${graph_degree} \
-    --num-threads ${num_threads} \
-    --embedding-mode ${embedding_mode} \
-    --embedding-model ${embedding_model} \
-    ${recompute_flag} ${force_flag} ${extra_args}
-
-  # Print where the index is stored for downstream rsync
-  echo "INDEX_OUT_DIR=~/.leann/indexes/${index_name}"

tests/conftest.py

@@ -0,0 +1,41 @@
+"""Pytest configuration and fixtures for LEANN tests."""
+
+import os
+
+import pytest
+
+
+@pytest.fixture(autouse=True)
+def test_environment():
+    """Set up test environment variables."""
+    # Mark as test environment to skip memory-intensive operations
+    os.environ["CI"] = "true"
+    yield
+
+
+@pytest.fixture(scope="session", autouse=True)
+def cleanup_session():
+    """Session-level cleanup to ensure no hanging processes."""
+    yield
+
+    # Basic cleanup after all tests
+    try:
+        import os
+
+        import psutil
+
+        current_process = psutil.Process(os.getpid())
+        children = current_process.children(recursive=True)
+
+        for child in children:
+            try:
+                child.terminate()
+            except psutil.NoSuchProcess:
+                pass
+
+        # Give them time to terminate gracefully
+        psutil.wait_procs(children, timeout=3)
+
+    except Exception:
+        # Don't fail tests due to cleanup errors
+        pass

tests/test_astchunk_integration.py

@@ -1,397 +0,0 @@
-"""
-Test suite for astchunk integration with LEANN.
-Tests AST-aware chunking functionality, language detection, and fallback mechanisms.
-"""
-
-import os
-import subprocess
-import sys
-import tempfile
-from pathlib import Path
-from unittest.mock import patch
-
-import pytest
-
-# Add apps directory to path for imports
-sys.path.insert(0, str(Path(__file__).parent.parent / "apps"))
-
-from typing import Optional
-
-from chunking import (
-    create_ast_chunks,
-    create_text_chunks,
-    create_traditional_chunks,
-    detect_code_files,
-    get_language_from_extension,
-)
-
-
-class MockDocument:
-    """Mock LlamaIndex Document for testing."""
-
-    def __init__(self, content: str, file_path: str = "", metadata: Optional[dict] = None):
-        self.content = content
-        self.metadata = metadata or {}
-        if file_path:
-            self.metadata["file_path"] = file_path
-
-    def get_content(self) -> str:
-        return self.content
-
-
-class TestCodeFileDetection:
-    """Test code file detection and language mapping."""
-
-    def test_detect_code_files_python(self):
-        """Test detection of Python files."""
-        docs = [
-            MockDocument("print('hello')", "/path/to/file.py"),
-            MockDocument("This is text", "/path/to/file.txt"),
-        ]
-
-        code_docs, text_docs = detect_code_files(docs)
-
-        assert len(code_docs) == 1
-        assert len(text_docs) == 1
-        assert code_docs[0].metadata["language"] == "python"
-        assert code_docs[0].metadata["is_code"] is True
-        assert text_docs[0].metadata["is_code"] is False
-
-    def test_detect_code_files_multiple_languages(self):
-        """Test detection of multiple programming languages."""
-        docs = [
-            MockDocument("def func():", "/path/to/script.py"),
-            MockDocument("public class Test {}", "/path/to/Test.java"),
-            MockDocument("interface ITest {}", "/path/to/test.ts"),
-            MockDocument("using System;", "/path/to/Program.cs"),
-            MockDocument("Regular text content", "/path/to/document.txt"),
-        ]
-
-        code_docs, text_docs = detect_code_files(docs)
-
-        assert len(code_docs) == 4
-        assert len(text_docs) == 1
-
-        languages = [doc.metadata["language"] for doc in code_docs]
-        assert "python" in languages
-        assert "java" in languages
-        assert "typescript" in languages
-        assert "csharp" in languages
-
-    def test_detect_code_files_no_file_path(self):
-        """Test handling of documents without file paths."""
-        docs = [
-            MockDocument("some content"),
-            MockDocument("other content", metadata={"some_key": "value"}),
-        ]
-
-        code_docs, text_docs = detect_code_files(docs)
-
-        assert len(code_docs) == 0
-        assert len(text_docs) == 2
-        for doc in text_docs:
-            assert doc.metadata["is_code"] is False
-
-    def test_get_language_from_extension(self):
-        """Test language detection from file extensions."""
-        assert get_language_from_extension("test.py") == "python"
-        assert get_language_from_extension("Test.java") == "java"
-        assert get_language_from_extension("component.tsx") == "typescript"
-        assert get_language_from_extension("Program.cs") == "csharp"
-        assert get_language_from_extension("document.txt") is None
-        assert get_language_from_extension("") is None
-
-
-class TestChunkingFunctions:
-    """Test various chunking functionality."""
-
-    def test_create_traditional_chunks(self):
-        """Test traditional text chunking."""
-        docs = [
-            MockDocument(
-                "This is a test document. It has multiple sentences. We want to test chunking."
-            )
-        ]
-
-        chunks = create_traditional_chunks(docs, chunk_size=50, chunk_overlap=10)
-
-        assert len(chunks) > 0
-        assert all(isinstance(chunk, str) for chunk in chunks)
-        assert all(len(chunk.strip()) > 0 for chunk in chunks)
-
-    def test_create_traditional_chunks_empty_docs(self):
-        """Test traditional chunking with empty documents."""
-        chunks = create_traditional_chunks([], chunk_size=50, chunk_overlap=10)
-        assert chunks == []
-
-    @pytest.mark.skipif(
-        os.environ.get("CI") == "true",
-        reason="Skip astchunk tests in CI - dependency may not be available",
-    )
-    def test_create_ast_chunks_with_astchunk_available(self):
-        """Test AST chunking when astchunk is available."""
-        python_code = '''
-def hello_world():
-    """Print hello world message."""
-    print("Hello, World!")
-
-def add_numbers(a, b):
-    """Add two numbers and return the result."""
-    return a + b
-
-class Calculator:
-    """A simple calculator class."""
-
-    def __init__(self):
-        self.history = []
-
-    def add(self, a, b):
-        result = a + b
-        self.history.append(f"{a} + {b} = {result}")
-        return result
-'''
-
-        docs = [MockDocument(python_code, "/test/calculator.py", {"language": "python"})]
-
-        try:
-            chunks = create_ast_chunks(docs, max_chunk_size=200, chunk_overlap=50)
-
-            # Should have multiple chunks due to different functions/classes
-            assert len(chunks) > 0
-            assert all(isinstance(chunk, str) for chunk in chunks)
-            assert all(len(chunk.strip()) > 0 for chunk in chunks)
-
-            # Check that code structure is somewhat preserved
-            combined_content = " ".join(chunks)
-            assert "def hello_world" in combined_content
-            assert "class Calculator" in combined_content
-
-        except ImportError:
-            # astchunk not available, should fall back to traditional chunking
-            chunks = create_ast_chunks(docs, max_chunk_size=200, chunk_overlap=50)
-            assert len(chunks) > 0  # Should still get chunks from fallback
-
-    def test_create_ast_chunks_fallback_to_traditional(self):
-        """Test AST chunking falls back to traditional when astchunk is not available."""
-        docs = [MockDocument("def test(): pass", "/test/script.py", {"language": "python"})]
-
-        # Mock astchunk import to fail
-        with patch("chunking.create_ast_chunks"):
-            # First call (actual test) should import astchunk and potentially fail
-            # Let's call the actual function to test the import error handling
-            chunks = create_ast_chunks(docs)
-
-            # Should return some chunks (either from astchunk or fallback)
-            assert isinstance(chunks, list)
-
-    def test_create_text_chunks_traditional_mode(self):
-        """Test text chunking in traditional mode."""
-        docs = [
-            MockDocument("def test(): pass", "/test/script.py"),
-            MockDocument("This is regular text.", "/test/doc.txt"),
-        ]
-
-        chunks = create_text_chunks(docs, use_ast_chunking=False, chunk_size=50, chunk_overlap=10)
-
-        assert len(chunks) > 0
-        assert all(isinstance(chunk, str) for chunk in chunks)
-
-    def test_create_text_chunks_ast_mode(self):
-        """Test text chunking in AST mode."""
-        docs = [
-            MockDocument("def test(): pass", "/test/script.py"),
-            MockDocument("This is regular text.", "/test/doc.txt"),
-        ]
-
-        chunks = create_text_chunks(
-            docs,
-            use_ast_chunking=True,
-            ast_chunk_size=100,
-            ast_chunk_overlap=20,
-            chunk_size=50,
-            chunk_overlap=10,
-        )
-
-        assert len(chunks) > 0
-        assert all(isinstance(chunk, str) for chunk in chunks)
-
-    def test_create_text_chunks_custom_extensions(self):
-        """Test text chunking with custom code file extensions."""
-        docs = [
-            MockDocument("function test() {}", "/test/script.js"),  # Not in default extensions
-            MockDocument("Regular text", "/test/doc.txt"),
-        ]
-
-        # First without custom extensions - should treat .js as text
-        chunks_without = create_text_chunks(docs, use_ast_chunking=True, code_file_extensions=None)
-
-        # Then with custom extensions - should treat .js as code
-        chunks_with = create_text_chunks(
-            docs, use_ast_chunking=True, code_file_extensions=[".js", ".jsx"]
-        )
-
-        # Both should return chunks
-        assert len(chunks_without) > 0
-        assert len(chunks_with) > 0
-
-
-class TestIntegrationWithDocumentRAG:
-    """Integration tests with the document RAG system."""
-
-    @pytest.fixture
-    def temp_code_dir(self):
-        """Create a temporary directory with sample code files."""
-        with tempfile.TemporaryDirectory() as temp_dir:
-            temp_path = Path(temp_dir)
-
-            # Create sample Python file
-            python_file = temp_path / "example.py"
-            python_file.write_text('''
-def fibonacci(n):
-    """Calculate fibonacci number."""
-    if n <= 1:
-        return n
-    return fibonacci(n-1) + fibonacci(n-2)
-
-class MathUtils:
-    @staticmethod
-    def factorial(n):
-        if n <= 1:
-            return 1
-        return n * MathUtils.factorial(n-1)
-''')
-
-            # Create sample text file
-            text_file = temp_path / "readme.txt"
-            text_file.write_text("This is a sample text file for testing purposes.")
-
-            yield temp_path
-
-    @pytest.mark.skipif(
-        os.environ.get("CI") == "true",
-        reason="Skip integration tests in CI to avoid dependency issues",
-    )
-    def test_document_rag_with_ast_chunking(self, temp_code_dir):
-        """Test document RAG with AST chunking enabled."""
-        with tempfile.TemporaryDirectory() as index_dir:
-            cmd = [
-                sys.executable,
-                "apps/document_rag.py",
-                "--llm",
-                "simulated",
-                "--embedding-model",
-                "facebook/contriever",
-                "--embedding-mode",
-                "sentence-transformers",
-                "--index-dir",
-                index_dir,
-                "--data-dir",
-                str(temp_code_dir),
-                "--enable-code-chunking",
-                "--query",
-                "How does the fibonacci function work?",
-            ]
-
-            env = os.environ.copy()
-            env["HF_HUB_DISABLE_SYMLINKS"] = "1"
-            env["TOKENIZERS_PARALLELISM"] = "false"
-
-            try:
-                result = subprocess.run(
-                    cmd,
-                    capture_output=True,
-                    text=True,
-                    timeout=300,  # 5 minutes
-                    env=env,
-                )
-
-                # Should succeed even if astchunk is not available (fallback)
-                assert result.returncode == 0, f"Command failed: {result.stderr}"
-
-                output = result.stdout + result.stderr
-                assert "Index saved to" in output or "Using existing index" in output
-
-            except subprocess.TimeoutExpired:
-                pytest.skip("Test timed out - likely due to model download in CI")
-
-    @pytest.mark.skipif(
-        os.environ.get("CI") == "true",
-        reason="Skip integration tests in CI to avoid dependency issues",
-    )
-    def test_code_rag_application(self, temp_code_dir):
-        """Test the specialized code RAG application."""
-        with tempfile.TemporaryDirectory() as index_dir:
-            cmd = [
-                sys.executable,
-                "apps/code_rag.py",
-                "--llm",
-                "simulated",
-                "--embedding-model",
-                "facebook/contriever",
-                "--index-dir",
-                index_dir,
-                "--repo-dir",
-                str(temp_code_dir),
-                "--query",
-                "What classes are defined in this code?",
-            ]
-
-            env = os.environ.copy()
-            env["HF_HUB_DISABLE_SYMLINKS"] = "1"
-            env["TOKENIZERS_PARALLELISM"] = "false"
-
-            try:
-                result = subprocess.run(cmd, capture_output=True, text=True, timeout=300, env=env)
-
-                # Should succeed
-                assert result.returncode == 0, f"Command failed: {result.stderr}"
-
-                output = result.stdout + result.stderr
-                assert "Using AST-aware chunking" in output or "traditional chunking" in output
-
-            except subprocess.TimeoutExpired:
-                pytest.skip("Test timed out - likely due to model download in CI")
-
-
-class TestErrorHandling:
-    """Test error handling and edge cases."""
-
-    def test_text_chunking_empty_documents(self):
-        """Test text chunking with empty document list."""
-        chunks = create_text_chunks([])
-        assert chunks == []
-
-    def test_text_chunking_invalid_parameters(self):
-        """Test text chunking with invalid parameters."""
-        docs = [MockDocument("test content")]
-
-        # Should handle negative chunk sizes gracefully
-        chunks = create_text_chunks(
-            docs, chunk_size=0, chunk_overlap=0, ast_chunk_size=0, ast_chunk_overlap=0
-        )
-
-        # Should still return some result
-        assert isinstance(chunks, list)
-
-    def test_create_ast_chunks_no_language(self):
-        """Test AST chunking with documents missing language metadata."""
-        docs = [MockDocument("def test(): pass", "/test/script.py")]  # No language set
-
-        chunks = create_ast_chunks(docs)
-
-        # Should fall back to traditional chunking
-        assert isinstance(chunks, list)
-        assert len(chunks) >= 0  # May be empty if fallback also fails
-
-    def test_create_ast_chunks_empty_content(self):
-        """Test AST chunking with empty content."""
-        docs = [MockDocument("", "/test/script.py", {"language": "python"})]
-
-        chunks = create_ast_chunks(docs)
-
-        # Should handle empty content gracefully
-        assert isinstance(chunks, list)
-
-
-if __name__ == "__main__":
-    pytest.main([__file__, "-v"])

tests/test_basic.py

@@ -64,9 +64,6 @@ def test_backend_basic(backend_name):
         assert isinstance(results[0], SearchResult)
         assert "topic 2" in results[0].text or "document" in results[0].text
 
-        # Ensure cleanup to avoid hanging background servers
-        searcher.cleanup()
-
 
 @pytest.mark.skipif(
     os.environ.get("CI") == "true", reason="Skip model tests in CI to avoid MPS memory issues"
@@ -93,5 +90,3 @@ def test_large_index():
         searcher = LeannSearcher(index_path)
         results = searcher.search(["word10 word20"], top_k=10)
         assert len(results[0]) == 10
-        # Cleanup
-        searcher.cleanup()

tests/test_document_rag.py

@@ -57,55 +57,7 @@ def test_document_rag_simulated(test_data_dir):
         assert "This is a simulated answer" in output
 
 
-@pytest.mark.skipif(
-    os.environ.get("CI") == "true",
-    reason="Skip AST chunking tests in CI to avoid dependency issues",
-)
-def test_document_rag_with_ast_chunking(test_data_dir):
-    """Test document_rag with AST-aware chunking enabled."""
-    with tempfile.TemporaryDirectory() as temp_dir:
-        # Use a subdirectory that doesn't exist yet to force index creation
-        index_dir = Path(temp_dir) / "test_ast_index"
-        cmd = [
-            sys.executable,
-            "apps/document_rag.py",
-            "--llm",
-            "simulated",
-            "--embedding-model",
-            "facebook/contriever",
-            "--embedding-mode",
-            "sentence-transformers",
-            "--index-dir",
-            str(index_dir),
-            "--data-dir",
-            str(test_data_dir),
-            "--enable-code-chunking",  # Enable AST chunking
-            "--query",
-            "What is Pride and Prejudice about?",
-        ]
-
-        env = os.environ.copy()
-        env["HF_HUB_DISABLE_SYMLINKS"] = "1"
-        env["TOKENIZERS_PARALLELISM"] = "false"
-
-        result = subprocess.run(cmd, capture_output=True, text=True, timeout=600, env=env)
-
-        # Check return code
-        assert result.returncode == 0, f"Command failed: {result.stderr}"
-
-        # Verify output
-        output = result.stdout + result.stderr
-        assert "Index saved to" in output or "Using existing index" in output
-        assert "This is a simulated answer" in output
-
-        # Should mention AST chunking if code files are present
-        # (might not be relevant for the test data, but command should succeed)
-
-
 @pytest.mark.skipif(not os.environ.get("OPENAI_API_KEY"), reason="OpenAI API key not available")
-@pytest.mark.skipif(
-    os.environ.get("CI") == "true", reason="Skip OpenAI tests in CI to avoid API costs"
-)
 def test_document_rag_openai(test_data_dir):
     """Test document_rag with OpenAI embeddings."""
     with tempfile.TemporaryDirectory() as temp_dir:

tests/test_readme_examples.py

@@ -16,9 +16,6 @@ def test_readme_basic_example(backend_name):
     # Skip on macOS CI due to MPS environment issues with all-MiniLM-L6-v2
     if os.environ.get("CI") == "true" and platform.system() == "Darwin":
         pytest.skip("Skipping on macOS CI due to MPS environment issues with all-MiniLM-L6-v2")
-    # Skip DiskANN on CI (Linux runners) due to C++ extension memory/hardware constraints
-    if os.environ.get("CI") == "true" and backend_name == "diskann":
-        pytest.skip("Skip DiskANN tests in CI due to resource constraints and instability")
 
     # This is the exact code from README (with smaller model for CI)
     from leann import LeannBuilder, LeannChat, LeannSearcher
@@ -62,9 +59,6 @@ def test_readme_basic_example(backend_name):
         # The second text about banana-crocodile should be more relevant
         assert "banana" in results[0].text or "crocodile" in results[0].text
 
-        # Ensure we cleanup background embedding server
-        searcher.cleanup()
-
         # Chat with your data (using simulated LLM to avoid external dependencies)
         chat = LeannChat(INDEX_PATH, llm_config={"type": "simulated"})
         response = chat.ask("How much storage does LEANN save?", top_k=1)
@@ -72,8 +66,6 @@ def test_readme_basic_example(backend_name):
         # Verify chat works
         assert isinstance(response, str)
         assert len(response) > 0
-        # Cleanup chat resources
-        chat.cleanup()
 
 
 def test_readme_imports():