docs: Group iMessage with WeChat under chat history

- Move iMessage from agent memory to chat history category
- Group WeChat and iMessage together as personal chat history
- Keep ChatGPT and Claude as AI agent memory
- Better categorization based on feedback

Resolves #127

.gitattributes

@@ -1 +0,0 @@
-paper_plot/data/big_graph_degree_data.npz filter=lfs diff=lfs merge=lfs -text

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,50 @@
+name: Bug Report
+description: Report a bug in LEANN
+labels: ["bug"]
+
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: What happened?
+      description: A clear description of the bug
+    validations:
+      required: true
+
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: How to reproduce
+      placeholder: |
+        1. Install with...
+        2. Run command...
+        3. See error
+    validations:
+      required: true
+
+  - type: textarea
+    id: error
+    attributes:
+      label: Error message
+      description: Paste any error messages
+      render: shell
+
+  - type: input
+    id: version
+    attributes:
+      label: LEANN Version
+      placeholder: "0.1.0"
+    validations:
+      required: true
+
+  - type: dropdown
+    id: os
+    attributes:
+      label: Operating System
+      options:
+        - macOS
+        - Linux
+        - Windows
+        - Docker
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Documentation
+    url: https://github.com/LEANN-RAG/LEANN-RAG/tree/main/docs
+    about: Read the docs first
+  - name: Discussions
+    url: https://github.com/LEANN-RAG/LEANN-RAG/discussions
+    about: Ask questions and share ideas

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,27 @@
+name: Feature Request
+description: Suggest a new feature for LEANN
+labels: ["enhancement"]
+
+body:
+  - type: textarea
+    id: problem
+    attributes:
+      label: What problem does this solve?
+      description: Describe the problem or need
+    validations:
+      required: true
+
+  - type: textarea
+    id: solution
+    attributes:
+      label: Proposed solution
+      description: How would you like this to work?
+    validations:
+      required: true
+
+  - type: textarea
+    id: example
+    attributes:
+      label: Example usage
+      description: Show how the API might look
+      render: python

.github/pull_request_template.md

@@ -0,0 +1,13 @@
+## What does this PR do?
+
+<!-- Brief description of your changes -->
+
+## Related Issues
+
+Fixes #
+
+## Checklist
+
+- [ ] Tests pass (`uv run pytest`)
+- [ ] Code formatted (`ruff format` and `ruff check`)
+- [ ] Pre-commit hooks pass (`pre-commit run --all-files`)

.github/workflows/build-and-publish.yml

@@ -6,15 +6,7 @@ on:
   pull_request:
     branches: [ main ]
   workflow_dispatch:
-    inputs:
-      debug_enabled:
-        type: boolean
-        description: 'Run with tmate debugging enabled (SSH access to runner)'
-        required: false
-        default: false
 
 jobs:
   build:
     uses: ./.github/workflows/build-reusable.yml
-    with:
-      debug_enabled: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled || false }}

.github/workflows/build-reusable.yml

@@ -8,11 +8,6 @@ on:
         required: false
         type: string
         default: ''
-      debug_enabled:
-        description: 'Enable tmate debugging session for troubleshooting'
-        required: false
-        type: boolean
-        default: false
 
 jobs:
   lint:
@@ -33,7 +28,7 @@ jobs:
 
       - name: Install ruff
         run: |
-          uv tool install ruff==0.12.7
+          uv tool install ruff
 
       - name: Run ruff check
         run: |
@@ -59,20 +54,51 @@ jobs:
             python: '3.12'
           - os: ubuntu-22.04
             python: '3.13'
-          - os: macos-latest
+          # ARM64 Linux builds
+          - os: ubuntu-24.04-arm
             python: '3.9'
-          - os: macos-latest
+          - os: ubuntu-24.04-arm
             python: '3.10'
-          - os: macos-latest
+          - os: ubuntu-24.04-arm
             python: '3.11'
-          - os: macos-latest
+          - os: ubuntu-24.04-arm
             python: '3.12'
-          - os: macos-latest
+          - os: ubuntu-24.04-arm
             python: '3.13'
+          - os: macos-14
+            python: '3.9'
+          - os: macos-14
+            python: '3.10'
+          - os: macos-14
+            python: '3.11'
+          - os: macos-14
+            python: '3.12'
+          - os: macos-14
+            python: '3.13'
+          - os: macos-15
+            python: '3.9'
+          - os: macos-15
+            python: '3.10'
+          - os: macos-15
+            python: '3.11'
+          - os: macos-15
+            python: '3.12'
+          - os: macos-15
+            python: '3.13'
+          - os: macos-13
+            python: '3.9'
+          - os: macos-13
+            python: '3.10'
+          - os: macos-13
+            python: '3.11'
+          - os: macos-13
+            python: '3.12'
+          # Note: macos-13 + Python 3.13 excluded due to PyTorch compatibility
+          # (PyTorch 2.5+ supports Python 3.13 but not Intel Mac x86_64)
     runs-on: ${{ matrix.os }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           ref: ${{ inputs.ref }}
           submodules: recursive
@@ -83,21 +109,56 @@ jobs:
           python-version: ${{ matrix.python }}
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v4
+        uses: astral-sh/setup-uv@v6
 
       - name: Install system dependencies (Ubuntu)
         if: runner.os == 'Linux'
         run: |
           sudo apt-get update
           sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \
-            pkg-config libopenblas-dev patchelf libabsl-dev libaio-dev libprotobuf-dev
+            pkg-config libabsl-dev libaio-dev libprotobuf-dev \
+            patchelf
 
-          # Install Intel MKL for DiskANN
-          wget -q https://registrationcenter-download.intel.com/akdlm/IRC_NAS/79153e0f-74d7-45af-b8c2-258941adf58a/intel-onemkl-2025.0.0.940.sh
-          sudo sh intel-onemkl-2025.0.0.940.sh -a --components intel.oneapi.lin.mkl.devel --action install --eula accept -s
-          source /opt/intel/oneapi/setvars.sh
-          echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV
-          echo "LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH" >> $GITHUB_ENV
+          # Debug: Show system information
+          echo "🔍 System Information:"
+          echo "Architecture: $(uname -m)"
+          echo "OS: $(uname -a)"
+          echo "CPU info: $(lscpu | head -5)"
+
+          # Install math library based on architecture
+          ARCH=$(uname -m)
+          echo "🔍 Setting up math library for architecture: $ARCH"
+
+          if [[ "$ARCH" == "x86_64" ]]; then
+            # Install Intel MKL for DiskANN on x86_64
+            echo "📦 Installing Intel MKL for x86_64..."
+            wget -q https://registrationcenter-download.intel.com/akdlm/IRC_NAS/79153e0f-74d7-45af-b8c2-258941adf58a/intel-onemkl-2025.0.0.940.sh
+            sudo sh intel-onemkl-2025.0.0.940.sh -a --components intel.oneapi.lin.mkl.devel --action install --eula accept -s
+            source /opt/intel/oneapi/setvars.sh
+            echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV
+            echo "LD_LIBRARY_PATH=/opt/intel/oneapi/compiler/latest/linux/compiler/lib/intel64_lin" >> $GITHUB_ENV
+            echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/intel/oneapi/mkl/latest/lib/intel64" >> $GITHUB_ENV
+            echo "✅ Intel MKL installed for x86_64"
+
+            # Debug: Check MKL installation
+            echo "🔍 MKL Installation Check:"
+            ls -la /opt/intel/oneapi/mkl/latest/ || echo "MKL directory not found"
+            ls -la /opt/intel/oneapi/mkl/latest/lib/ || echo "MKL lib directory not found"
+
+          elif [[ "$ARCH" == "aarch64" ]]; then
+            # Use OpenBLAS for ARM64 (MKL installer not compatible with ARM64)
+            echo "📦 Installing OpenBLAS for ARM64..."
+            sudo apt-get install -y libopenblas-dev liblapack-dev liblapacke-dev
+            echo "✅ OpenBLAS installed for ARM64"
+
+            # Debug: Check OpenBLAS installation
+            echo "🔍 OpenBLAS Installation Check:"
+            dpkg -l | grep openblas || echo "OpenBLAS package not found"
+            ls -la /usr/lib/aarch64-linux-gnu/openblas/ || echo "OpenBLAS directory not found"
+          fi
+
+          # Debug: Show final library paths
+          echo "🔍 Final LD_LIBRARY_PATH: $LD_LIBRARY_PATH"
 
       - name: Install system dependencies (macOS)
         if: runner.os == 'macOS'
@@ -114,41 +175,70 @@ jobs:
             uv pip install --system delocate
           fi
 
+      - name: Set macOS environment variables
+        if: runner.os == 'macOS'
+        run: |
+          # Use brew --prefix to automatically detect Homebrew installation path
+          HOMEBREW_PREFIX=$(brew --prefix)
+          echo "HOMEBREW_PREFIX=${HOMEBREW_PREFIX}" >> $GITHUB_ENV
+          echo "OpenMP_ROOT=${HOMEBREW_PREFIX}/opt/libomp" >> $GITHUB_ENV
+
+          # Set CMAKE_PREFIX_PATH to let CMake find all packages automatically
+          echo "CMAKE_PREFIX_PATH=${HOMEBREW_PREFIX}" >> $GITHUB_ENV
+
+          # Set compiler flags for OpenMP (required for both backends)
+          echo "LDFLAGS=-L${HOMEBREW_PREFIX}/opt/libomp/lib" >> $GITHUB_ENV
+          echo "CPPFLAGS=-I${HOMEBREW_PREFIX}/opt/libomp/include" >> $GITHUB_ENV
+
       - name: Build packages
         run: |
-          # Build core (platform independent) on all platforms for consistency
+          # Build core (platform independent)
           cd packages/leann-core
           uv build
           cd ../..
 
           # Build HNSW backend
           cd packages/leann-backend-hnsw
-          if [ "${{ matrix.os }}" == "macos-latest" ]; then
-            # Use system clang instead of homebrew LLVM for better compatibility
+          if [[ "${{ matrix.os }}" == macos-* ]]; then
+            # Use system clang for better compatibility
             export CC=clang
             export CXX=clang++
-            export MACOSX_DEPLOYMENT_TARGET=11.0
-            uv build --wheel --python python
+            # Homebrew libraries on each macOS version require matching minimum version
+            if [[ "${{ matrix.os }}" == "macos-13" ]]; then
+              export MACOSX_DEPLOYMENT_TARGET=13.0
+            elif [[ "${{ matrix.os }}" == "macos-14" ]]; then
+              export MACOSX_DEPLOYMENT_TARGET=14.0
+            elif [[ "${{ matrix.os }}" == "macos-15" ]]; then
+              export MACOSX_DEPLOYMENT_TARGET=15.0
+            fi
+            uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
           else
-            uv build --wheel --python python
+            uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
           fi
           cd ../..
 
           # Build DiskANN backend
           cd packages/leann-backend-diskann
-          if [ "${{ matrix.os }}" == "macos-latest" ]; then
-            # Use system clang instead of homebrew LLVM for better compatibility
+          if [[ "${{ matrix.os }}" == macos-* ]]; then
+            # Use system clang for better compatibility
             export CC=clang
             export CXX=clang++
-            # sgesdd_ is only available on macOS 13.3+
-            export MACOSX_DEPLOYMENT_TARGET=13.3
-            uv build --wheel --python python
+            # DiskANN requires macOS 13.3+ for sgesdd_ LAPACK function
+            # But Homebrew libraries on each macOS version require matching minimum version
+            if [[ "${{ matrix.os }}" == "macos-13" ]]; then
+              export MACOSX_DEPLOYMENT_TARGET=13.3
+            elif [[ "${{ matrix.os }}" == "macos-14" ]]; then
+              export MACOSX_DEPLOYMENT_TARGET=14.0
+            elif [[ "${{ matrix.os }}" == "macos-15" ]]; then
+              export MACOSX_DEPLOYMENT_TARGET=15.0
+            fi
+            uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
           else
-            uv build --wheel --python python
+            uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
           fi
           cd ../..
 
-          # Build meta package (platform independent) on all platforms
+          # Build meta package (platform independent)
           cd packages/leann
           uv build
           cd ../..
@@ -165,15 +255,10 @@ jobs:
           fi
           cd ../..
 
-          # Repair DiskANN wheel - use show first to debug
+          # Repair DiskANN wheel
           cd packages/leann-backend-diskann
           if [ -d dist ]; then
-            echo "Checking DiskANN wheel contents before repair:"
-            unzip -l dist/*.whl | grep -E "\.so|\.pyd|_diskannpy" || echo "No .so files found"
-            auditwheel show dist/*.whl || echo "auditwheel show failed"
             auditwheel repair dist/*.whl -w dist_repaired
-            echo "Checking DiskANN wheel contents after repair:"
-            unzip -l dist_repaired/*.whl | grep -E "\.so|\.pyd|_diskannpy" || echo "No .so files found after repair"
             rm -rf dist
             mv dist_repaired dist
           fi
@@ -182,10 +267,24 @@ jobs:
       - name: Repair wheels (macOS)
         if: runner.os == 'macOS'
         run: |
+          # Determine deployment target based on runner OS
+          # Must match the Homebrew libraries for each macOS version
+          if [[ "${{ matrix.os }}" == "macos-13" ]]; then
+            HNSW_TARGET="13.0"
+            DISKANN_TARGET="13.3"
+          elif [[ "${{ matrix.os }}" == "macos-14" ]]; then
+            HNSW_TARGET="14.0"
+            DISKANN_TARGET="14.0"
+          elif [[ "${{ matrix.os }}" == "macos-15" ]]; then
+            HNSW_TARGET="15.0"
+            DISKANN_TARGET="15.0"
+          fi
+
           # Repair HNSW wheel
           cd packages/leann-backend-hnsw
           if [ -d dist ]; then
-            delocate-wheel -w dist_repaired -v dist/*.whl
+            export MACOSX_DEPLOYMENT_TARGET=$HNSW_TARGET
+            delocate-wheel -w dist_repaired -v --require-target-macos-version $HNSW_TARGET dist/*.whl
             rm -rf dist
             mv dist_repaired dist
           fi
@@ -194,7 +293,8 @@ jobs:
           # Repair DiskANN wheel
           cd packages/leann-backend-diskann
           if [ -d dist ]; then
-            delocate-wheel -w dist_repaired -v dist/*.whl
+            export MACOSX_DEPLOYMENT_TARGET=$DISKANN_TARGET
+            delocate-wheel -w dist_repaired -v --require-target-macos-version $DISKANN_TARGET dist/*.whl
             rm -rf dist
             mv dist_repaired dist
           fi
@@ -205,242 +305,34 @@ jobs:
           echo "📦 Built packages:"
           find packages/*/dist -name "*.whl" -o -name "*.tar.gz" | sort
 
+
       - name: Install built packages for testing
         run: |
           # Create a virtual environment with the correct Python version
-          uv venv --python python${{ matrix.python }}
+          uv venv --python ${{ matrix.python }}
           source .venv/bin/activate || source .venv/Scripts/activate
 
-          # Install the built wheels directly to ensure we use locally built packages
-          # Use only locally built wheels on all platforms for full consistency
-          FIND_LINKS="--find-links packages/leann-core/dist --find-links packages/leann/dist"
-          FIND_LINKS="$FIND_LINKS --find-links packages/leann-backend-hnsw/dist --find-links packages/leann-backend-diskann/dist"
-
-          uv pip install leann-core leann leann-backend-hnsw leann-backend-diskann \
-            $FIND_LINKS --force-reinstall
+          # Install packages using --find-links to prioritize local builds
+          uv pip install --find-links packages/leann-core/dist --find-links packages/leann-backend-hnsw/dist --find-links packages/leann-backend-diskann/dist packages/leann-core/dist/*.whl || uv pip install --find-links packages/leann-core/dist packages/leann-core/dist/*.tar.gz
+          uv pip install --find-links packages/leann-core/dist packages/leann-backend-hnsw/dist/*.whl
+          uv pip install --find-links packages/leann-core/dist packages/leann-backend-diskann/dist/*.whl
+          uv pip install packages/leann/dist/*.whl || uv pip install packages/leann/dist/*.tar.gz
 
           # Install test dependencies using extras
           uv pip install -e ".[test]"
 
-          # Debug: Check if _diskannpy module is installed correctly
-          echo "Checking installed DiskANN module structure:"
-          python -c "import leann_backend_diskann; print('leann_backend_diskann location:', leann_backend_diskann.__file__)" || echo "Failed to import leann_backend_diskann"
-          python -c "from leann_backend_diskann import _diskannpy; print('_diskannpy imported successfully')" || echo "Failed to import _diskannpy"
-          ls -la $(python -c "import leann_backend_diskann; import os; print(os.path.dirname(leann_backend_diskann.__file__))" 2>/dev/null) 2>/dev/null || echo "Failed to list module directory"
-
-          # Extra debugging for Python 3.13
-          if [[ "${{ matrix.python }}" == "3.13" ]]; then
-            echo "=== Python 3.13 Debug Info ==="
-            echo "Python version details:"
-            python --version
-            python -c "import sys; print(f'sys.version_info: {sys.version_info}')"
-
-            echo "Pytest version:"
-            python -m pytest --version
-
-            echo "Testing basic pytest collection:"
-            if [[ "$RUNNER_OS" == "Linux" ]]; then
-              timeout --signal=INT 10 python -m pytest --collect-only tests/test_ci_minimal.py -v || echo "Collection timed out or failed"
-            else
-              # No timeout on macOS/Windows
-              python -m pytest --collect-only tests/test_ci_minimal.py -v || echo "Collection failed"
-            fi
-
-            echo "Testing single simple test:"
-            if [[ "$RUNNER_OS" == "Linux" ]]; then
-              timeout --signal=INT 10 python -m pytest tests/test_ci_minimal.py::test_package_imports --full-trace -v || echo "Simple test timed out or failed"
-            else
-              # No timeout on macOS/Windows
-              python -m pytest tests/test_ci_minimal.py::test_package_imports --full-trace -v || echo "Simple test failed"
-            fi
-          fi
-
-      # Enable tmate debugging session if requested
-      - name: Setup tmate session for debugging
-        if: ${{ inputs.debug_enabled }}
-        uses: mxschmitt/action-tmate@v3
-        with:
-          detached: true
-          timeout-minutes: 30
-          limit-access-to-actor: true
-
       - name: Run tests with pytest
-        # Timeout hierarchy:
-        # 1. Individual test timeout: 20s (see pyproject.toml markers)
-        # 2. Pytest session timeout: 300s (see pyproject.toml [tool.pytest.ini_options])
-        # 3. Outer shell timeout: 360s (300s + 60s buffer for cleanup)
-        # 4. GitHub Actions job timeout: 6 hours (default)
         env:
-          CI: true  # Mark as CI environment to skip memory-intensive tests
+          CI: true
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           HF_HUB_DISABLE_SYMLINKS: 1
           TOKENIZERS_PARALLELISM: false
-          PYTORCH_ENABLE_MPS_FALLBACK: 0  # Disable MPS on macOS CI to avoid memory issues
-          OMP_NUM_THREADS: 1  # Disable OpenMP parallelism to avoid libomp crashes
-          MKL_NUM_THREADS: 1  # Single thread for MKL operations
+          PYTORCH_ENABLE_MPS_FALLBACK: 0
+          OMP_NUM_THREADS: 1
+          MKL_NUM_THREADS: 1
         run: |
-          # Activate virtual environment
           source .venv/bin/activate || source .venv/Scripts/activate
-
-          # Define comprehensive diagnostic function
-          diag() {
-            echo "===== COMPREHENSIVE DIAGNOSTICS BEGIN ====="
-            date
-            echo ""
-            echo "### Current Shell Info ###"
-            echo "Shell PID: $$"
-            echo "Shell PPID: $PPID"
-            echo "Current directory: $(pwd)"
-            echo ""
-
-            echo "### Process Tree (full) ###"
-            pstree -ap 2>/dev/null || ps auxf || true
-            echo ""
-
-            echo "### All Python/Pytest Processes ###"
-            ps -ef | grep -E 'python|pytest' | grep -v grep || true
-            echo ""
-
-            echo "### Embedding Server Processes ###"
-            ps -ef | grep -E 'embedding|zmq|diskann' | grep -v grep || true
-            echo ""
-
-            echo "### Network Listeners ###"
-            ss -ltnp 2>/dev/null || netstat -ltn 2>/dev/null || true
-            echo ""
-
-            echo "### Open File Descriptors (lsof) ###"
-            lsof -p $$ 2>/dev/null | head -20 || true
-            echo ""
-
-            echo "### Zombie Processes ###"
-            ps aux | grep '<defunct>' || echo "No zombie processes"
-            echo ""
-
-            echo "### Current Jobs ###"
-            jobs -l || true
-            echo ""
-
-            echo "### /proc/PID/fd for current shell ###"
-            ls -la /proc/$$/fd 2>/dev/null || true
-            echo ""
-
-            echo "===== COMPREHENSIVE DIAGNOSTICS END ====="
-          }
-
-                    # Enable verbose logging for debugging
-          export PYTHONUNBUFFERED=1
-          export PYTEST_CURRENT_TEST=1
-
-          # Run all tests with extensive logging
-          if [[ "$RUNNER_OS" == "Linux" ]]; then
-            echo "🚀 Starting Linux test execution with timeout..."
-            echo "Current time: $(date)"
-            echo "Shell PID: $$"
-            echo "Python: $(python --version)"
-            echo "Pytest: $(pytest --version)"
-
-            # Show environment variables for debugging
-            echo "📦 Environment variables:"
-            env | grep -E "PYTHON|PYTEST|CI|RUNNER" | sort
-
-            # Set trap for diagnostics
-            trap diag INT TERM EXIT
-
-            echo "📋 Pre-test diagnostics:"
-            ps -ef | grep -E 'python|pytest' | grep -v grep || echo "No python/pytest processes before test"
-
-            # Check for any listening ports before test
-            echo "🔌 Pre-test network state:"
-            ss -ltn 2>/dev/null | grep -E "555[0-9]|556[0-9]" || echo "No embedding server ports open"
-
-            # Set timeouts - outer must be larger than pytest's internal timeout
-            # IMPORTANT: Keep PYTEST_TIMEOUT_SEC in sync with pyproject.toml [tool.pytest.ini_options] timeout
-            PYTEST_TIMEOUT_SEC=${PYTEST_TIMEOUT_SEC:-300}  # Default 300s, matches pyproject.toml
-            BUFFER_SEC=${TIMEOUT_BUFFER_SEC:-60}  # Buffer for cleanup after pytest timeout
-            OUTER_TIMEOUT_SEC=${OUTER_TIMEOUT_SEC:-$((PYTEST_TIMEOUT_SEC + BUFFER_SEC))}
-
-            echo "⏰ Timeout configuration:"
-            echo "   - Pytest internal timeout: ${PYTEST_TIMEOUT_SEC}s (from pyproject.toml)"
-            echo "   - Cleanup buffer: ${BUFFER_SEC}s"
-            echo "   - Outer shell timeout: ${OUTER_TIMEOUT_SEC}s (${PYTEST_TIMEOUT_SEC}s + ${BUFFER_SEC}s buffer)"
-            echo "   - This ensures pytest can complete its own timeout handling and cleanup"
-
-            echo "🏃 Running pytest with ${OUTER_TIMEOUT_SEC}s outer timeout..."
-
-            # Export for inner shell
-            export PYTEST_TIMEOUT_SEC OUTER_TIMEOUT_SEC BUFFER_SEC
-
-            timeout --preserve-status --signal=INT --kill-after=10 ${OUTER_TIMEOUT_SEC} bash -c '
-              echo "⏱️ Pytest starting at: $(date)"
-              echo "Running command: pytest tests/ -vv --maxfail=3 --tb=short --capture=no"
-
-              # Run pytest with maximum verbosity and no output capture
-              pytest tests/ -vv --maxfail=3 --tb=short --capture=no --log-cli-level=DEBUG 2>&1 | tee pytest.log
-              PYTEST_EXIT=${PIPESTATUS[0]}
-
-              echo "✅ Pytest finished at: $(date) with exit code: $PYTEST_EXIT"
-              echo "Last 20 lines of pytest output:"
-              tail -20 pytest.log || true
-
-              # Immediately check for leftover processes
-              echo "🔍 Post-pytest process check:"
-              ps -ef | grep -E "python|pytest|embedding" | grep -v grep || echo "No leftover processes"
-
-              # Clean up any children before exit
-              echo "🧹 Cleaning up child processes..."
-              pkill -TERM -P $$ 2>/dev/null || true
-              sleep 0.5
-              pkill -KILL -P $$ 2>/dev/null || true
-
-              echo "📊 Final check before exit:"
-              ps -ef | grep -E "python|pytest|embedding" | grep -v grep || echo "All clean"
-
-              exit $PYTEST_EXIT
-            '
-
-            EXIT_CODE=$?
-            echo "🔚 Timeout command exited with code: $EXIT_CODE"
-
-                        if [ $EXIT_CODE -eq 124 ]; then
-              echo "⚠️ TIMEOUT TRIGGERED - Tests took more than ${OUTER_TIMEOUT_SEC} seconds!"
-              echo "📸 Capturing full diagnostics..."
-              diag
-
-              # Run diagnostic script if available
-              if [ -f scripts/diagnose_hang.sh ]; then
-                echo "🔍 Running diagnostic script..."
-                bash scripts/diagnose_hang.sh || true
-              fi
-
-              # More aggressive cleanup
-              echo "💀 Killing all Python processes owned by runner..."
-              pkill -9 -u runner python || true
-              pkill -9 -u runner pytest || true
-            elif [ $EXIT_CODE -ne 0 ]; then
-              echo "❌ Tests failed with exit code: $EXIT_CODE"
-            else
-              echo "✅ All tests passed!"
-            fi
-
-                        # Always show final state
-            echo "📍 Final state check:"
-            ps -ef | grep -E 'python|pytest|embedding' | grep -v grep || echo "No Python processes remaining"
-
-            exit $EXIT_CODE
-          else
-            # For macOS/Windows, run without GNU timeout
-            echo "🚀 Running tests on $RUNNER_OS..."
-            pytest tests/ -vv --maxfail=3 --tb=short --capture=no --log-cli-level=INFO
-          fi
-
-      # Provide tmate session on test failure for debugging
-      - name: Setup tmate session on failure
-        if: ${{ failure() && (inputs.debug_enabled || contains(github.event.head_commit.message, '[debug]')) }}
-        uses: mxschmitt/action-tmate@v3
-        with:
-          timeout-minutes: 30
-          limit-access-to-actor: true
+          pytest tests/ -v --tb=short
 
       - name: Run sanity checks (optional)
         run: |
@@ -458,3 +350,53 @@ jobs:
         with:
           name: packages-${{ matrix.os }}-py${{ matrix.python }}
           path: packages/*/dist/
+
+
+  arch-smoke:
+    name: Arch Linux smoke test (install & import)
+    needs: build
+    runs-on: ubuntu-latest
+    container:
+      image: archlinux:latest
+
+    steps:
+      - name: Prepare system
+        run: |
+          pacman -Syu --noconfirm
+          pacman -S --noconfirm python python-pip gcc git zlib openssl
+
+      - name: Download ALL wheel artifacts from this run
+        uses: actions/download-artifact@v5
+        with:
+          # Don't specify name, download all artifacts
+          path: ./wheels
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+
+      - name: Create virtual environment and install wheels
+        run: |
+          uv venv
+          source .venv/bin/activate || source .venv/Scripts/activate
+          uv pip install --find-links wheels leann-core
+          uv pip install --find-links wheels leann-backend-hnsw
+          uv pip install --find-links wheels leann-backend-diskann
+          uv pip install --find-links wheels leann
+
+      - name: Import & tiny runtime check
+        env:
+          OMP_NUM_THREADS: 1
+          MKL_NUM_THREADS: 1
+        run: |
+          source .venv/bin/activate || source .venv/Scripts/activate
+          python - <<'PY'
+          import leann
+          import leann_backend_hnsw as h
+          import leann_backend_diskann as d
+          from leann import LeannBuilder, LeannSearcher
+          b = LeannBuilder(backend_name="hnsw")
+          b.add_text("hello arch")
+          b.build_index("arch_demo.leann")
+          s = LeannSearcher("arch_demo.leann")
+          print("search:", s.search("hello", top_k=1))
+          PY

.github/workflows/link-check.yml

@@ -14,6 +14,6 @@ jobs:
       - uses: actions/checkout@v4
       - uses: lycheeverse/lychee-action@v2
         with:
-          args: --no-progress --insecure README.md docs/ apps/ examples/ benchmarks/
+          args: --no-progress --insecure --user-agent 'curl/7.68.0' README.md docs/ apps/ examples/ benchmarks/
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -18,9 +18,11 @@ demo/experiment_results/**/*.json
 *.eml
 *.emlx
 *.json
+!.vscode/*.json
 *.sh
 *.txt
 !CMakeLists.txt
+!llms.txt
 latency_breakdown*.json
 experiment_results/eval_results/diskann/*.json
 aws/
@@ -92,3 +94,10 @@ packages/leann-backend-diskann/third_party/DiskANN/_deps/
 batchtest.py
 tests/__pytest_cache__/
 tests/__pycache__/
+paru-bin/
+
+CLAUDE.md
+CLAUDE.local.md
+.claude/*.local.*
+.claude/local/*
+benchmarks/data/

.gitmodules

@@ -14,3 +14,6 @@
 [submodule "packages/leann-backend-hnsw/third_party/libzmq"]
 	path = packages/leann-backend-hnsw/third_party/libzmq
 	url = https://github.com/zeromq/libzmq.git
+[submodule "packages/astchunk-leann"]
+	path = packages/astchunk-leann
+	url = https://github.com/yichuan-w/astchunk-leann.git

.pre-commit-config.yaml

@@ -13,4 +13,5 @@ repos:
     rev: v0.12.7  # Fixed version to match pyproject.toml
     hooks:
       - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format

.vscode/extensions.json

@@ -0,0 +1,5 @@
+{
+    "recommendations": [
+        "charliermarsh.ruff",
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,22 @@
+{
+  "python.defaultInterpreterPath": ".venv/bin/python",
+  "python.terminal.activateEnvironment": true,
+  "[python]": {
+    "editor.defaultFormatter": "charliermarsh.ruff",
+    "editor.formatOnSave": true,
+    "editor.codeActionsOnSave": {
+      "source.organizeImports": "explicit",
+      "source.fixAll": "explicit"
+    },
+    "editor.insertSpaces": true,
+    "editor.tabSize": 4
+  },
+  "ruff.enable": true,
+  "files.watcherExclude": {
+    "**/.venv/**": true,
+    "**/__pycache__/**": true,
+    "**/*.egg-info/**": true,
+    "**/build/**": true,
+    "**/dist/**": true
+  }
+}

README.md

@@ -3,10 +3,13 @@
 </p>
 
 <p align="center">
-  <img src="https://img.shields.io/badge/Python-3.9%2B-blue.svg" alt="Python 3.9+">
+  <img src="https://img.shields.io/badge/Python-3.9%20%7C%203.10%20%7C%203.11%20%7C%203.12%20%7C%203.13-blue.svg" alt="Python Versions">
+  <img src="https://github.com/yichuan-w/LEANN/actions/workflows/build-and-publish.yml/badge.svg" alt="CI Status">
+  <img src="https://img.shields.io/badge/Platform-Ubuntu%20%26%20Arch%20%26%20WSL%20%7C%20macOS%20(ARM64%2FIntel)-lightgrey" alt="Platform">
   <img src="https://img.shields.io/badge/License-MIT-green.svg" alt="MIT License">
-  <img src="https://img.shields.io/badge/Platform-Linux%20%7C%20macOS-lightgrey" alt="Platform">
-  <img src="https://img.shields.io/badge/MCP-Native%20Integration-blue?style=flat-square" alt="MCP Integration">
+  <img src="https://img.shields.io/badge/MCP-Native%20Integration-blue" alt="MCP Integration">
+  <a href="https://join.slack.com/t/leann-e2u9779/shared_invite/zt-3ckd2f6w1-OX08~NN4gkWhh10PRVBj1Q"><img src="https://img.shields.io/badge/Slack-Join-4A154B?logo=slack&logoColor=white" alt="Join Slack">
+  <a href="assets/wechat_user_group.JPG" title="Join WeChat group"><img src="https://img.shields.io/badge/WeChat-Join-2DC100?logo=wechat&logoColor=white" alt="Join WeChat group"></a>
 </p>
 
 <h2 align="center" tabindex="-1" class="heading-element" dir="auto">
@@ -17,7 +20,7 @@ LEANN is an innovative vector database that democratizes personal AI. Transform
 
 LEANN achieves this through *graph-based selective recomputation* with *high-degree preserving pruning*, computing embeddings on-demand instead of storing them all. [Illustration Fig →](#️-architecture--how-it-works) | [Paper →](https://arxiv.org/abs/2506.08276)
 
-**Ready to RAG Everything?** Transform your laptop into a personal AI assistant that can semantic search your **[file system](#-personal-data-manager-process-any-documents-pdf-txt-md)**, **[emails](#-your-personal-email-secretary-rag-on-apple-mail)**, **[browser history](#-time-machine-for-the-web-rag-your-entire-browser-history)**, **[chat history](#-wechat-detective-unlock-your-golden-memories)**, **[codebase](#-claude-code-integration-transform-your-development-workflow)**\* , or external knowledge bases (i.e., 60M documents) - all on your laptop, with zero cloud costs and complete privacy.
+**Ready to RAG Everything?** Transform your laptop into a personal AI assistant that can semantic search your **[file system](#-personal-data-manager-process-any-documents-pdf-txt-md)**, **[emails](#-your-personal-email-secretary-rag-on-apple-mail)**, **[browser history](#-time-machine-for-the-web-rag-your-entire-browser-history)**, **[chat history](#-wechat-detective-unlock-your-golden-memories)** ([WeChat](#-wechat-detective-unlock-your-golden-memories), [iMessage](#-imessage-history-your-personal-conversation-archive)), **[agent memory](#-chatgpt-chat-history-your-personal-ai-conversation-archive)** ([ChatGPT](#-chatgpt-chat-history-your-personal-ai-conversation-archive), [Claude](#-claude-chat-history-your-personal-ai-conversation-archive)), **[codebase](#-claude-code-integration-transform-your-development-workflow)**\* , or external knowledge bases (i.e., 60M documents) - all on your laptop, with zero cloud costs and complete privacy.
 
 
 \* Claude Code only supports basic `grep`-style keyword search. **LEANN** is a drop-in **semantic search MCP service fully compatible with Claude Code**, unlocking intelligent retrieval without changing your workflow. 🔥 Check out [the easy setup →](packages/leann-mcp/README.md)
@@ -30,7 +33,7 @@ LEANN achieves this through *graph-based selective recomputation* with *high-deg
   <img src="assets/effects.png" alt="LEANN vs Traditional Vector DB Storage Comparison" width="70%">
 </p>
 
-> **The numbers speak for themselves:** Index 60 million text chunks in just 6GB instead of 201GB. From emails to browser history, everything fits on your laptop. [See detailed benchmarks for different applications below ↓](#storage-comparison)
+> **The numbers speak for themselves:** Index 60 million text chunks in just 6GB instead of 201GB. From emails to browser history, everything fits on your laptop. [See detailed benchmarks for different applications below ↓](#-storage-comparison)
 
 
 🔒 **Privacy:** Your data never leaves your laptop. No OpenAI, no cloud, no "terms of service".
@@ -69,6 +72,8 @@ uv venv
 source .venv/bin/activate
 uv pip install leann
 ```
+<!--
+> Low-resource? See “Low-resource setups” in the [Configuration Guide](docs/configuration-guide.md#low-resource-setups). -->
 
 <details>
 <summary>
@@ -84,15 +89,60 @@ git submodule update --init --recursive
 ```
 
 **macOS:**
+
+Note: DiskANN requires MacOS 13.3 or later.
+
 ```bash
-brew install llvm libomp boost protobuf zeromq pkgconf
-CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv sync
+brew install libomp boost protobuf zeromq pkgconf
+uv sync --extra diskann
 ```
 
-**Linux:**
+**Linux (Ubuntu/Debian):**
+
+Note: On Ubuntu 20.04, you may need to build a newer Abseil and pin Protobuf (e.g., v3.20.x) for building DiskANN. See [Issue #30](https://github.com/yichuan-w/LEANN/issues/30) for a step-by-step note.
+
+You can manually install [Intel oneAPI MKL](https://www.intel.com/content/www/us/en/developer/tools/oneapi/onemkl.html) instead of `libmkl-full-dev` for DiskANN. You can also use `libopenblas-dev` for building HNSW only, by removing `--extra diskann` in the command below.
+
 ```bash
-sudo apt-get install libomp-dev libboost-all-dev protobuf-compiler libabsl-dev libmkl-full-dev libaio-dev libzmq3-dev
-uv sync
+sudo apt-get update && sudo apt-get install -y \
+  libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \
+  pkg-config libabsl-dev libaio-dev libprotobuf-dev \
+  libmkl-full-dev
+
+uv sync --extra diskann
+```
+
+**Linux (Arch Linux):**
+
+```bash
+sudo pacman -Syu && sudo pacman -S --needed base-devel cmake pkgconf git gcc \
+  boost boost-libs protobuf abseil-cpp libaio zeromq
+
+# For MKL in DiskANN
+sudo pacman -S --needed base-devel git
+git clone https://aur.archlinux.org/paru-bin.git
+cd paru-bin && makepkg -si
+paru -S intel-oneapi-mkl intel-oneapi-compiler
+source /opt/intel/oneapi/setvars.sh
+
+uv sync --extra diskann
+```
+
+**Linux (RHEL / CentOS Stream / Oracle / Rocky / AlmaLinux):**
+
+See [Issue #50](https://github.com/yichuan-w/LEANN/issues/50) for more details.
+
+```bash
+sudo dnf groupinstall -y "Development Tools"
+sudo dnf install -y libomp-devel boost-devel protobuf-compiler protobuf-devel \
+  abseil-cpp-devel libaio-devel zeromq-devel pkgconf-pkg-config
+
+# For MKL in DiskANN
+sudo dnf install -y intel-oneapi-mkl intel-oneapi-mkl-devel \
+  intel-oneapi-openmp || sudo dnf install -y intel-oneapi-compiler
+source /opt/intel/oneapi/setvars.sh
+
+uv sync --extra diskann
 ```
 
 </details>
@@ -126,7 +176,9 @@ response = chat.ask("How much storage does LEANN save?", top_k=1)
 
 ## RAG on Everything!
 
-LEANN supports RAG on various data sources including documents (`.pdf`, `.txt`, `.md`), Apple Mail, Google Search History, WeChat, and more.
+LEANN supports RAG on various data sources including documents (`.pdf`, `.txt`, `.md`), Apple Mail, Google Search History, WeChat, ChatGPT conversations, Claude conversations, iMessage conversations, and more.
+
+
 
 ### Generation Model Setup
 
@@ -170,7 +222,8 @@ ollama pull llama3.2:1b
 
 </details>
 
-### ⭐ Flexible Configuration
+
+## ⭐ Flexible Configuration
 
 LEANN provides flexible parameters for embedding models, search strategies, and data processing to fit your specific needs.
 
@@ -183,34 +236,34 @@ All RAG examples share these common parameters. **Interactive mode** is availabl
 
 ```bash
 # Core Parameters (General preprocessing for all examples)
---index-dir DIR          # Directory to store the index (default: current directory)
---query "YOUR QUESTION"  # Single query mode. Omit for interactive chat (type 'quit' to exit), and now you can play with your index interactively
---max-items N           # Limit data preprocessing (default: -1, process all data)
---force-rebuild         # Force rebuild index even if it exists
+--index-dir DIR              # Directory to store the index (default: current directory)
+--query "YOUR QUESTION"      # Single query mode. Omit for interactive chat (type 'quit' to exit), and now you can play with your index interactively
+--max-items N                # Limit data preprocessing (default: -1, process all data)
+--force-rebuild              # Force rebuild index even if it exists
 
 # Embedding Parameters
---embedding-model MODEL  # e.g., facebook/contriever, text-embedding-3-small, nomic-embed-text, mlx-community/Qwen3-Embedding-0.6B-8bit or nomic-embed-text
---embedding-mode MODE    # sentence-transformers, openai, mlx, or ollama
+--embedding-model MODEL      # e.g., facebook/contriever, text-embedding-3-small, mlx-community/Qwen3-Embedding-0.6B-8bit or nomic-embed-text
+--embedding-mode MODE        # sentence-transformers, openai, mlx, or ollama
 
 # LLM Parameters (Text generation models)
---llm TYPE              # LLM backend: openai, ollama, or hf (default: openai)
---llm-model MODEL       # Model name (default: gpt-4o) e.g., gpt-4o-mini, llama3.2:1b, Qwen/Qwen2.5-1.5B-Instruct
---thinking-budget LEVEL # Thinking budget for reasoning models: low/medium/high (supported by o3, o3-mini, GPT-Oss:20b, and other reasoning models)
+--llm TYPE                   # LLM backend: openai, ollama, or hf (default: openai)
+--llm-model MODEL            # Model name (default: gpt-4o) e.g., gpt-4o-mini, llama3.2:1b, Qwen/Qwen2.5-1.5B-Instruct
+--thinking-budget LEVEL      # Thinking budget for reasoning models: low/medium/high (supported by o3, o3-mini, GPT-Oss:20b, and other reasoning models)
 
 # Search Parameters
---top-k N               # Number of results to retrieve (default: 20)
---search-complexity N   # Search complexity for graph traversal (default: 32)
+--top-k N                    # Number of results to retrieve (default: 20)
+--search-complexity N        # Search complexity for graph traversal (default: 32)
 
 # Chunking Parameters
---chunk-size N          # Size of text chunks (default varies by source: 256 for most, 192 for WeChat)
---chunk-overlap N       # Overlap between chunks (default varies: 25-128 depending on source)
+--chunk-size N               # Size of text chunks (default varies by source: 256 for most, 192 for WeChat)
+--chunk-overlap N            # Overlap between chunks (default varies: 25-128 depending on source)
 
 # Index Building Parameters
---backend-name NAME     # Backend to use: hnsw or diskann (default: hnsw)
---graph-degree N        # Graph degree for index construction (default: 32)
---build-complexity N    # Build complexity for index construction (default: 64)
---no-compact           # Disable compact index storage (compact storage IS enabled to save storage by default)
---no-recompute         # Disable embedding recomputation (recomputation IS enabled to save storage by default)
+--backend-name NAME          # Backend to use: hnsw or diskann (default: hnsw)
+--graph-degree N             # Graph degree for index construction (default: 32)
+--build-complexity N         # Build complexity for index construction (default: 64)
+--compact / --no-compact     # Use compact storage (default: true). Must be `no-compact` for `no-recompute` build.
+--recompute / --no-recompute # Enable/disable embedding recomputation (default: enabled). Should not do a `no-recompute` search in a `recompute` build.
 ```
 
 </details>
@@ -246,6 +299,12 @@ python -m apps.document_rag --data-dir "~/Documents/Papers" --chunk-size 1024
 
 # Filter only markdown and Python files with smaller chunks
 python -m apps.document_rag --data-dir "./docs" --chunk-size 256 --file-types .md .py
+
+# Enable AST-aware chunking for code files
+python -m apps.document_rag --enable-code-chunking --data-dir "./my_project"
+
+# Or use the specialized code RAG for better code understanding
+python -m apps.code_rag --repo-dir "./my_codebase" --query "How does authentication work?"
 ```
 
 </details>
@@ -418,26 +477,268 @@ Once the index is built, you can ask questions like:
 
 </details>
 
+### 🤖 ChatGPT Chat History: Your Personal AI Conversation Archive!
+
+Transform your ChatGPT conversations into a searchable knowledge base! Search through all your ChatGPT discussions about coding, research, brainstorming, and more.
+
+```bash
+python -m apps.chatgpt_rag --export-path chatgpt_export.html --query "How do I create a list in Python?"
+```
+
+**Unlock your AI conversation history.** Never lose track of valuable insights from your ChatGPT discussions again.
+
+<details>
+<summary><strong>📋 Click to expand: How to Export ChatGPT Data</strong></summary>
+
+**Step-by-step export process:**
+
+1. **Sign in to ChatGPT**
+2. **Click your profile icon** in the top right corner
+3. **Navigate to Settings** → **Data Controls**
+4. **Click "Export"** under Export Data
+5. **Confirm the export** request
+6. **Download the ZIP file** from the email link (expires in 24 hours)
+7. **Extract or use directly** with LEANN
+
+**Supported formats:**
+- `.html` files from ChatGPT exports
+- `.zip` archives from ChatGPT
+- Directories with multiple export files
+
+</details>
+
+<details>
+<summary><strong>📋 Click to expand: ChatGPT-Specific Arguments</strong></summary>
+
+#### Parameters
+```bash
+--export-path PATH           # Path to ChatGPT export file (.html/.zip) or directory (default: ./chatgpt_export)
+--separate-messages         # Process each message separately instead of concatenated conversations
+--chunk-size N              # Text chunk size (default: 512)
+--chunk-overlap N           # Overlap between chunks (default: 128)
+```
+
+#### Example Commands
+```bash
+# Basic usage with HTML export
+python -m apps.chatgpt_rag --export-path conversations.html
+
+# Process ZIP archive from ChatGPT
+python -m apps.chatgpt_rag --export-path chatgpt_export.zip
+
+# Search with specific query
+python -m apps.chatgpt_rag --export-path chatgpt_data.html --query "Python programming help"
+
+# Process individual messages for fine-grained search
+python -m apps.chatgpt_rag --separate-messages --export-path chatgpt_export.html
+
+# Process directory containing multiple exports
+python -m apps.chatgpt_rag --export-path ./chatgpt_exports/ --max-items 1000
+```
+
+</details>
+
+<details>
+<summary><strong>💡 Click to expand: Example queries you can try</strong></summary>
+
+Once your ChatGPT conversations are indexed, you can search with queries like:
+- "What did I ask ChatGPT about Python programming?"
+- "Show me conversations about machine learning algorithms"
+- "Find discussions about web development frameworks"
+- "What coding advice did ChatGPT give me?"
+- "Search for conversations about debugging techniques"
+- "Find ChatGPT's recommendations for learning resources"
+
+</details>
+
+### 🤖 Claude Chat History: Your Personal AI Conversation Archive!
+
+Transform your Claude conversations into a searchable knowledge base! Search through all your Claude discussions about coding, research, brainstorming, and more.
+
+```bash
+python -m apps.claude_rag --export-path claude_export.json --query "What did I ask about Python dictionaries?"
+```
+
+**Unlock your AI conversation history.** Never lose track of valuable insights from your Claude discussions again.
+
+<details>
+<summary><strong>📋 Click to expand: How to Export Claude Data</strong></summary>
+
+**Step-by-step export process:**
+
+1. **Open Claude** in your browser
+2. **Navigate to Settings** (look for gear icon or settings menu)
+3. **Find Export/Download** options in your account settings
+4. **Download conversation data** (usually in JSON format)
+5. **Place the file** in your project directory
+
+*Note: Claude export methods may vary depending on the interface you're using. Check Claude's help documentation for the most current export instructions.*
+
+**Supported formats:**
+- `.json` files (recommended)
+- `.zip` archives containing JSON data
+- Directories with multiple export files
+
+</details>
+
+<details>
+<summary><strong>📋 Click to expand: Claude-Specific Arguments</strong></summary>
+
+#### Parameters
+```bash
+--export-path PATH           # Path to Claude export file (.json/.zip) or directory (default: ./claude_export)
+--separate-messages         # Process each message separately instead of concatenated conversations
+--chunk-size N              # Text chunk size (default: 512)
+--chunk-overlap N           # Overlap between chunks (default: 128)
+```
+
+#### Example Commands
+```bash
+# Basic usage with JSON export
+python -m apps.claude_rag --export-path my_claude_conversations.json
+
+# Process ZIP archive from Claude
+python -m apps.claude_rag --export-path claude_export.zip
+
+# Search with specific query
+python -m apps.claude_rag --export-path claude_data.json --query "machine learning advice"
+
+# Process individual messages for fine-grained search
+python -m apps.claude_rag --separate-messages --export-path claude_export.json
+
+# Process directory containing multiple exports
+python -m apps.claude_rag --export-path ./claude_exports/ --max-items 1000
+```
+
+</details>
+
+<details>
+<summary><strong>💡 Click to expand: Example queries you can try</strong></summary>
+
+Once your Claude conversations are indexed, you can search with queries like:
+- "What did I ask Claude about Python programming?"
+- "Show me conversations about machine learning algorithms"
+- "Find discussions about software architecture patterns"
+- "What debugging advice did Claude give me?"
+- "Search for conversations about data structures"
+- "Find Claude's recommendations for learning resources"
+
+</details>
+
+### 💬 iMessage History: Your Personal Conversation Archive!
+
+Transform your iMessage conversations into a searchable knowledge base! Search through all your text messages, group chats, and conversations with friends, family, and colleagues.
+
+```bash
+python -m apps.imessage_rag --query "What did we discuss about the weekend plans?"
+```
+
+**Unlock your message history.** Never lose track of important conversations, shared links, or memorable moments from your iMessage history.
+
+<details>
+<summary><strong>📋 Click to expand: How to Access iMessage Data</strong></summary>
+
+**iMessage data location:**
+
+iMessage conversations are stored in a SQLite database on your Mac at:
+```
+~/Library/Messages/chat.db
+```
+
+**Important setup requirements:**
+
+1. **Grant Full Disk Access** to your terminal or IDE:
+   - Open **System Preferences** → **Security & Privacy** → **Privacy**
+   - Select **Full Disk Access** from the left sidebar
+   - Click the **+** button and add your terminal app (Terminal, iTerm2) or IDE (VS Code, etc.)
+   - Restart your terminal/IDE after granting access
+
+2. **Alternative: Use a backup database**
+   - If you have Time Machine backups or manual copies of the database
+   - Use `--db-path` to specify a custom location
+
+**Supported formats:**
+- Direct access to `~/Library/Messages/chat.db` (default)
+- Custom database path with `--db-path`
+- Works with backup copies of the database
+
+</details>
+
+<details>
+<summary><strong>📋 Click to expand: iMessage-Specific Arguments</strong></summary>
+
+#### Parameters
+```bash
+--db-path PATH                    # Path to chat.db file (default: ~/Library/Messages/chat.db)
+--concatenate-conversations       # Group messages by conversation (default: True)
+--no-concatenate-conversations    # Process each message individually
+--chunk-size N                    # Text chunk size (default: 1000)
+--chunk-overlap N                 # Overlap between chunks (default: 200)
+```
+
+#### Example Commands
+```bash
+# Basic usage (requires Full Disk Access)
+python -m apps.imessage_rag
+
+# Search with specific query
+python -m apps.imessage_rag --query "family dinner plans"
+
+# Use custom database path
+python -m apps.imessage_rag --db-path /path/to/backup/chat.db
+
+# Process individual messages instead of conversations
+python -m apps.imessage_rag --no-concatenate-conversations
+
+# Limit processing for testing
+python -m apps.imessage_rag --max-items 100 --query "weekend"
+```
+
+</details>
+
+<details>
+<summary><strong>💡 Click to expand: Example queries you can try</strong></summary>
+
+Once your iMessage conversations are indexed, you can search with queries like:
+- "What did we discuss about vacation plans?"
+- "Find messages about restaurant recommendations"
+- "Show me conversations with John about the project"
+- "Search for shared links about technology"
+- "Find group chat discussions about weekend events"
+- "What did mom say about the family gathering?"
+
+</details>
+
 ### 🚀 Claude Code Integration: Transform Your Development Workflow!
 
+<details>
+<summary><strong>NEW!! AST‑Aware Code Chunking</strong></summary>
+
+LEANN features intelligent code chunking that preserves semantic boundaries (functions, classes, methods) for Python, Java, C#, and TypeScript, improving code understanding compared to text-based chunking.
+
+📖 Read the [AST Chunking Guide →](docs/ast_chunking_guide.md)
+
+</details>
+
 **The future of code assistance is here.** Transform your development workflow with LEANN's native MCP integration for Claude Code. Index your entire codebase and get intelligent code assistance directly in your IDE.
 
 **Key features:**
-- 🔍 **Semantic code search** across your entire project
+- 🔍 **Semantic code search** across your entire project, fully local index and lightweight
+- 🧠 **AST-aware chunking** preserves code structure (functions, classes)
 - 📚 **Context-aware assistance** for debugging and development
 - 🚀 **Zero-config setup** with automatic language detection
 
 ```bash
 # Install LEANN globally for MCP integration
-uv tool install leann-core
-
+uv tool install leann-core --with leann
+claude mcp add --scope user leann-server -- leann_mcp
 # Setup is automatic - just start using Claude Code!
 ```
 Try our fully agentic pipeline with auto query rewriting, semantic search planning, and more:
 
 ![LEANN MCP Integration](assets/mcp_leann.png)
 
-**Ready to supercharge your coding?** [Complete Setup Guide →](packages/leann-mcp/README.md)
+**🔥 Ready to supercharge your coding?** [Complete Setup Guide →](packages/leann-mcp/README.md)
 
 ## 🖥️ Command Line Interface
 
@@ -454,7 +755,8 @@ leann --help
 **To make it globally available:**
 ```bash
 # Install the LEANN CLI globally using uv tool
-uv tool install leann-core
+uv tool install leann-core --with leann
+
 
 # Now you can use leann from anywhere without activating venv
 leann --help
@@ -467,7 +769,7 @@ leann --help
 ### Usage Examples
 
 ```bash
-# build from a specific directory, and my_docs is the index name
+# build from a specific directory, and my_docs is the index name(Here you can also build from multiple dict or multiple files)
 leann build my-docs --docs ./your_documents
 
 # Search your documents
@@ -478,30 +780,36 @@ leann ask my-docs --interactive
 
 # List all your indexes
 leann list
+
+# Remove an index
+leann remove my-docs
 ```
 
 **Key CLI features:**
-- Auto-detects document formats (PDF, TXT, MD, DOCX)
-- Smart text chunking with overlap
+- Auto-detects document formats (PDF, TXT, MD, DOCX, PPTX + code files)
+- **🧠 AST-aware chunking** for Python, Java, C#, TypeScript files
+- Smart text chunking with overlap for all other content
 - Multiple LLM providers (Ollama, OpenAI, HuggingFace)
-- Organized index storage in `~/.leann/indexes/`
+- Organized index storage in `.leann/indexes/` (project-local)
 - Support for advanced search parameters
 
 <details>
 <summary><strong>📋 Click to expand: Complete CLI Reference</strong></summary>
 
+You can use `leann --help`, or `leann build --help`, `leann search --help`, `leann ask --help`, `leann list --help`, `leann remove --help` to get the complete CLI reference.
+
 **Build Command:**
 ```bash
-leann build INDEX_NAME --docs DIRECTORY [OPTIONS]
+leann build INDEX_NAME --docs DIRECTORY|FILE [DIRECTORY|FILE ...] [OPTIONS]
 
 Options:
   --backend {hnsw,diskann}     Backend to use (default: hnsw)
   --embedding-model MODEL      Embedding model (default: facebook/contriever)
-  --graph-degree N            Graph degree (default: 32)
-  --complexity N              Build complexity (default: 64)
-  --force                     Force rebuild existing index
-  --compact                   Use compact storage (default: true)
-  --recompute                 Enable recomputation (default: true)
+  --graph-degree N             Graph degree (default: 32)
+  --complexity N               Build complexity (default: 64)
+  --force                      Force rebuild existing index
+  --compact / --no-compact     Use compact storage (default: true). Must be `no-compact` for `no-recompute` build.
+  --recompute / --no-recompute Enable recomputation (default: true)
 ```
 
 **Search Command:**
@@ -509,9 +817,9 @@ Options:
 leann search INDEX_NAME QUERY [OPTIONS]
 
 Options:
-  --top-k N                   Number of results (default: 5)
-  --complexity N              Search complexity (default: 64)
-  --recompute-embeddings      Use recomputation for highest accuracy
+  --top-k N                     Number of results (default: 5)
+  --complexity N                Search complexity (default: 64)
+  --recompute / --no-recompute  Enable/disable embedding recomputation (default: enabled). Should not do a `no-recompute` search in a `recompute` build.
   --pruning-strategy {global,local,proportional}
 ```
 
@@ -526,8 +834,73 @@ Options:
   --top-k N                  Retrieval count (default: 20)
 ```
 
+**List Command:**
+```bash
+leann list
+
+# Lists all indexes across all projects with status indicators:
+# ✅ - Index is complete and ready to use
+# ❌ - Index is incomplete or corrupted
+# 📁 - CLI-created index (in .leann/indexes/)
+# 📄 - App-created index (*.leann.meta.json files)
+```
+
+**Remove Command:**
+```bash
+leann remove INDEX_NAME [OPTIONS]
+
+Options:
+  --force, -f    Force removal without confirmation
+
+# Smart removal: automatically finds and safely removes indexes
+# - Shows all matching indexes across projects
+# - Requires confirmation for cross-project removal
+# - Interactive selection when multiple matches found
+# - Supports both CLI and app-created indexes
+```
+
 </details>
 
+## 🚀 Advanced Features
+
+### 🎯 Metadata Filtering
+
+LEANN supports a simple metadata filtering system to enable sophisticated use cases like document filtering by date/type, code search by file extension, and content management based on custom criteria.
+
+```python
+# Add metadata during indexing
+builder.add_text(
+    "def authenticate_user(token): ...",
+    metadata={"file_extension": ".py", "lines_of_code": 25}
+)
+
+# Search with filters
+results = searcher.search(
+    query="authentication function",
+    metadata_filters={
+        "file_extension": {"==": ".py"},
+        "lines_of_code": {"<": 100}
+    }
+)
+```
+
+**Supported operators**: `==`, `!=`, `<`, `<=`, `>`, `>=`, `in`, `not_in`, `contains`, `starts_with`, `ends_with`, `is_true`, `is_false`
+
+📖 **[Complete Metadata filtering guide →](docs/metadata_filtering.md)**
+
+### 🔍 Grep Search
+
+For exact text matching instead of semantic search, use the `use_grep` parameter:
+
+```python
+# Exact text search
+results = searcher.search("banana‑crocodile", use_grep=True, top_k=1)
+```
+
+**Use cases**: Finding specific code patterns, error messages, function names, or exact phrases where semantic similarity isn't needed.
+
+📖 **[Complete grep search guide →](docs/grep_search.md)**
+
 ## 🏗️ Architecture & How It Works
 
 <p align="center">
@@ -567,6 +940,7 @@ Options:
 ```bash
 uv pip install -e ".[dev]"  # Install dev dependencies
 python benchmarks/run_evaluation.py    # Will auto-download evaluation data and run benchmarks
+python benchmarks/run_evaluation.py benchmarks/data/indices/rpj_wiki/rpj_wiki --num-queries 2000    # After downloading data, you can run the benchmark with our biggest index
 ```
 
 The evaluation script downloads data automatically on first run. The last three results were tested with partial personal data, and you can reproduce them with your own data!
@@ -606,12 +980,16 @@ MIT License - see [LICENSE](LICENSE) for details.
 
 Core Contributors: [Yichuan Wang](https://yichuan-w.github.io/) & [Zhifei Li](https://github.com/andylizf).
 
+Active Contributors: [Gabriel Dehan](https://github.com/gabriel-dehan)
+
+
 We welcome more contributors! Feel free to open issues or submit PRs.
 
 This work is done at [**Berkeley Sky Computing Lab**](https://sky.cs.berkeley.edu/).
 
----
+## Star History
 
+[![Star History Chart](https://api.star-history.com/svg?repos=yichuan-w/LEANN&type=Date)](https://www.star-history.com/#yichuan-w/LEANN&Date)
 <p align="center">
   <strong>⭐ Star us on GitHub if Leann is useful for your research or applications!</strong>
 </p>

apps/base_rag_example.py

@@ -10,7 +10,7 @@ from typing import Any
 
 import dotenv
 from leann.api import LeannBuilder, LeannChat
-from llama_index.core.node_parser import SentenceSplitter
+from leann.registry import register_project_directory
 
 dotenv.load_dotenv()
 
@@ -69,14 +69,14 @@ class BaseRAGExample(ABC):
             "--embedding-model",
             type=str,
             default=embedding_model_default,
-            help=f"Embedding model to use (default: {embedding_model_default})",
+            help=f"Embedding model to use (default: {embedding_model_default}), we provide facebook/contriever, text-embedding-3-small,mlx-community/Qwen3-Embedding-0.6B-8bit or nomic-embed-text",
         )
         embedding_group.add_argument(
             "--embedding-mode",
             type=str,
             default="sentence-transformers",
             choices=["sentence-transformers", "openai", "mlx", "ollama"],
-            help="Embedding backend mode (default: sentence-transformers)",
+            help="Embedding backend mode (default: sentence-transformers), we provide sentence-transformers, openai, mlx, or ollama",
         )
 
         # LLM parameters
@@ -86,13 +86,13 @@ class BaseRAGExample(ABC):
             type=str,
             default="openai",
             choices=["openai", "ollama", "hf", "simulated"],
-            help="LLM backend to use (default: openai)",
+            help="LLM backend: openai, ollama, or hf (default: openai)",
         )
         llm_group.add_argument(
             "--llm-model",
             type=str,
             default=None,
-            help="LLM model name (default: gpt-4o for openai, llama3.2:1b for ollama)",
+            help="Model name (default: gpt-4o) e.g., gpt-4o-mini, llama3.2:1b, Qwen/Qwen2.5-1.5B-Instruct",
         )
         llm_group.add_argument(
             "--llm-host",
@@ -108,6 +108,38 @@ class BaseRAGExample(ABC):
             help="Thinking budget for reasoning models (low/medium/high). Supported by GPT-Oss:20b and other reasoning models.",
         )
 
+        # AST Chunking parameters
+        ast_group = parser.add_argument_group("AST Chunking Parameters")
+        ast_group.add_argument(
+            "--use-ast-chunking",
+            action="store_true",
+            help="Enable AST-aware chunking for code files (requires astchunk)",
+        )
+        ast_group.add_argument(
+            "--ast-chunk-size",
+            type=int,
+            default=512,
+            help="Maximum characters per AST chunk (default: 512)",
+        )
+        ast_group.add_argument(
+            "--ast-chunk-overlap",
+            type=int,
+            default=64,
+            help="Overlap between AST chunks (default: 64)",
+        )
+        ast_group.add_argument(
+            "--code-file-extensions",
+            nargs="+",
+            default=None,
+            help="Additional code file extensions to process with AST chunking (e.g., .py .java .cs .ts)",
+        )
+        ast_group.add_argument(
+            "--ast-fallback-traditional",
+            action="store_true",
+            default=True,
+            help="Fall back to traditional chunking if AST chunking fails (default: True)",
+        )
+
         # Search parameters
         search_group = parser.add_argument_group("Search Parameters")
         search_group.add_argument(
@@ -214,6 +246,11 @@ class BaseRAGExample(ABC):
         builder.build_index(index_path)
         print(f"Index saved to: {index_path}")
 
+        # Register project directory so leann list can discover this index
+        # The index is saved as args.index_dir/index_name.leann
+        # We want to register the current working directory where the app is run
+        register_project_directory(Path.cwd())
+
         return index_path
 
     async def run_interactive_chat(self, args, index_path: str):
@@ -262,7 +299,6 @@ class BaseRAGExample(ABC):
         chat = LeannChat(
             index_path,
             llm_config=self.get_llm_config(args),
-            system_prompt=f"You are a helpful assistant that answers questions about {self.name} data.",
             complexity=args.search_complexity,
         )
 
@@ -304,21 +340,3 @@ class BaseRAGExample(ABC):
             await self.run_single_query(args, index_path, args.query)
         else:
             await self.run_interactive_chat(args, index_path)
-
-
-def create_text_chunks(documents, chunk_size=256, chunk_overlap=25) -> list[str]:
-    """Helper function to create text chunks from documents."""
-    node_parser = SentenceSplitter(
-        chunk_size=chunk_size,
-        chunk_overlap=chunk_overlap,
-        separator=" ",
-        paragraph_separator="\n\n",
-    )
-
-    all_texts = []
-    for doc in documents:
-        nodes = node_parser.get_nodes_from_documents([doc])
-        if nodes:
-            all_texts.extend(node.get_content() for node in nodes)
-
-    return all_texts

apps/browser_rag.py

@@ -10,7 +10,8 @@ from pathlib import Path
 # Add parent directory to path for imports
 sys.path.insert(0, str(Path(__file__).parent))
 
-from base_rag_example import BaseRAGExample, create_text_chunks
+from base_rag_example import BaseRAGExample
+from chunking import create_text_chunks
 
 from .history_data.history import ChromeHistoryReader
 

apps/chatgpt_data/chatgpt_reader.py

@@ -0,0 +1,413 @@
+"""
+ChatGPT export data reader.
+
+Reads and processes ChatGPT export data from chat.html files.
+"""
+
+import re
+from pathlib import Path
+from typing import Any
+from zipfile import ZipFile
+
+from bs4 import BeautifulSoup
+from llama_index.core import Document
+from llama_index.core.readers.base import BaseReader
+
+
+class ChatGPTReader(BaseReader):
+    """
+    ChatGPT export data reader.
+
+    Reads ChatGPT conversation data from exported chat.html files or zip archives.
+    Processes conversations into structured documents with metadata.
+    """
+
+    def __init__(self, concatenate_conversations: bool = True) -> None:
+        """
+        Initialize.
+
+        Args:
+            concatenate_conversations: Whether to concatenate messages within conversations for better context
+        """
+        try:
+            from bs4 import BeautifulSoup  # noqa
+        except ImportError:
+            raise ImportError("`beautifulsoup4` package not found: `pip install beautifulsoup4`")
+
+        self.concatenate_conversations = concatenate_conversations
+
+    def _extract_html_from_zip(self, zip_path: Path) -> str | None:
+        """
+        Extract chat.html from ChatGPT export zip file.
+
+        Args:
+            zip_path: Path to the ChatGPT export zip file
+
+        Returns:
+            HTML content as string, or None if not found
+        """
+        try:
+            with ZipFile(zip_path, "r") as zip_file:
+                # Look for chat.html or conversations.html
+                html_files = [
+                    f
+                    for f in zip_file.namelist()
+                    if f.endswith(".html") and ("chat" in f.lower() or "conversation" in f.lower())
+                ]
+
+                if not html_files:
+                    print(f"No HTML chat file found in {zip_path}")
+                    return None
+
+                # Use the first HTML file found
+                html_file = html_files[0]
+                print(f"Found HTML file: {html_file}")
+
+                with zip_file.open(html_file) as f:
+                    return f.read().decode("utf-8", errors="ignore")
+
+        except Exception as e:
+            print(f"Error extracting HTML from zip {zip_path}: {e}")
+            return None
+
+    def _parse_chatgpt_html(self, html_content: str) -> list[dict]:
+        """
+        Parse ChatGPT HTML export to extract conversations.
+
+        Args:
+            html_content: HTML content from ChatGPT export
+
+        Returns:
+            List of conversation dictionaries
+        """
+        soup = BeautifulSoup(html_content, "html.parser")
+        conversations = []
+
+        # Try different possible structures for ChatGPT exports
+        # Structure 1: Look for conversation containers
+        conversation_containers = soup.find_all(
+            ["div", "section"], class_=re.compile(r"conversation|chat", re.I)
+        )
+
+        if not conversation_containers:
+            # Structure 2: Look for message containers directly
+            conversation_containers = [soup]  # Use the entire document as one conversation
+
+        for container in conversation_containers:
+            conversation = self._extract_conversation_from_container(container)
+            if conversation and conversation.get("messages"):
+                conversations.append(conversation)
+
+        # If no structured conversations found, try to extract all text as one conversation
+        if not conversations:
+            all_text = soup.get_text(separator="\n", strip=True)
+            if all_text:
+                conversations.append(
+                    {
+                        "title": "ChatGPT Conversation",
+                        "messages": [{"role": "mixed", "content": all_text, "timestamp": None}],
+                        "timestamp": None,
+                    }
+                )
+
+        return conversations
+
+    def _extract_conversation_from_container(self, container) -> dict | None:
+        """
+        Extract conversation data from a container element.
+
+        Args:
+            container: BeautifulSoup element containing conversation
+
+        Returns:
+            Dictionary with conversation data or None
+        """
+        messages = []
+
+        # Look for message elements with various possible structures
+        message_selectors = ['[class*="message"]', '[class*="chat"]', "[data-message]", "p", "div"]
+
+        for selector in message_selectors:
+            message_elements = container.select(selector)
+            if message_elements:
+                break
+        else:
+            message_elements = []
+
+        # If no structured messages found, treat the entire container as one message
+        if not message_elements:
+            text_content = container.get_text(separator="\n", strip=True)
+            if text_content:
+                messages.append({"role": "mixed", "content": text_content, "timestamp": None})
+        else:
+            for element in message_elements:
+                message = self._extract_message_from_element(element)
+                if message:
+                    messages.append(message)
+
+        if not messages:
+            return None
+
+        # Try to extract conversation title
+        title_element = container.find(["h1", "h2", "h3", "title"])
+        title = title_element.get_text(strip=True) if title_element else "ChatGPT Conversation"
+
+        # Try to extract timestamp from various possible locations
+        timestamp = self._extract_timestamp_from_container(container)
+
+        return {"title": title, "messages": messages, "timestamp": timestamp}
+
+    def _extract_message_from_element(self, element) -> dict | None:
+        """
+        Extract message data from an element.
+
+        Args:
+            element: BeautifulSoup element containing message
+
+        Returns:
+            Dictionary with message data or None
+        """
+        text_content = element.get_text(separator=" ", strip=True)
+
+        # Skip empty or very short messages
+        if not text_content or len(text_content.strip()) < 3:
+            return None
+
+        # Try to determine role (user/assistant) from class names or content
+        role = "mixed"  # Default role
+
+        class_names = " ".join(element.get("class", [])).lower()
+        if "user" in class_names or "human" in class_names:
+            role = "user"
+        elif "assistant" in class_names or "ai" in class_names or "gpt" in class_names:
+            role = "assistant"
+        elif text_content.lower().startswith(("you:", "user:", "me:")):
+            role = "user"
+            text_content = re.sub(r"^(you|user|me):\s*", "", text_content, flags=re.IGNORECASE)
+        elif text_content.lower().startswith(("chatgpt:", "assistant:", "ai:")):
+            role = "assistant"
+            text_content = re.sub(
+                r"^(chatgpt|assistant|ai):\s*", "", text_content, flags=re.IGNORECASE
+            )
+
+        # Try to extract timestamp
+        timestamp = self._extract_timestamp_from_element(element)
+
+        return {"role": role, "content": text_content, "timestamp": timestamp}
+
+    def _extract_timestamp_from_element(self, element) -> str | None:
+        """Extract timestamp from element."""
+        # Look for timestamp in various attributes and child elements
+        timestamp_attrs = ["data-timestamp", "timestamp", "datetime"]
+        for attr in timestamp_attrs:
+            if element.get(attr):
+                return element.get(attr)
+
+        # Look for time elements
+        time_element = element.find("time")
+        if time_element:
+            return time_element.get("datetime") or time_element.get_text(strip=True)
+
+        # Look for date-like text patterns
+        text = element.get_text()
+        date_patterns = [r"\d{4}-\d{2}-\d{2}", r"\d{1,2}/\d{1,2}/\d{4}", r"\w+ \d{1,2}, \d{4}"]
+
+        for pattern in date_patterns:
+            match = re.search(pattern, text)
+            if match:
+                return match.group()
+
+        return None
+
+    def _extract_timestamp_from_container(self, container) -> str | None:
+        """Extract timestamp from conversation container."""
+        return self._extract_timestamp_from_element(container)
+
+    def _create_concatenated_content(self, conversation: dict) -> str:
+        """
+        Create concatenated content from conversation messages.
+
+        Args:
+            conversation: Dictionary containing conversation data
+
+        Returns:
+            Formatted concatenated content
+        """
+        title = conversation.get("title", "ChatGPT Conversation")
+        messages = conversation.get("messages", [])
+        timestamp = conversation.get("timestamp", "Unknown")
+
+        # Build message content
+        message_parts = []
+        for message in messages:
+            role = message.get("role", "mixed")
+            content = message.get("content", "")
+            msg_timestamp = message.get("timestamp", "")
+
+            if role == "user":
+                prefix = "[You]"
+            elif role == "assistant":
+                prefix = "[ChatGPT]"
+            else:
+                prefix = "[Message]"
+
+            # Add timestamp if available
+            if msg_timestamp:
+                prefix += f" ({msg_timestamp})"
+
+            message_parts.append(f"{prefix}: {content}")
+
+        concatenated_text = "\n\n".join(message_parts)
+
+        # Create final document content
+        doc_content = f"""Conversation: {title}
+Date: {timestamp}
+Messages ({len(messages)} messages):
+
+{concatenated_text}
+"""
+        return doc_content
+
+    def load_data(self, input_dir: str | None = None, **load_kwargs: Any) -> list[Document]:
+        """
+        Load ChatGPT export data.
+
+        Args:
+            input_dir: Directory containing ChatGPT export files or path to specific file
+            **load_kwargs:
+                max_count (int): Maximum number of conversations to process
+                chatgpt_export_path (str): Specific path to ChatGPT export file/directory
+                include_metadata (bool): Whether to include metadata in documents
+        """
+        docs: list[Document] = []
+        max_count = load_kwargs.get("max_count", -1)
+        chatgpt_export_path = load_kwargs.get("chatgpt_export_path", input_dir)
+        include_metadata = load_kwargs.get("include_metadata", True)
+
+        if not chatgpt_export_path:
+            print("No ChatGPT export path provided")
+            return docs
+
+        export_path = Path(chatgpt_export_path)
+
+        if not export_path.exists():
+            print(f"ChatGPT export path not found: {export_path}")
+            return docs
+
+        html_content = None
+
+        # Handle different input types
+        if export_path.is_file():
+            if export_path.suffix.lower() == ".zip":
+                # Extract HTML from zip file
+                html_content = self._extract_html_from_zip(export_path)
+            elif export_path.suffix.lower() == ".html":
+                # Read HTML file directly
+                try:
+                    with open(export_path, encoding="utf-8", errors="ignore") as f:
+                        html_content = f.read()
+                except Exception as e:
+                    print(f"Error reading HTML file {export_path}: {e}")
+                    return docs
+            else:
+                print(f"Unsupported file type: {export_path.suffix}")
+                return docs
+
+        elif export_path.is_dir():
+            # Look for HTML files in directory
+            html_files = list(export_path.glob("*.html"))
+            zip_files = list(export_path.glob("*.zip"))
+
+            if html_files:
+                # Use first HTML file found
+                html_file = html_files[0]
+                print(f"Found HTML file: {html_file}")
+                try:
+                    with open(html_file, encoding="utf-8", errors="ignore") as f:
+                        html_content = f.read()
+                except Exception as e:
+                    print(f"Error reading HTML file {html_file}: {e}")
+                    return docs
+
+            elif zip_files:
+                # Use first zip file found
+                zip_file = zip_files[0]
+                print(f"Found zip file: {zip_file}")
+                html_content = self._extract_html_from_zip(zip_file)
+
+            else:
+                print(f"No HTML or zip files found in {export_path}")
+                return docs
+
+        if not html_content:
+            print("No HTML content found to process")
+            return docs
+
+        # Parse conversations from HTML
+        print("Parsing ChatGPT conversations from HTML...")
+        conversations = self._parse_chatgpt_html(html_content)
+
+        if not conversations:
+            print("No conversations found in HTML content")
+            return docs
+
+        print(f"Found {len(conversations)} conversations")
+
+        # Process conversations into documents
+        count = 0
+        for conversation in conversations:
+            if max_count > 0 and count >= max_count:
+                break
+
+            if self.concatenate_conversations:
+                # Create one document per conversation with concatenated messages
+                doc_content = self._create_concatenated_content(conversation)
+
+                metadata = {}
+                if include_metadata:
+                    metadata = {
+                        "title": conversation.get("title", "ChatGPT Conversation"),
+                        "timestamp": conversation.get("timestamp", "Unknown"),
+                        "message_count": len(conversation.get("messages", [])),
+                        "source": "ChatGPT Export",
+                    }
+
+                doc = Document(text=doc_content, metadata=metadata)
+                docs.append(doc)
+                count += 1
+
+            else:
+                # Create separate documents for each message
+                for message in conversation.get("messages", []):
+                    if max_count > 0 and count >= max_count:
+                        break
+
+                    role = message.get("role", "mixed")
+                    content = message.get("content", "")
+                    msg_timestamp = message.get("timestamp", "")
+
+                    if not content.strip():
+                        continue
+
+                    # Create document content with context
+                    doc_content = f"""Conversation: {conversation.get("title", "ChatGPT Conversation")}
+Role: {role}
+Timestamp: {msg_timestamp or conversation.get("timestamp", "Unknown")}
+Message: {content}
+"""
+
+                    metadata = {}
+                    if include_metadata:
+                        metadata = {
+                            "conversation_title": conversation.get("title", "ChatGPT Conversation"),
+                            "role": role,
+                            "timestamp": msg_timestamp or conversation.get("timestamp", "Unknown"),
+                            "source": "ChatGPT Export",
+                        }
+
+                    doc = Document(text=doc_content, metadata=metadata)
+                    docs.append(doc)
+                    count += 1
+
+        print(f"Created {len(docs)} documents from ChatGPT export")
+        return docs

apps/chatgpt_rag.py

@@ -0,0 +1,186 @@
+"""
+ChatGPT RAG example using the unified interface.
+Supports ChatGPT export data from chat.html files.
+"""
+
+import sys
+from pathlib import Path
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent))
+
+from base_rag_example import BaseRAGExample
+from chunking import create_text_chunks
+
+from .chatgpt_data.chatgpt_reader import ChatGPTReader
+
+
+class ChatGPTRAG(BaseRAGExample):
+    """RAG example for ChatGPT conversation data."""
+
+    def __init__(self):
+        # Set default values BEFORE calling super().__init__
+        self.max_items_default = -1  # Process all conversations by default
+        self.embedding_model_default = (
+            "sentence-transformers/all-MiniLM-L6-v2"  # Fast 384-dim model
+        )
+
+        super().__init__(
+            name="ChatGPT",
+            description="Process and query ChatGPT conversation exports with LEANN",
+            default_index_name="chatgpt_conversations_index",
+        )
+
+    def _add_specific_arguments(self, parser):
+        """Add ChatGPT-specific arguments."""
+        chatgpt_group = parser.add_argument_group("ChatGPT Parameters")
+        chatgpt_group.add_argument(
+            "--export-path",
+            type=str,
+            default="./chatgpt_export",
+            help="Path to ChatGPT export file (.zip or .html) or directory containing exports (default: ./chatgpt_export)",
+        )
+        chatgpt_group.add_argument(
+            "--concatenate-conversations",
+            action="store_true",
+            default=True,
+            help="Concatenate messages within conversations for better context (default: True)",
+        )
+        chatgpt_group.add_argument(
+            "--separate-messages",
+            action="store_true",
+            help="Process each message as a separate document (overrides --concatenate-conversations)",
+        )
+        chatgpt_group.add_argument(
+            "--chunk-size", type=int, default=512, help="Text chunk size (default: 512)"
+        )
+        chatgpt_group.add_argument(
+            "--chunk-overlap", type=int, default=128, help="Text chunk overlap (default: 128)"
+        )
+
+    def _find_chatgpt_exports(self, export_path: Path) -> list[Path]:
+        """
+        Find ChatGPT export files in the given path.
+
+        Args:
+            export_path: Path to search for exports
+
+        Returns:
+            List of paths to ChatGPT export files
+        """
+        export_files = []
+
+        if export_path.is_file():
+            if export_path.suffix.lower() in [".zip", ".html"]:
+                export_files.append(export_path)
+        elif export_path.is_dir():
+            # Look for zip and html files
+            export_files.extend(export_path.glob("*.zip"))
+            export_files.extend(export_path.glob("*.html"))
+
+        return export_files
+
+    async def load_data(self, args) -> list[str]:
+        """Load ChatGPT export data and convert to text chunks."""
+        export_path = Path(args.export_path)
+
+        if not export_path.exists():
+            print(f"ChatGPT export path not found: {export_path}")
+            print(
+                "Please ensure you have exported your ChatGPT data and placed it in the correct location."
+            )
+            print("\nTo export your ChatGPT data:")
+            print("1. Sign in to ChatGPT")
+            print("2. Click on your profile icon → Settings → Data Controls")
+            print("3. Click 'Export' under Export Data")
+            print("4. Download the zip file from the email link")
+            print("5. Extract or place the file/directory at the specified path")
+            return []
+
+        # Find export files
+        export_files = self._find_chatgpt_exports(export_path)
+
+        if not export_files:
+            print(f"No ChatGPT export files (.zip or .html) found in: {export_path}")
+            return []
+
+        print(f"Found {len(export_files)} ChatGPT export files")
+
+        # Create reader with appropriate settings
+        concatenate = args.concatenate_conversations and not args.separate_messages
+        reader = ChatGPTReader(concatenate_conversations=concatenate)
+
+        # Process each export file
+        all_documents = []
+        total_processed = 0
+
+        for i, export_file in enumerate(export_files):
+            print(f"\nProcessing export file {i + 1}/{len(export_files)}: {export_file.name}")
+
+            try:
+                # Apply max_items limit per file
+                max_per_file = -1
+                if args.max_items > 0:
+                    remaining = args.max_items - total_processed
+                    if remaining <= 0:
+                        break
+                    max_per_file = remaining
+
+                # Load conversations
+                documents = reader.load_data(
+                    chatgpt_export_path=str(export_file),
+                    max_count=max_per_file,
+                    include_metadata=True,
+                )
+
+                if documents:
+                    all_documents.extend(documents)
+                    total_processed += len(documents)
+                    print(f"Processed {len(documents)} conversations from this file")
+                else:
+                    print(f"No conversations loaded from {export_file}")
+
+            except Exception as e:
+                print(f"Error processing {export_file}: {e}")
+                continue
+
+        if not all_documents:
+            print("No conversations found to process!")
+            print("\nTroubleshooting:")
+            print("- Ensure the export file is a valid ChatGPT export")
+            print("- Check that the HTML file contains conversation data")
+            print("- Try extracting the zip file and pointing to the HTML file directly")
+            return []
+
+        print(f"\nTotal conversations processed: {len(all_documents)}")
+        print("Now starting to split into text chunks... this may take some time")
+
+        # Convert to text chunks
+        all_texts = create_text_chunks(
+            all_documents, chunk_size=args.chunk_size, chunk_overlap=args.chunk_overlap
+        )
+
+        print(f"Created {len(all_texts)} text chunks from {len(all_documents)} conversations")
+        return all_texts
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    # Example queries for ChatGPT RAG
+    print("\n🤖 ChatGPT RAG Example")
+    print("=" * 50)
+    print("\nExample queries you can try:")
+    print("- 'What did I ask about Python programming?'")
+    print("- 'Show me conversations about machine learning'")
+    print("- 'Find discussions about travel planning'")
+    print("- 'What advice did ChatGPT give me about career development?'")
+    print("- 'Search for conversations about cooking recipes'")
+    print("\nTo get started:")
+    print("1. Export your ChatGPT data from Settings → Data Controls → Export")
+    print("2. Place the downloaded zip file or extracted HTML in ./chatgpt_export/")
+    print("3. Run this script to build your personal ChatGPT knowledge base!")
+    print("\nOr run without --query for interactive mode\n")
+
+    rag = ChatGPTRAG()
+    asyncio.run(rag.run())

apps/chunking/__init__.py

@@ -0,0 +1,44 @@
+"""Unified chunking utilities facade.
+
+This module re-exports the packaged utilities from `leann.chunking_utils` so
+that both repo apps (importing `chunking`) and installed wheels share one
+single implementation. When running from the repo without installation, it
+adds the `packages/leann-core/src` directory to `sys.path` as a fallback.
+"""
+
+import sys
+from pathlib import Path
+
+try:
+    from leann.chunking_utils import (
+        CODE_EXTENSIONS,
+        create_ast_chunks,
+        create_text_chunks,
+        create_traditional_chunks,
+        detect_code_files,
+        get_language_from_extension,
+    )
+except Exception:  # pragma: no cover - best-effort fallback for dev environment
+    repo_root = Path(__file__).resolve().parents[2]
+    leann_src = repo_root / "packages" / "leann-core" / "src"
+    if leann_src.exists():
+        sys.path.insert(0, str(leann_src))
+        from leann.chunking_utils import (
+            CODE_EXTENSIONS,
+            create_ast_chunks,
+            create_text_chunks,
+            create_traditional_chunks,
+            detect_code_files,
+            get_language_from_extension,
+        )
+    else:
+        raise
+
+__all__ = [
+    "CODE_EXTENSIONS",
+    "create_ast_chunks",
+    "create_text_chunks",
+    "create_traditional_chunks",
+    "detect_code_files",
+    "get_language_from_extension",
+]

apps/claude_data/claude_reader.py

@@ -0,0 +1,420 @@
+"""
+Claude export data reader.
+
+Reads and processes Claude conversation data from exported JSON files.
+"""
+
+import json
+from pathlib import Path
+from typing import Any
+from zipfile import ZipFile
+
+from llama_index.core import Document
+from llama_index.core.readers.base import BaseReader
+
+
+class ClaudeReader(BaseReader):
+    """
+    Claude export data reader.
+
+    Reads Claude conversation data from exported JSON files or zip archives.
+    Processes conversations into structured documents with metadata.
+    """
+
+    def __init__(self, concatenate_conversations: bool = True) -> None:
+        """
+        Initialize.
+
+        Args:
+            concatenate_conversations: Whether to concatenate messages within conversations for better context
+        """
+        self.concatenate_conversations = concatenate_conversations
+
+    def _extract_json_from_zip(self, zip_path: Path) -> list[str]:
+        """
+        Extract JSON files from Claude export zip file.
+
+        Args:
+            zip_path: Path to the Claude export zip file
+
+        Returns:
+            List of JSON content strings, or empty list if not found
+        """
+        json_contents = []
+        try:
+            with ZipFile(zip_path, "r") as zip_file:
+                # Look for JSON files
+                json_files = [f for f in zip_file.namelist() if f.endswith(".json")]
+
+                if not json_files:
+                    print(f"No JSON files found in {zip_path}")
+                    return []
+
+                print(f"Found {len(json_files)} JSON files in archive")
+
+                for json_file in json_files:
+                    with zip_file.open(json_file) as f:
+                        content = f.read().decode("utf-8", errors="ignore")
+                        json_contents.append(content)
+
+        except Exception as e:
+            print(f"Error extracting JSON from zip {zip_path}: {e}")
+
+        return json_contents
+
+    def _parse_claude_json(self, json_content: str) -> list[dict]:
+        """
+        Parse Claude JSON export to extract conversations.
+
+        Args:
+            json_content: JSON content from Claude export
+
+        Returns:
+            List of conversation dictionaries
+        """
+        try:
+            data = json.loads(json_content)
+        except json.JSONDecodeError as e:
+            print(f"Error parsing JSON: {e}")
+            return []
+
+        conversations = []
+
+        # Handle different possible JSON structures
+        if isinstance(data, list):
+            # If data is a list of conversations
+            for item in data:
+                conversation = self._extract_conversation_from_json(item)
+                if conversation:
+                    conversations.append(conversation)
+        elif isinstance(data, dict):
+            # Check for common structures
+            if "conversations" in data:
+                # Structure: {"conversations": [...]}
+                for item in data["conversations"]:
+                    conversation = self._extract_conversation_from_json(item)
+                    if conversation:
+                        conversations.append(conversation)
+            elif "messages" in data:
+                # Single conversation with messages
+                conversation = self._extract_conversation_from_json(data)
+                if conversation:
+                    conversations.append(conversation)
+            else:
+                # Try to treat the whole object as a conversation
+                conversation = self._extract_conversation_from_json(data)
+                if conversation:
+                    conversations.append(conversation)
+
+        return conversations
+
+    def _extract_conversation_from_json(self, conv_data: dict) -> dict | None:
+        """
+        Extract conversation data from a JSON object.
+
+        Args:
+            conv_data: Dictionary containing conversation data
+
+        Returns:
+            Dictionary with conversation data or None
+        """
+        if not isinstance(conv_data, dict):
+            return None
+
+        messages = []
+
+        # Look for messages in various possible structures
+        message_sources = []
+        if "messages" in conv_data:
+            message_sources = conv_data["messages"]
+        elif "chat" in conv_data:
+            message_sources = conv_data["chat"]
+        elif "conversation" in conv_data:
+            message_sources = conv_data["conversation"]
+        else:
+            # If no clear message structure, try to extract from the object itself
+            if "content" in conv_data and "role" in conv_data:
+                message_sources = [conv_data]
+
+        for msg_data in message_sources:
+            message = self._extract_message_from_json(msg_data)
+            if message:
+                messages.append(message)
+
+        if not messages:
+            return None
+
+        # Extract conversation metadata
+        title = self._extract_title_from_conversation(conv_data, messages)
+        timestamp = self._extract_timestamp_from_conversation(conv_data)
+
+        return {"title": title, "messages": messages, "timestamp": timestamp}
+
+    def _extract_message_from_json(self, msg_data: dict) -> dict | None:
+        """
+        Extract message data from a JSON message object.
+
+        Args:
+            msg_data: Dictionary containing message data
+
+        Returns:
+            Dictionary with message data or None
+        """
+        if not isinstance(msg_data, dict):
+            return None
+
+        # Extract content from various possible fields
+        content = ""
+        content_fields = ["content", "text", "message", "body"]
+        for field in content_fields:
+            if msg_data.get(field):
+                content = str(msg_data[field])
+                break
+
+        if not content or len(content.strip()) < 3:
+            return None
+
+        # Extract role (user/assistant/human/ai/claude)
+        role = "mixed"  # Default role
+        role_fields = ["role", "sender", "from", "author", "type"]
+        for field in role_fields:
+            if msg_data.get(field):
+                role_value = str(msg_data[field]).lower()
+                if role_value in ["user", "human", "person"]:
+                    role = "user"
+                elif role_value in ["assistant", "ai", "claude", "bot"]:
+                    role = "assistant"
+                break
+
+        # Extract timestamp
+        timestamp = self._extract_timestamp_from_message(msg_data)
+
+        return {"role": role, "content": content, "timestamp": timestamp}
+
+    def _extract_timestamp_from_message(self, msg_data: dict) -> str | None:
+        """Extract timestamp from message data."""
+        timestamp_fields = ["timestamp", "created_at", "date", "time"]
+        for field in timestamp_fields:
+            if msg_data.get(field):
+                return str(msg_data[field])
+        return None
+
+    def _extract_timestamp_from_conversation(self, conv_data: dict) -> str | None:
+        """Extract timestamp from conversation data."""
+        timestamp_fields = ["timestamp", "created_at", "date", "updated_at", "last_updated"]
+        for field in timestamp_fields:
+            if conv_data.get(field):
+                return str(conv_data[field])
+        return None
+
+    def _extract_title_from_conversation(self, conv_data: dict, messages: list) -> str:
+        """Extract or generate title for conversation."""
+        # Try to find explicit title
+        title_fields = ["title", "name", "subject", "topic"]
+        for field in title_fields:
+            if conv_data.get(field):
+                return str(conv_data[field])
+
+        # Generate title from first user message
+        for message in messages:
+            if message.get("role") == "user":
+                content = message.get("content", "")
+                if content:
+                    # Use first 50 characters as title
+                    title = content[:50].strip()
+                    if len(content) > 50:
+                        title += "..."
+                    return title
+
+        return "Claude Conversation"
+
+    def _create_concatenated_content(self, conversation: dict) -> str:
+        """
+        Create concatenated content from conversation messages.
+
+        Args:
+            conversation: Dictionary containing conversation data
+
+        Returns:
+            Formatted concatenated content
+        """
+        title = conversation.get("title", "Claude Conversation")
+        messages = conversation.get("messages", [])
+        timestamp = conversation.get("timestamp", "Unknown")
+
+        # Build message content
+        message_parts = []
+        for message in messages:
+            role = message.get("role", "mixed")
+            content = message.get("content", "")
+            msg_timestamp = message.get("timestamp", "")
+
+            if role == "user":
+                prefix = "[You]"
+            elif role == "assistant":
+                prefix = "[Claude]"
+            else:
+                prefix = "[Message]"
+
+            # Add timestamp if available
+            if msg_timestamp:
+                prefix += f" ({msg_timestamp})"
+
+            message_parts.append(f"{prefix}: {content}")
+
+        concatenated_text = "\n\n".join(message_parts)
+
+        # Create final document content
+        doc_content = f"""Conversation: {title}
+Date: {timestamp}
+Messages ({len(messages)} messages):
+
+{concatenated_text}
+"""
+        return doc_content
+
+    def load_data(self, input_dir: str | None = None, **load_kwargs: Any) -> list[Document]:
+        """
+        Load Claude export data.
+
+        Args:
+            input_dir: Directory containing Claude export files or path to specific file
+            **load_kwargs:
+                max_count (int): Maximum number of conversations to process
+                claude_export_path (str): Specific path to Claude export file/directory
+                include_metadata (bool): Whether to include metadata in documents
+        """
+        docs: list[Document] = []
+        max_count = load_kwargs.get("max_count", -1)
+        claude_export_path = load_kwargs.get("claude_export_path", input_dir)
+        include_metadata = load_kwargs.get("include_metadata", True)
+
+        if not claude_export_path:
+            print("No Claude export path provided")
+            return docs
+
+        export_path = Path(claude_export_path)
+
+        if not export_path.exists():
+            print(f"Claude export path not found: {export_path}")
+            return docs
+
+        json_contents = []
+
+        # Handle different input types
+        if export_path.is_file():
+            if export_path.suffix.lower() == ".zip":
+                # Extract JSON from zip file
+                json_contents = self._extract_json_from_zip(export_path)
+            elif export_path.suffix.lower() == ".json":
+                # Read JSON file directly
+                try:
+                    with open(export_path, encoding="utf-8", errors="ignore") as f:
+                        json_contents.append(f.read())
+                except Exception as e:
+                    print(f"Error reading JSON file {export_path}: {e}")
+                    return docs
+            else:
+                print(f"Unsupported file type: {export_path.suffix}")
+                return docs
+
+        elif export_path.is_dir():
+            # Look for JSON files in directory
+            json_files = list(export_path.glob("*.json"))
+            zip_files = list(export_path.glob("*.zip"))
+
+            if json_files:
+                print(f"Found {len(json_files)} JSON files in directory")
+                for json_file in json_files:
+                    try:
+                        with open(json_file, encoding="utf-8", errors="ignore") as f:
+                            json_contents.append(f.read())
+                    except Exception as e:
+                        print(f"Error reading JSON file {json_file}: {e}")
+                        continue
+
+            if zip_files:
+                print(f"Found {len(zip_files)} ZIP files in directory")
+                for zip_file in zip_files:
+                    zip_contents = self._extract_json_from_zip(zip_file)
+                    json_contents.extend(zip_contents)
+
+            if not json_files and not zip_files:
+                print(f"No JSON or ZIP files found in {export_path}")
+                return docs
+
+        if not json_contents:
+            print("No JSON content found to process")
+            return docs
+
+        # Parse conversations from JSON content
+        print("Parsing Claude conversations from JSON...")
+        all_conversations = []
+        for json_content in json_contents:
+            conversations = self._parse_claude_json(json_content)
+            all_conversations.extend(conversations)
+
+        if not all_conversations:
+            print("No conversations found in JSON content")
+            return docs
+
+        print(f"Found {len(all_conversations)} conversations")
+
+        # Process conversations into documents
+        count = 0
+        for conversation in all_conversations:
+            if max_count > 0 and count >= max_count:
+                break
+
+            if self.concatenate_conversations:
+                # Create one document per conversation with concatenated messages
+                doc_content = self._create_concatenated_content(conversation)
+
+                metadata = {}
+                if include_metadata:
+                    metadata = {
+                        "title": conversation.get("title", "Claude Conversation"),
+                        "timestamp": conversation.get("timestamp", "Unknown"),
+                        "message_count": len(conversation.get("messages", [])),
+                        "source": "Claude Export",
+                    }
+
+                doc = Document(text=doc_content, metadata=metadata)
+                docs.append(doc)
+                count += 1
+
+            else:
+                # Create separate documents for each message
+                for message in conversation.get("messages", []):
+                    if max_count > 0 and count >= max_count:
+                        break
+
+                    role = message.get("role", "mixed")
+                    content = message.get("content", "")
+                    msg_timestamp = message.get("timestamp", "")
+
+                    if not content.strip():
+                        continue
+
+                    # Create document content with context
+                    doc_content = f"""Conversation: {conversation.get("title", "Claude Conversation")}
+Role: {role}
+Timestamp: {msg_timestamp or conversation.get("timestamp", "Unknown")}
+Message: {content}
+"""
+
+                    metadata = {}
+                    if include_metadata:
+                        metadata = {
+                            "conversation_title": conversation.get("title", "Claude Conversation"),
+                            "role": role,
+                            "timestamp": msg_timestamp or conversation.get("timestamp", "Unknown"),
+                            "source": "Claude Export",
+                        }
+
+                    doc = Document(text=doc_content, metadata=metadata)
+                    docs.append(doc)
+                    count += 1
+
+        print(f"Created {len(docs)} documents from Claude export")
+        return docs

apps/claude_rag.py

@@ -0,0 +1,189 @@
+"""
+Claude RAG example using the unified interface.
+Supports Claude export data from JSON files.
+"""
+
+import sys
+from pathlib import Path
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent))
+
+from base_rag_example import BaseRAGExample
+from chunking import create_text_chunks
+
+from .claude_data.claude_reader import ClaudeReader
+
+
+class ClaudeRAG(BaseRAGExample):
+    """RAG example for Claude conversation data."""
+
+    def __init__(self):
+        # Set default values BEFORE calling super().__init__
+        self.max_items_default = -1  # Process all conversations by default
+        self.embedding_model_default = (
+            "sentence-transformers/all-MiniLM-L6-v2"  # Fast 384-dim model
+        )
+
+        super().__init__(
+            name="Claude",
+            description="Process and query Claude conversation exports with LEANN",
+            default_index_name="claude_conversations_index",
+        )
+
+    def _add_specific_arguments(self, parser):
+        """Add Claude-specific arguments."""
+        claude_group = parser.add_argument_group("Claude Parameters")
+        claude_group.add_argument(
+            "--export-path",
+            type=str,
+            default="./claude_export",
+            help="Path to Claude export file (.json or .zip) or directory containing exports (default: ./claude_export)",
+        )
+        claude_group.add_argument(
+            "--concatenate-conversations",
+            action="store_true",
+            default=True,
+            help="Concatenate messages within conversations for better context (default: True)",
+        )
+        claude_group.add_argument(
+            "--separate-messages",
+            action="store_true",
+            help="Process each message as a separate document (overrides --concatenate-conversations)",
+        )
+        claude_group.add_argument(
+            "--chunk-size", type=int, default=512, help="Text chunk size (default: 512)"
+        )
+        claude_group.add_argument(
+            "--chunk-overlap", type=int, default=128, help="Text chunk overlap (default: 128)"
+        )
+
+    def _find_claude_exports(self, export_path: Path) -> list[Path]:
+        """
+        Find Claude export files in the given path.
+
+        Args:
+            export_path: Path to search for exports
+
+        Returns:
+            List of paths to Claude export files
+        """
+        export_files = []
+
+        if export_path.is_file():
+            if export_path.suffix.lower() in [".zip", ".json"]:
+                export_files.append(export_path)
+        elif export_path.is_dir():
+            # Look for zip and json files
+            export_files.extend(export_path.glob("*.zip"))
+            export_files.extend(export_path.glob("*.json"))
+
+        return export_files
+
+    async def load_data(self, args) -> list[str]:
+        """Load Claude export data and convert to text chunks."""
+        export_path = Path(args.export_path)
+
+        if not export_path.exists():
+            print(f"Claude export path not found: {export_path}")
+            print(
+                "Please ensure you have exported your Claude data and placed it in the correct location."
+            )
+            print("\nTo export your Claude data:")
+            print("1. Open Claude in your browser")
+            print("2. Look for export/download options in settings or conversation menu")
+            print("3. Download the conversation data (usually in JSON format)")
+            print("4. Place the file/directory at the specified path")
+            print(
+                "\nNote: Claude export methods may vary. Check Claude's help documentation for current instructions."
+            )
+            return []
+
+        # Find export files
+        export_files = self._find_claude_exports(export_path)
+
+        if not export_files:
+            print(f"No Claude export files (.json or .zip) found in: {export_path}")
+            return []
+
+        print(f"Found {len(export_files)} Claude export files")
+
+        # Create reader with appropriate settings
+        concatenate = args.concatenate_conversations and not args.separate_messages
+        reader = ClaudeReader(concatenate_conversations=concatenate)
+
+        # Process each export file
+        all_documents = []
+        total_processed = 0
+
+        for i, export_file in enumerate(export_files):
+            print(f"\nProcessing export file {i + 1}/{len(export_files)}: {export_file.name}")
+
+            try:
+                # Apply max_items limit per file
+                max_per_file = -1
+                if args.max_items > 0:
+                    remaining = args.max_items - total_processed
+                    if remaining <= 0:
+                        break
+                    max_per_file = remaining
+
+                # Load conversations
+                documents = reader.load_data(
+                    claude_export_path=str(export_file),
+                    max_count=max_per_file,
+                    include_metadata=True,
+                )
+
+                if documents:
+                    all_documents.extend(documents)
+                    total_processed += len(documents)
+                    print(f"Processed {len(documents)} conversations from this file")
+                else:
+                    print(f"No conversations loaded from {export_file}")
+
+            except Exception as e:
+                print(f"Error processing {export_file}: {e}")
+                continue
+
+        if not all_documents:
+            print("No conversations found to process!")
+            print("\nTroubleshooting:")
+            print("- Ensure the export file is a valid Claude export")
+            print("- Check that the JSON file contains conversation data")
+            print("- Try using a different export format or method")
+            print("- Check Claude's documentation for current export procedures")
+            return []
+
+        print(f"\nTotal conversations processed: {len(all_documents)}")
+        print("Now starting to split into text chunks... this may take some time")
+
+        # Convert to text chunks
+        all_texts = create_text_chunks(
+            all_documents, chunk_size=args.chunk_size, chunk_overlap=args.chunk_overlap
+        )
+
+        print(f"Created {len(all_texts)} text chunks from {len(all_documents)} conversations")
+        return all_texts
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    # Example queries for Claude RAG
+    print("\n🤖 Claude RAG Example")
+    print("=" * 50)
+    print("\nExample queries you can try:")
+    print("- 'What did I ask Claude about Python programming?'")
+    print("- 'Show me conversations about machine learning'")
+    print("- 'Find discussions about code optimization'")
+    print("- 'What advice did Claude give me about software design?'")
+    print("- 'Search for conversations about debugging techniques'")
+    print("\nTo get started:")
+    print("1. Export your Claude conversation data")
+    print("2. Place the JSON/ZIP file in ./claude_export/")
+    print("3. Run this script to build your personal Claude knowledge base!")
+    print("\nOr run without --query for interactive mode\n")
+
+    rag = ClaudeRAG()
+    asyncio.run(rag.run())

apps/code_rag.py

@@ -0,0 +1,211 @@
+"""
+Code RAG example using AST-aware chunking for optimal code understanding.
+Specialized for code repositories with automatic language detection and
+optimized chunking parameters.
+"""
+
+import sys
+from pathlib import Path
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent))
+
+from base_rag_example import BaseRAGExample
+from chunking import CODE_EXTENSIONS, create_text_chunks
+from llama_index.core import SimpleDirectoryReader
+
+
+class CodeRAG(BaseRAGExample):
+    """Specialized RAG example for code repositories with AST-aware chunking."""
+
+    def __init__(self):
+        super().__init__(
+            name="Code",
+            description="Process and query code repositories with AST-aware chunking",
+            default_index_name="code_index",
+        )
+        # Override defaults for code-specific usage
+        self.embedding_model_default = "facebook/contriever"  # Good for code
+        self.max_items_default = -1  # Process all code files by default
+
+    def _add_specific_arguments(self, parser):
+        """Add code-specific arguments."""
+        code_group = parser.add_argument_group("Code Repository Parameters")
+
+        code_group.add_argument(
+            "--repo-dir",
+            type=str,
+            default=".",
+            help="Code repository directory to index (default: current directory)",
+        )
+        code_group.add_argument(
+            "--include-extensions",
+            nargs="+",
+            default=list(CODE_EXTENSIONS.keys()),
+            help="File extensions to include (default: supported code extensions)",
+        )
+        code_group.add_argument(
+            "--exclude-dirs",
+            nargs="+",
+            default=[
+                ".git",
+                "__pycache__",
+                "node_modules",
+                "venv",
+                ".venv",
+                "build",
+                "dist",
+                "target",
+            ],
+            help="Directories to exclude from indexing",
+        )
+        code_group.add_argument(
+            "--max-file-size",
+            type=int,
+            default=1000000,  # 1MB
+            help="Maximum file size in bytes to process (default: 1MB)",
+        )
+        code_group.add_argument(
+            "--include-comments",
+            action="store_true",
+            help="Include comments in chunking (useful for documentation)",
+        )
+        code_group.add_argument(
+            "--preserve-imports",
+            action="store_true",
+            default=True,
+            help="Try to preserve import statements in chunks (default: True)",
+        )
+
+    async def load_data(self, args) -> list[str]:
+        """Load code files and convert to AST-aware chunks."""
+        print(f"🔍 Scanning code repository: {args.repo_dir}")
+        print(f"📁 Including extensions: {args.include_extensions}")
+        print(f"🚫 Excluding directories: {args.exclude_dirs}")
+
+        # Check if repository directory exists
+        repo_path = Path(args.repo_dir)
+        if not repo_path.exists():
+            raise ValueError(f"Repository directory not found: {args.repo_dir}")
+
+        # Load code files with filtering
+        reader_kwargs = {
+            "recursive": True,
+            "encoding": "utf-8",
+            "required_exts": args.include_extensions,
+            "exclude_hidden": True,
+        }
+
+        # Create exclusion filter
+        def file_filter(file_path: str) -> bool:
+            """Filter out unwanted files and directories."""
+            path = Path(file_path)
+
+            # Check file size
+            try:
+                if path.stat().st_size > args.max_file_size:
+                    print(f"⚠️ Skipping large file: {path.name} ({path.stat().st_size} bytes)")
+                    return False
+            except Exception:
+                return False
+
+            # Check if in excluded directory
+            for exclude_dir in args.exclude_dirs:
+                if exclude_dir in path.parts:
+                    return False
+
+            return True
+
+        try:
+            # Load documents with file filtering
+            documents = SimpleDirectoryReader(
+                args.repo_dir,
+                file_extractor=None,  # Use default extractors
+                **reader_kwargs,
+            ).load_data(show_progress=True)
+
+            # Apply custom filtering
+            filtered_docs = []
+            for doc in documents:
+                file_path = doc.metadata.get("file_path", "")
+                if file_filter(file_path):
+                    filtered_docs.append(doc)
+
+            documents = filtered_docs
+
+        except Exception as e:
+            print(f"❌ Error loading code files: {e}")
+            return []
+
+        if not documents:
+            print(
+                f"❌ No code files found in {args.repo_dir} with extensions {args.include_extensions}"
+            )
+            return []
+
+        print(f"✅ Loaded {len(documents)} code files")
+
+        # Show breakdown by language/extension
+        ext_counts = {}
+        for doc in documents:
+            file_path = doc.metadata.get("file_path", "")
+            if file_path:
+                ext = Path(file_path).suffix.lower()
+                ext_counts[ext] = ext_counts.get(ext, 0) + 1
+
+        print("📊 Files by extension:")
+        for ext, count in sorted(ext_counts.items()):
+            print(f"   {ext}: {count} files")
+
+        # Use AST-aware chunking by default for code
+        print(
+            f"🧠 Using AST-aware chunking (chunk_size: {args.ast_chunk_size}, overlap: {args.ast_chunk_overlap})"
+        )
+
+        all_texts = create_text_chunks(
+            documents,
+            chunk_size=256,  # Fallback for non-code files
+            chunk_overlap=64,
+            use_ast_chunking=True,  # Always use AST for code RAG
+            ast_chunk_size=args.ast_chunk_size,
+            ast_chunk_overlap=args.ast_chunk_overlap,
+            code_file_extensions=args.include_extensions,
+            ast_fallback_traditional=True,
+        )
+
+        # Apply max_items limit if specified
+        if args.max_items > 0 and len(all_texts) > args.max_items:
+            print(f"⏳ Limiting to {args.max_items} chunks (from {len(all_texts)})")
+            all_texts = all_texts[: args.max_items]
+
+        print(f"✅ Generated {len(all_texts)} code chunks")
+        return all_texts
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    # Example queries for code RAG
+    print("\n💻 Code RAG Example")
+    print("=" * 50)
+    print("\nExample queries you can try:")
+    print("- 'How does the embedding computation work?'")
+    print("- 'What are the main classes in this codebase?'")
+    print("- 'Show me the search implementation'")
+    print("- 'How is error handling implemented?'")
+    print("- 'What design patterns are used?'")
+    print("- 'Explain the chunking logic'")
+    print("\n🚀 Features:")
+    print("- ✅ AST-aware chunking preserves code structure")
+    print("- ✅ Automatic language detection")
+    print("- ✅ Smart filtering of large files and common excludes")
+    print("- ✅ Optimized for code understanding")
+    print("\nUsage examples:")
+    print("  python -m apps.code_rag --repo-dir ./my_project")
+    print(
+        "  python -m apps.code_rag --include-extensions .py .js --query 'How does authentication work?'"
+    )
+    print("\nOr run without --query for interactive mode\n")
+
+    rag = CodeRAG()
+    asyncio.run(rag.run())

apps/document_rag.py

@@ -9,7 +9,8 @@ from pathlib import Path
 # Add parent directory to path for imports
 sys.path.insert(0, str(Path(__file__).parent))
 
-from base_rag_example import BaseRAGExample, create_text_chunks
+from base_rag_example import BaseRAGExample
+from chunking import create_text_chunks
 from llama_index.core import SimpleDirectoryReader
 
 
@@ -44,6 +45,11 @@ class DocumentRAG(BaseRAGExample):
         doc_group.add_argument(
             "--chunk-overlap", type=int, default=128, help="Text chunk overlap (default: 128)"
         )
+        doc_group.add_argument(
+            "--enable-code-chunking",
+            action="store_true",
+            help="Enable AST-aware chunking for code files in the data directory",
+        )
 
     async def load_data(self, args) -> list[str]:
         """Load documents and convert to text chunks."""
@@ -76,9 +82,22 @@ class DocumentRAG(BaseRAGExample):
 
         print(f"Loaded {len(documents)} documents")
 
-        # Convert to text chunks
+        # Determine chunking strategy
+        use_ast = args.enable_code_chunking or getattr(args, "use_ast_chunking", False)
+
+        if use_ast:
+            print("Using AST-aware chunking for code files")
+
+        # Convert to text chunks with optional AST support
         all_texts = create_text_chunks(
-            documents, chunk_size=args.chunk_size, chunk_overlap=args.chunk_overlap
+            documents,
+            chunk_size=args.chunk_size,
+            chunk_overlap=args.chunk_overlap,
+            use_ast_chunking=use_ast,
+            ast_chunk_size=getattr(args, "ast_chunk_size", 512),
+            ast_chunk_overlap=getattr(args, "ast_chunk_overlap", 64),
+            code_file_extensions=getattr(args, "code_file_extensions", None),
+            ast_fallback_traditional=getattr(args, "ast_fallback_traditional", True),
         )
 
         # Apply max_items limit if specified
@@ -102,6 +121,10 @@ if __name__ == "__main__":
     print(
         "- 'What is the problem of developing pan gu model Huawei meets? (盘古大模型开发中遇到什么问题?)'"
     )
+    print("\n🚀 NEW: Code-aware chunking available!")
+    print("- Use --enable-code-chunking to enable AST-aware chunking for code files")
+    print("- Supports Python, Java, C#, TypeScript files")
+    print("- Better semantic understanding of code structure")
     print("\nOr run without --query for interactive mode\n")
 
     rag = DocumentRAG()

apps/email_rag.py

@@ -9,7 +9,8 @@ from pathlib import Path
 # Add parent directory to path for imports
 sys.path.insert(0, str(Path(__file__).parent))
 
-from base_rag_example import BaseRAGExample, create_text_chunks
+from base_rag_example import BaseRAGExample
+from chunking import create_text_chunks
 
 from .email_data.LEANN_email_reader import EmlxReader
 

apps/history_data/history.py

@@ -74,7 +74,7 @@ class ChromeHistoryReader(BaseReader):
                 if count >= max_count and max_count > 0:
                     break
 
-                last_visit, url, title, visit_count, typed_count, hidden = row
+                last_visit, url, title, visit_count, typed_count, _hidden = row
 
                 # Create document content with metadata embedded in text
                 doc_content = f"""

apps/imessage_data/__init__.py

@@ -0,0 +1 @@
+"""iMessage data processing module."""

apps/imessage_data/imessage_reader.py

@@ -0,0 +1,342 @@
+"""
+iMessage data reader.
+
+Reads and processes iMessage conversation data from the macOS Messages database.
+"""
+
+import sqlite3
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+from llama_index.core import Document
+from llama_index.core.readers.base import BaseReader
+
+
+class IMessageReader(BaseReader):
+    """
+    iMessage data reader.
+
+    Reads iMessage conversation data from the macOS Messages database (chat.db).
+    Processes conversations into structured documents with metadata.
+    """
+
+    def __init__(self, concatenate_conversations: bool = True) -> None:
+        """
+        Initialize.
+
+        Args:
+            concatenate_conversations: Whether to concatenate messages within conversations for better context
+        """
+        self.concatenate_conversations = concatenate_conversations
+
+    def _get_default_chat_db_path(self) -> Path:
+        """
+        Get the default path to the iMessage chat database.
+
+        Returns:
+            Path to the chat.db file
+        """
+        home = Path.home()
+        return home / "Library" / "Messages" / "chat.db"
+
+    def _convert_cocoa_timestamp(self, cocoa_timestamp: int) -> str:
+        """
+        Convert Cocoa timestamp to readable format.
+
+        Args:
+            cocoa_timestamp: Timestamp in Cocoa format (nanoseconds since 2001-01-01)
+
+        Returns:
+            Formatted timestamp string
+        """
+        if cocoa_timestamp == 0:
+            return "Unknown"
+
+        try:
+            # Cocoa timestamp is nanoseconds since 2001-01-01 00:00:00 UTC
+            # Convert to seconds and add to Unix epoch
+            cocoa_epoch = datetime(2001, 1, 1)
+            unix_timestamp = cocoa_timestamp / 1_000_000_000  # Convert nanoseconds to seconds
+            message_time = cocoa_epoch.timestamp() + unix_timestamp
+            return datetime.fromtimestamp(message_time).strftime("%Y-%m-%d %H:%M:%S")
+        except (ValueError, OSError):
+            return "Unknown"
+
+    def _get_contact_name(self, handle_id: str) -> str:
+        """
+        Get a readable contact name from handle ID.
+
+        Args:
+            handle_id: The handle ID (phone number or email)
+
+        Returns:
+            Formatted contact name
+        """
+        if not handle_id:
+            return "Unknown"
+
+        # Clean up phone numbers and emails for display
+        if "@" in handle_id:
+            return handle_id  # Email address
+        elif handle_id.startswith("+"):
+            return handle_id  # International phone number
+        else:
+            # Try to format as phone number
+            digits = "".join(filter(str.isdigit, handle_id))
+            if len(digits) == 10:
+                return f"({digits[:3]}) {digits[3:6]}-{digits[6:]}"
+            elif len(digits) == 11 and digits[0] == "1":
+                return f"+1 ({digits[1:4]}) {digits[4:7]}-{digits[7:]}"
+            else:
+                return handle_id
+
+    def _read_messages_from_db(self, db_path: Path) -> list[dict]:
+        """
+        Read messages from the iMessage database.
+
+        Args:
+            db_path: Path to the chat.db file
+
+        Returns:
+            List of message dictionaries
+        """
+        if not db_path.exists():
+            print(f"iMessage database not found at: {db_path}")
+            return []
+
+        try:
+            # Connect to the database
+            conn = sqlite3.connect(str(db_path))
+            cursor = conn.cursor()
+
+            # Query to get messages with chat and handle information
+            query = """
+            SELECT
+                m.ROWID as message_id,
+                m.text,
+                m.date,
+                m.is_from_me,
+                m.service,
+                c.chat_identifier,
+                c.display_name as chat_display_name,
+                h.id as handle_id,
+                c.ROWID as chat_id
+            FROM message m
+            LEFT JOIN chat_message_join cmj ON m.ROWID = cmj.message_id
+            LEFT JOIN chat c ON cmj.chat_id = c.ROWID
+            LEFT JOIN handle h ON m.handle_id = h.ROWID
+            WHERE m.text IS NOT NULL AND m.text != ''
+            ORDER BY c.ROWID, m.date
+            """
+
+            cursor.execute(query)
+            rows = cursor.fetchall()
+
+            messages = []
+            for row in rows:
+                (
+                    message_id,
+                    text,
+                    date,
+                    is_from_me,
+                    service,
+                    chat_identifier,
+                    chat_display_name,
+                    handle_id,
+                    chat_id,
+                ) = row
+
+                message = {
+                    "message_id": message_id,
+                    "text": text,
+                    "timestamp": self._convert_cocoa_timestamp(date),
+                    "is_from_me": bool(is_from_me),
+                    "service": service or "iMessage",
+                    "chat_identifier": chat_identifier or "Unknown",
+                    "chat_display_name": chat_display_name or "Unknown Chat",
+                    "handle_id": handle_id or "Unknown",
+                    "contact_name": self._get_contact_name(handle_id or ""),
+                    "chat_id": chat_id,
+                }
+                messages.append(message)
+
+            conn.close()
+            print(f"Found {len(messages)} messages in database")
+            return messages
+
+        except sqlite3.Error as e:
+            print(f"Error reading iMessage database: {e}")
+            return []
+        except Exception as e:
+            print(f"Unexpected error reading iMessage database: {e}")
+            return []
+
+    def _group_messages_by_chat(self, messages: list[dict]) -> dict[int, list[dict]]:
+        """
+        Group messages by chat ID.
+
+        Args:
+            messages: List of message dictionaries
+
+        Returns:
+            Dictionary mapping chat_id to list of messages
+        """
+        chats = {}
+        for message in messages:
+            chat_id = message["chat_id"]
+            if chat_id not in chats:
+                chats[chat_id] = []
+            chats[chat_id].append(message)
+
+        return chats
+
+    def _create_concatenated_content(self, chat_id: int, messages: list[dict]) -> str:
+        """
+        Create concatenated content from chat messages.
+
+        Args:
+            chat_id: The chat ID
+            messages: List of messages in the chat
+
+        Returns:
+            Concatenated text content
+        """
+        if not messages:
+            return ""
+
+        # Get chat info from first message
+        first_msg = messages[0]
+        chat_name = first_msg["chat_display_name"]
+        chat_identifier = first_msg["chat_identifier"]
+
+        # Build message content
+        message_parts = []
+        for message in messages:
+            timestamp = message["timestamp"]
+            is_from_me = message["is_from_me"]
+            text = message["text"]
+            contact_name = message["contact_name"]
+
+            if is_from_me:
+                prefix = "[You]"
+            else:
+                prefix = f"[{contact_name}]"
+
+            if timestamp != "Unknown":
+                prefix += f" ({timestamp})"
+
+            message_parts.append(f"{prefix}: {text}")
+
+        concatenated_text = "\n\n".join(message_parts)
+
+        doc_content = f"""Chat: {chat_name}
+Identifier: {chat_identifier}
+Messages ({len(messages)} messages):
+
+{concatenated_text}
+"""
+        return doc_content
+
+    def _create_individual_content(self, message: dict) -> str:
+        """
+        Create content for individual message.
+
+        Args:
+            message: Message dictionary
+
+        Returns:
+            Formatted message content
+        """
+        timestamp = message["timestamp"]
+        is_from_me = message["is_from_me"]
+        text = message["text"]
+        contact_name = message["contact_name"]
+        chat_name = message["chat_display_name"]
+
+        sender = "You" if is_from_me else contact_name
+
+        return f"""Message from {sender} in chat "{chat_name}"
+Time: {timestamp}
+Content: {text}
+"""
+
+    def load_data(self, input_dir: str | None = None, **load_kwargs: Any) -> list[Document]:
+        """
+        Load iMessage data and return as documents.
+
+        Args:
+            input_dir: Optional path to directory containing chat.db file.
+                      If not provided, uses default macOS location.
+            **load_kwargs: Additional arguments (unused)
+
+        Returns:
+            List of Document objects containing iMessage data
+        """
+        docs = []
+
+        # Determine database path
+        if input_dir:
+            db_path = Path(input_dir) / "chat.db"
+        else:
+            db_path = self._get_default_chat_db_path()
+
+        print(f"Reading iMessage database from: {db_path}")
+
+        # Read messages from database
+        messages = self._read_messages_from_db(db_path)
+        if not messages:
+            return docs
+
+        if self.concatenate_conversations:
+            # Group messages by chat and create concatenated documents
+            chats = self._group_messages_by_chat(messages)
+
+            for chat_id, chat_messages in chats.items():
+                if not chat_messages:
+                    continue
+
+                content = self._create_concatenated_content(chat_id, chat_messages)
+
+                # Create metadata
+                first_msg = chat_messages[0]
+                last_msg = chat_messages[-1]
+
+                metadata = {
+                    "source": "iMessage",
+                    "chat_id": chat_id,
+                    "chat_name": first_msg["chat_display_name"],
+                    "chat_identifier": first_msg["chat_identifier"],
+                    "message_count": len(chat_messages),
+                    "first_message_date": first_msg["timestamp"],
+                    "last_message_date": last_msg["timestamp"],
+                    "participants": list(
+                        {msg["contact_name"] for msg in chat_messages if not msg["is_from_me"]}
+                    ),
+                }
+
+                doc = Document(text=content, metadata=metadata)
+                docs.append(doc)
+
+        else:
+            # Create individual documents for each message
+            for message in messages:
+                content = self._create_individual_content(message)
+
+                metadata = {
+                    "source": "iMessage",
+                    "message_id": message["message_id"],
+                    "chat_id": message["chat_id"],
+                    "chat_name": message["chat_display_name"],
+                    "chat_identifier": message["chat_identifier"],
+                    "timestamp": message["timestamp"],
+                    "is_from_me": message["is_from_me"],
+                    "contact_name": message["contact_name"],
+                    "service": message["service"],
+                }
+
+                doc = Document(text=content, metadata=metadata)
+                docs.append(doc)
+
+        print(f"Created {len(docs)} documents from iMessage data")
+        return docs

apps/imessage_rag.py

@@ -0,0 +1,125 @@
+"""
+iMessage RAG Example.
+
+This example demonstrates how to build a RAG system on your iMessage conversation history.
+"""
+
+import asyncio
+from pathlib import Path
+
+from leann.chunking_utils import create_text_chunks
+
+from apps.base_rag_example import BaseRAGExample
+from apps.imessage_data.imessage_reader import IMessageReader
+
+
+class IMessageRAG(BaseRAGExample):
+    """RAG example for iMessage conversation history."""
+
+    def __init__(self):
+        super().__init__(
+            name="iMessage",
+            description="RAG on your iMessage conversation history",
+            default_index_name="imessage_index",
+        )
+
+    def _add_specific_arguments(self, parser):
+        """Add iMessage-specific arguments."""
+        imessage_group = parser.add_argument_group("iMessage Parameters")
+        imessage_group.add_argument(
+            "--db-path",
+            type=str,
+            default=None,
+            help="Path to iMessage chat.db file (default: ~/Library/Messages/chat.db)",
+        )
+        imessage_group.add_argument(
+            "--concatenate-conversations",
+            action="store_true",
+            default=True,
+            help="Concatenate messages within conversations for better context (default: True)",
+        )
+        imessage_group.add_argument(
+            "--no-concatenate-conversations",
+            action="store_true",
+            help="Process each message individually instead of concatenating by conversation",
+        )
+        imessage_group.add_argument(
+            "--chunk-size",
+            type=int,
+            default=1000,
+            help="Maximum characters per text chunk (default: 1000)",
+        )
+        imessage_group.add_argument(
+            "--chunk-overlap",
+            type=int,
+            default=200,
+            help="Overlap between text chunks (default: 200)",
+        )
+
+    async def load_data(self, args) -> list[str]:
+        """Load iMessage history and convert to text chunks."""
+        print("Loading iMessage conversation history...")
+
+        # Determine concatenation setting
+        concatenate = args.concatenate_conversations and not args.no_concatenate_conversations
+
+        # Initialize iMessage reader
+        reader = IMessageReader(concatenate_conversations=concatenate)
+
+        # Load documents
+        try:
+            if args.db_path:
+                # Use custom database path
+                db_dir = str(Path(args.db_path).parent)
+                documents = reader.load_data(input_dir=db_dir)
+            else:
+                # Use default macOS location
+                documents = reader.load_data()
+
+        except Exception as e:
+            print(f"Error loading iMessage data: {e}")
+            print("\nTroubleshooting tips:")
+            print("1. Make sure you have granted Full Disk Access to your terminal/IDE")
+            print("2. Check that the iMessage database exists at ~/Library/Messages/chat.db")
+            print("3. Try specifying a custom path with --db-path if you have a backup")
+            return []
+
+        if not documents:
+            print("No iMessage conversations found!")
+            return []
+
+        print(f"Loaded {len(documents)} iMessage documents")
+
+        # Show some statistics
+        total_messages = sum(doc.metadata.get("message_count", 1) for doc in documents)
+        print(f"Total messages: {total_messages}")
+
+        if concatenate:
+            # Show chat statistics
+            chat_names = [doc.metadata.get("chat_name", "Unknown") for doc in documents]
+            unique_chats = len(set(chat_names))
+            print(f"Unique conversations: {unique_chats}")
+
+        # Convert to text chunks
+        all_texts = create_text_chunks(
+            documents,
+            chunk_size=args.chunk_size,
+            chunk_overlap=args.chunk_overlap,
+        )
+
+        # Apply max_items limit if specified
+        if args.max_items > 0:
+            all_texts = all_texts[: args.max_items]
+            print(f"Limited to {len(all_texts)} text chunks (max_items={args.max_items})")
+
+        return all_texts
+
+
+async def main():
+    """Main entry point."""
+    app = IMessageRAG()
+    await app.run()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

benchmarks/benchmark_no_recompute.py

@@ -0,0 +1,148 @@
+import argparse
+import os
+import time
+from pathlib import Path
+
+from leann import LeannBuilder, LeannSearcher
+
+
+def _meta_exists(index_path: str) -> bool:
+    p = Path(index_path)
+    return (p.parent / f"{p.stem}.meta.json").exists()
+
+
+def ensure_index(index_path: str, backend_name: str, num_docs: int, is_recompute: bool) -> None:
+    # if _meta_exists(index_path):
+    #     return
+    kwargs = {}
+    if backend_name == "hnsw":
+        kwargs["is_compact"] = is_recompute
+    builder = LeannBuilder(
+        backend_name=backend_name,
+        embedding_model=os.getenv("LEANN_EMBED_MODEL", "facebook/contriever"),
+        embedding_mode=os.getenv("LEANN_EMBED_MODE", "sentence-transformers"),
+        graph_degree=32,
+        complexity=64,
+        is_recompute=is_recompute,
+        num_threads=4,
+        **kwargs,
+    )
+    for i in range(num_docs):
+        builder.add_text(
+            f"This is a test document number {i}. It contains some repeated text for benchmarking."
+        )
+    builder.build_index(index_path)
+
+
+def _bench_group(
+    index_path: str,
+    recompute: bool,
+    query: str,
+    repeats: int,
+    complexity: int = 32,
+    top_k: int = 10,
+) -> float:
+    # Independent searcher per group; fixed port when recompute
+    searcher = LeannSearcher(index_path=index_path)
+
+    # Warm-up once
+    _ = searcher.search(
+        query,
+        top_k=top_k,
+        complexity=complexity,
+        recompute_embeddings=recompute,
+    )
+
+    def _once() -> float:
+        t0 = time.time()
+        _ = searcher.search(
+            query,
+            top_k=top_k,
+            complexity=complexity,
+            recompute_embeddings=recompute,
+        )
+        return time.time() - t0
+
+    if repeats <= 1:
+        t = _once()
+    else:
+        vals = [_once() for _ in range(repeats)]
+        vals.sort()
+        t = vals[len(vals) // 2]
+
+    searcher.cleanup()
+    return t
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--num-docs", type=int, default=5000)
+    parser.add_argument("--repeats", type=int, default=3)
+    parser.add_argument("--complexity", type=int, default=32)
+    args = parser.parse_args()
+
+    base = Path.cwd() / ".leann" / "indexes" / f"bench_n{args.num_docs}"
+    base.parent.mkdir(parents=True, exist_ok=True)
+    # ---------- Build HNSW variants ----------
+    hnsw_r = str(base / f"hnsw_recompute_n{args.num_docs}.leann")
+    hnsw_nr = str(base / f"hnsw_norecompute_n{args.num_docs}.leann")
+    ensure_index(hnsw_r, "hnsw", args.num_docs, True)
+    ensure_index(hnsw_nr, "hnsw", args.num_docs, False)
+
+    # ---------- Build DiskANN variants ----------
+    diskann_r = str(base / "diskann_r.leann")
+    diskann_nr = str(base / "diskann_nr.leann")
+    ensure_index(diskann_r, "diskann", args.num_docs, True)
+    ensure_index(diskann_nr, "diskann", args.num_docs, False)
+
+    # ---------- Helpers ----------
+    def _size_for(prefix: str) -> int:
+        p = Path(prefix)
+        base_dir = p.parent
+        stem = p.stem
+        total = 0
+        for f in base_dir.iterdir():
+            if f.is_file() and f.name.startswith(stem):
+                total += f.stat().st_size
+        return total
+
+    # ---------- HNSW benchmark ----------
+    t_hnsw_r = _bench_group(
+        hnsw_r, True, "test document number 42", repeats=args.repeats, complexity=args.complexity
+    )
+    t_hnsw_nr = _bench_group(
+        hnsw_nr, False, "test document number 42", repeats=args.repeats, complexity=args.complexity
+    )
+    size_hnsw_r = _size_for(hnsw_r)
+    size_hnsw_nr = _size_for(hnsw_nr)
+
+    print("Benchmark results (HNSW):")
+    print(f"  recompute=True:  search_time={t_hnsw_r:.3f}s, size={size_hnsw_r / 1024 / 1024:.1f}MB")
+    print(
+        f"  recompute=False: search_time={t_hnsw_nr:.3f}s, size={size_hnsw_nr / 1024 / 1024:.1f}MB"
+    )
+    print("  Expectation: no-recompute should be faster but larger on disk.")
+
+    # ---------- DiskANN benchmark ----------
+    t_diskann_r = _bench_group(
+        diskann_r, True, "DiskANN R test doc 123", repeats=args.repeats, complexity=args.complexity
+    )
+    t_diskann_nr = _bench_group(
+        diskann_nr,
+        False,
+        "DiskANN NR test doc 123",
+        repeats=args.repeats,
+        complexity=args.complexity,
+    )
+    size_diskann_r = _size_for(diskann_r)
+    size_diskann_nr = _size_for(diskann_nr)
+
+    print("\nBenchmark results (DiskANN):")
+    print(f"  build(recompute=True, partition): size={size_diskann_r / 1024 / 1024:.1f}MB")
+    print(f"  build(recompute=False):          size={size_diskann_nr / 1024 / 1024:.1f}MB")
+    print(f"  search recompute=True (final rerank): {t_diskann_r:.3f}s")
+    print(f"  search recompute=False (PQ only):     {t_diskann_nr:.3f}s")
+
+
+if __name__ == "__main__":
+    main()

benchmarks/data/.gitattributes

@@ -1,82 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.lz4 filter=lfs diff=lfs merge=lfs -text
-*.mds filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
-# Audio files - uncompressed
-*.pcm filter=lfs diff=lfs merge=lfs -text
-*.sam filter=lfs diff=lfs merge=lfs -text
-*.raw filter=lfs diff=lfs merge=lfs -text
-# Audio files - compressed
-*.aac filter=lfs diff=lfs merge=lfs -text
-*.flac filter=lfs diff=lfs merge=lfs -text
-*.mp3 filter=lfs diff=lfs merge=lfs -text
-*.ogg filter=lfs diff=lfs merge=lfs -text
-*.wav filter=lfs diff=lfs merge=lfs -text
-# Image files - uncompressed
-*.bmp filter=lfs diff=lfs merge=lfs -text
-*.gif filter=lfs diff=lfs merge=lfs -text
-*.png filter=lfs diff=lfs merge=lfs -text
-*.tiff filter=lfs diff=lfs merge=lfs -text
-# Image files - compressed
-*.jpg filter=lfs diff=lfs merge=lfs -text
-*.jpeg filter=lfs diff=lfs merge=lfs -text
-*.webp filter=lfs diff=lfs merge=lfs -text
-# Video files - compressed
-*.mp4 filter=lfs diff=lfs merge=lfs -text
-*.webm filter=lfs diff=lfs merge=lfs -text
-ground_truth/dpr/id_map.json filter=lfs diff=lfs merge=lfs -text
-indices/dpr/dpr_diskann.passages.idx filter=lfs diff=lfs merge=lfs -text
-indices/dpr/dpr_diskann.passages.jsonl filter=lfs diff=lfs merge=lfs -text
-indices/dpr/dpr_diskann_disk.index filter=lfs diff=lfs merge=lfs -text
-indices/dpr/leann.labels.map filter=lfs diff=lfs merge=lfs -text
-indices/rpj_wiki/leann.labels.map filter=lfs diff=lfs merge=lfs -text
-indices/rpj_wiki/rpj_wiki.index filter=lfs diff=lfs merge=lfs -text
-indices/rpj_wiki/rpj_wiki.passages.0.idx filter=lfs diff=lfs merge=lfs -text
-indices/rpj_wiki/rpj_wiki.passages.0.jsonl filter=lfs diff=lfs merge=lfs -text
-indices/rpj_wiki/rpj_wiki.passages.1.idx filter=lfs diff=lfs merge=lfs -text
-indices/rpj_wiki/rpj_wiki.passages.1.jsonl filter=lfs diff=lfs merge=lfs -text
-indices/rpj_wiki/rpj_wiki.passages.2.idx filter=lfs diff=lfs merge=lfs -text
-indices/rpj_wiki/rpj_wiki.passages.2.jsonl filter=lfs diff=lfs merge=lfs -text
-indices/rpj_wiki/rpj_wiki.passages.3.idx filter=lfs diff=lfs merge=lfs -text
-indices/rpj_wiki/rpj_wiki.passages.3.jsonl filter=lfs diff=lfs merge=lfs -text
-indices/rpj_wiki/rpj_wiki.passages.4.idx filter=lfs diff=lfs merge=lfs -text
-indices/rpj_wiki/rpj_wiki.passages.4.jsonl filter=lfs diff=lfs merge=lfs -text
-indices/rpj_wiki/rpj_wiki.passages.5.idx filter=lfs diff=lfs merge=lfs -text
-indices/rpj_wiki/rpj_wiki.passages.5.jsonl filter=lfs diff=lfs merge=lfs -text
-indices/rpj_wiki/rpj_wiki.passages.6.idx filter=lfs diff=lfs merge=lfs -text
-indices/rpj_wiki/rpj_wiki.passages.6.jsonl filter=lfs diff=lfs merge=lfs -text
-indices/rpj_wiki/rpj_wiki.passages.7.idx filter=lfs diff=lfs merge=lfs -text
-indices/rpj_wiki/rpj_wiki.passages.7.jsonl filter=lfs diff=lfs merge=lfs -text

benchmarks/data/README.md

@@ -0,0 +1,44 @@
+---
+license: mit
+---
+
+# LEANN-RAG Evaluation Data
+
+This repository contains the necessary data to run the recall evaluation scripts for the [LEANN-RAG](https://huggingface.co/LEANN-RAG) project.
+
+## Dataset Components
+
+This dataset is structured into three main parts:
+
+1.  **Pre-built LEANN Indices**:
+    *   `dpr/`: A pre-built index for the DPR dataset.
+    *   `rpj_wiki/`: A pre-built index for the RPJ-Wiki dataset.
+    These indices were created using the `leann-core` library and are required by the `LeannSearcher`.
+
+2.  **Ground Truth Data**:
+    *   `ground_truth/`: Contains the ground truth files (`flat_results_nq_k3.json`) for both the DPR and RPJ-Wiki datasets. These files map queries to the original passage IDs from the Natural Questions benchmark, evaluated using the Contriever model.
+
+3.  **Queries**:
+    *   `queries/`: Contains the `nq_open.jsonl` file with the Natural Questions queries used for the evaluation.
+
+## Usage
+
+To use this data, you can download it locally using the `huggingface-hub` library. First, install the library:
+
+```bash
+pip install huggingface-hub
+```
+
+Then, you can download the entire dataset to a local directory (e.g., `data/`) with the following Python script:
+
+```python
+from huggingface_hub import snapshot_download
+
+snapshot_download(
+    repo_id="LEANN-RAG/leann-rag-evaluation-data",
+    repo_type="dataset",
+    local_dir="data"
+)
+```
+
+This will download all the necessary files into a local `data` folder, preserving the repository structure. The evaluation scripts in the main [LEANN-RAG Space](https://huggingface.co/LEANN-RAG) are configured to work with this data structure.

benchmarks/diskann_vs_hnsw_speed_comparison.py

@@ -10,6 +10,7 @@ This benchmark compares search performance between DiskANN and HNSW backends:
 """
 
 import gc
+import multiprocessing as mp
 import tempfile
 import time
 from pathlib import Path
@@ -17,6 +18,12 @@ from typing import Any
 
 import numpy as np
 
+# Prefer 'fork' start method to avoid POSIX semaphore leaks on macOS
+try:
+    mp.set_start_method("fork", force=True)
+except Exception:
+    pass
+
 
 def create_test_texts(n_docs: int) -> list[str]:
     """Create synthetic test documents for benchmarking."""
@@ -113,10 +120,10 @@ def benchmark_backend(
         ]
         score_validity_rate = len(valid_scores) / len(all_scores) if all_scores else 0
 
-        # Clean up
+        # Clean up (ensure embedding server shutdown and object GC)
         try:
-            if hasattr(searcher, "__del__"):
-                searcher.__del__()
+            if hasattr(searcher, "cleanup"):
+                searcher.cleanup()
             del searcher
             del builder
             gc.collect()
@@ -259,10 +266,21 @@ if __name__ == "__main__":
         print(f"\n❌ Benchmark failed: {e}")
         sys.exit(1)
     finally:
-        # Ensure clean exit
+        # Ensure clean exit (forceful to prevent rare hangs from atexit/threads)
         try:
             gc.collect()
             print("\n🧹 Cleanup completed")
+            # Flush stdio to ensure message is visible before hard-exit
+            try:
+                import sys as _sys
+
+                _sys.stdout.flush()
+                _sys.stderr.flush()
+            except Exception:
+                pass
         except Exception:
             pass
-        sys.exit(0)
+        # Use os._exit to bypass atexit handlers that may hang in rare cases
+        import os as _os
+
+        _os._exit(0)

benchmarks/run_evaluation.py

@@ -12,7 +12,7 @@ import time
 from pathlib import Path
 
 import numpy as np
-from leann.api import LeannBuilder, LeannSearcher
+from leann.api import LeannBuilder, LeannChat, LeannSearcher
 
 
 def download_data_if_needed(data_root: Path, download_embeddings: bool = False):
@@ -197,6 +197,25 @@ def main():
     parser.add_argument(
         "--ef-search", type=int, default=120, help="The 'efSearch' parameter for HNSW."
     )
+    parser.add_argument(
+        "--batch-size",
+        type=int,
+        default=0,
+        help="Batch size for HNSW batched search (0 disables batching)",
+    )
+    parser.add_argument(
+        "--llm-type",
+        type=str,
+        choices=["ollama", "hf", "openai", "gemini", "simulated"],
+        default="ollama",
+        help="LLM backend type to optionally query during evaluation (default: ollama)",
+    )
+    parser.add_argument(
+        "--llm-model",
+        type=str,
+        default="qwen3:1.7b",
+        help="LLM model identifier for the chosen backend (default: qwen3:1.7b)",
+    )
     args = parser.parse_args()
 
     # --- Path Configuration ---
@@ -318,9 +337,24 @@ def main():
 
         for i in range(num_eval_queries):
             start_time = time.time()
-            new_results = searcher.search(queries[i], top_k=args.top_k, ef=args.ef_search)
+            new_results = searcher.search(
+                queries[i],
+                top_k=args.top_k,
+                complexity=args.ef_search,
+                batch_size=args.batch_size,
+            )
             search_times.append(time.time() - start_time)
 
+            # Optional: also call the LLM with configurable backend/model (does not affect recall)
+            llm_config = {"type": args.llm_type, "model": args.llm_model}
+            chat = LeannChat(args.index_path, llm_config=llm_config, searcher=searcher)
+            answer = chat.ask(
+                queries[i],
+                top_k=args.top_k,
+                complexity=args.ef_search,
+                batch_size=args.batch_size,
+            )
+            print(f"Answer: {answer}")
             # Correct Recall Calculation: Based on TEXT content
             new_texts = {result.text for result in new_results}
 

benchmarks/simple_mac_tpt_test.py

@@ -20,7 +20,7 @@ except ImportError:
 
 @dataclass
 class BenchmarkConfig:
-    model_path: str = "facebook/contriever"
+    model_path: str = "facebook/contriever-msmarco"
     batch_sizes: list[int] = None
     seq_length: int = 256
     num_runs: int = 5
@@ -34,7 +34,7 @@ class BenchmarkConfig:
 
     def __post_init__(self):
         if self.batch_sizes is None:
-            self.batch_sizes = [1, 2, 4, 8, 16, 32, 64]
+            self.batch_sizes = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024]
 
 
 class MLXBenchmark:
@@ -179,10 +179,16 @@ class Benchmark:
 
     def _run_inference(self, input_ids: torch.Tensor) -> float:
         attention_mask = torch.ones_like(input_ids)
-
+        # print shape of input_ids and attention_mask
+        print(f"input_ids shape: {input_ids.shape}")
+        print(f"attention_mask shape: {attention_mask.shape}")
         start_time = time.time()
         with torch.no_grad():
             self.model(input_ids=input_ids, attention_mask=attention_mask)
+        if torch.cuda.is_available():
+            torch.cuda.synchronize()
+        if torch.backends.mps.is_available():
+            torch.mps.synchronize()
         end_time = time.time()
 
         return end_time - start_time

docs/ast_chunking_guide.md

@@ -0,0 +1,143 @@
+# AST-Aware Code chunking guide
+
+## Overview
+
+This guide covers best practices for using AST-aware code chunking in LEANN. AST chunking provides better semantic understanding of code structure compared to traditional text-based chunking.
+
+## Quick Start
+
+### Basic Usage
+
+```bash
+# Enable AST chunking for mixed content (code + docs)
+python -m apps.document_rag --enable-code-chunking --data-dir ./my_project
+
+# Specialized code repository indexing
+python -m apps.code_rag --repo-dir ./my_codebase
+
+# Global CLI with AST support
+leann build my-code-index --docs ./src --use-ast-chunking
+```
+
+### Installation
+
+```bash
+# Install LEANN with AST chunking support
+uv pip install -e "."
+```
+
+#### For normal users (PyPI install)
+- Use `pip install leann` or `uv pip install leann`.
+- `astchunk` is pulled automatically from PyPI as a dependency; no extra steps.
+
+#### For developers (from source, editable)
+```bash
+git clone https://github.com/yichuan-w/LEANN.git leann
+cd leann
+git submodule update --init --recursive
+uv sync
+```
+- This repo vendors `astchunk` as a git submodule at `packages/astchunk-leann` (our fork).
+- `[tool.uv.sources]` maps the `astchunk` package to that path in editable mode.
+- You can edit code under `packages/astchunk-leann` and Python will use your changes immediately (no separate `pip install astchunk` needed).
+
+## Best Practices
+
+### When to Use AST Chunking
+
+✅ **Recommended for:**
+- Code repositories with multiple languages
+- Mixed documentation and code content
+- Complex codebases with deep function/class hierarchies
+- When working with Claude Code for code assistance
+
+❌ **Not recommended for:**
+- Pure text documents
+- Very large files (>1MB)
+- Languages not supported by tree-sitter
+
+### Optimal Configuration
+
+```bash
+# Recommended settings for most codebases
+python -m apps.code_rag \
+    --repo-dir ./src \
+    --ast-chunk-size 768 \
+    --ast-chunk-overlap 96 \
+    --exclude-dirs .git __pycache__ node_modules build dist
+```
+
+### Supported Languages
+
+| Extension | Language | Status |
+|-----------|----------|--------|
+| `.py` | Python | ✅ Full support |
+| `.java` | Java | ✅ Full support |
+| `.cs` | C# | ✅ Full support |
+| `.ts`, `.tsx` | TypeScript | ✅ Full support |
+| `.js`, `.jsx` | JavaScript | ✅ Via TypeScript parser |
+
+## Integration Examples
+
+### Document RAG with Code Support
+
+```python
+# Enable code chunking in document RAG
+python -m apps.document_rag \
+    --enable-code-chunking \
+    --data-dir ./project \
+    --query "How does authentication work in the codebase?"
+```
+
+### Claude Code Integration
+
+When using with Claude Code MCP server, AST chunking provides better context for:
+- Code completion and suggestions
+- Bug analysis and debugging
+- Architecture understanding
+- Refactoring assistance
+
+## Troubleshooting
+
+### Common Issues
+
+1. **Fallback to Traditional Chunking**
+   - Normal behavior for unsupported languages
+   - Check logs for specific language support
+
+2. **Performance with Large Files**
+   - Adjust `--max-file-size` parameter
+   - Use `--exclude-dirs` to skip unnecessary directories
+
+3. **Quality Issues**
+   - Try different `--ast-chunk-size` values (512, 768, 1024)
+   - Adjust overlap for better context preservation
+
+### Debug Mode
+
+```bash
+export LEANN_LOG_LEVEL=DEBUG
+python -m apps.code_rag --repo-dir ./my_code
+```
+
+## Migration from Traditional Chunking
+
+Existing workflows continue to work without changes. To enable AST chunking:
+
+```bash
+# Before
+python -m apps.document_rag --chunk-size 256
+
+# After (maintains traditional chunking for non-code files)
+python -m apps.document_rag --enable-code-chunking --chunk-size 256 --ast-chunk-size 768
+```
+
+## References
+
+- [astchunk GitHub Repository](https://github.com/yilinjz/astchunk)
+- [LEANN MCP Integration](../packages/leann-mcp/README.md)
+- [Research Paper](https://arxiv.org/html/2506.15655v1)
+
+---
+
+**Note**: AST chunking maintains full backward compatibility while enhancing code understanding capabilities.

docs/configuration-guide.md

@@ -52,7 +52,7 @@ Based on our experience developing LEANN, embedding models fall into three categ
 ### Quick Start: Cloud and Local Embedding Options
 
 **OpenAI Embeddings (Fastest Setup)**
-For immediate testing without local model downloads:
+For immediate testing without local model downloads(also if you [do not have GPU](https://github.com/yichuan-w/LEANN/issues/43) and do not care that much about your document leak, you should use this, we compute the embedding and recompute using openai API):
 ```bash
 # Set OpenAI embeddings (requires OPENAI_API_KEY)
 --embedding-mode openai --embedding-model text-embedding-3-small
@@ -97,29 +97,23 @@ ollama pull nomic-embed-text
 ```
 
 ### DiskANN
-**Best for**: Performance-critical applications and large datasets - **Production-ready with automatic graph partitioning**
+**Best for**: Large datasets, especially when you want `recompute=True`.
 
-**How it works:**
-- **Product Quantization (PQ) + Real-time Reranking**: Uses compressed PQ codes for fast graph traversal, then recomputes exact embeddings for final candidates
-- **Automatic Graph Partitioning**: When `is_recompute=True`, automatically partitions large indices and safely removes redundant files to save storage
-- **Superior Speed-Accuracy Trade-off**: Faster search than HNSW while maintaining high accuracy
+**Key advantages:**
+- **Faster search** on large datasets (3x+ speedup vs HNSW in many cases)
+- **Smart storage**: `recompute=True` enables automatic graph partitioning for smaller indexes
+- **Better scaling**: Designed for 100k+ documents
 
-**Trade-offs compared to HNSW:**
-- ✅ **Faster search latency** (typically 2-8x speedup)
-- ✅ **Better scaling** for large datasets
-- ✅ **Smart storage management** with automatic partitioning
-- ✅ **Better graph locality** with `--ldg-times` parameter for SSD optimization
-- ⚠️ **Slightly larger index size** due to PQ tables and graph metadata
+**Recompute behavior:**
+- `recompute=True` (recommended): Pure PQ traversal + final reranking - faster and enables partitioning
+- `recompute=False`: PQ + partial real distances during traversal - slower but higher accuracy
 
 ```bash
 # Recommended for most use cases
 --backend-name diskann --graph-degree 32 --build-complexity 64
-
-# For large-scale deployments
---backend-name diskann --graph-degree 64 --build-complexity 128
 ```
 
-**Performance Benchmark**: Run `python benchmarks/diskann_vs_hnsw_speed_comparison.py` to compare DiskANN and HNSW on your system.
+**Performance Benchmark**: Run `uv run benchmarks/diskann_vs_hnsw_speed_comparison.py` to compare DiskANN and HNSW on your system.
 
 ## LLM Selection: Engine and Model Comparison
 
@@ -273,24 +267,114 @@ Every configuration choice involves trade-offs:
 
 The key is finding the right balance for your specific use case. Start small and simple, measure performance, then scale up only where needed.
 
-## Deep Dive: Critical Configuration Decisions
+## Low-resource setups
 
-### When to Disable Recomputation
+If you don’t have a local GPU or builds/searches are too slow, use one or more of the options below.
 
-LEANN's recomputation feature provides exact distance calculations but can be disabled for extreme QPS requirements:
+### 1) Use OpenAI embeddings (no local compute)
+
+Fastest path with zero local GPU requirements. Set your API key and use OpenAI embeddings during build and search:
 
 ```bash
---no-recompute  # Disable selective recomputation
+export OPENAI_API_KEY=sk-...
+
+# Build with OpenAI embeddings
+leann build my-index \
+  --embedding-mode openai \
+  --embedding-model text-embedding-3-small
+
+# Search with OpenAI embeddings (recompute at query time)
+leann search my-index "your query" \
+  --recompute
 ```
 
-**Trade-offs**:
-- **With recomputation** (default): Exact distances, best quality, higher latency, minimal storage (only stores metadata, recomputes embeddings on-demand)
-- **Without recomputation**: Must store full embeddings, significantly higher memory and storage usage (10-100x more), but faster search
+### 2) Run remote builds with SkyPilot (cloud GPU)
+
+Offload embedding generation and index building to a GPU VM using [SkyPilot](https://skypilot.readthedocs.io/en/latest/). A template is provided at `sky/leann-build.yaml`.
+
+```bash
+# One-time: install and configure SkyPilot
+pip install skypilot
+
+# Launch with defaults (L4:1) and mount ./data to ~/leann-data; the build runs automatically
+sky launch -c leann-gpu sky/leann-build.yaml
+
+# Override parameters via -e key=value (optional)
+sky launch -c leann-gpu sky/leann-build.yaml \
+  -e index_name=my-index \
+  -e backend=hnsw \
+  -e embedding_mode=sentence-transformers \
+  -e embedding_model=Qwen/Qwen3-Embedding-0.6B
+
+# Copy the built index back to your local .leann (use rsync)
+rsync -Pavz leann-gpu:~/.leann/indexes/my-index ./.leann/indexes/
+```
+
+### 3) Disable recomputation to trade storage for speed
+
+If you need lower latency and have more storage/memory, disable recomputation. This stores full embeddings and avoids recomputing at search time.
+
+```bash
+# Build without recomputation (HNSW requires non-compact in this mode)
+leann build my-index --no-recompute --no-compact
+
+# Search without recomputation
+leann search my-index "your query" --no-recompute
+```
+
+When to use:
+- Extreme low latency requirements (high QPS, interactive assistants)
+- Read-heavy workloads where storage is cheaper than latency
+- No always-available GPU
+
+Constraints:
+- HNSW: when `--no-recompute` is set, LEANN automatically disables compact mode during build
+- DiskANN: supported; `--no-recompute` skips selective recompute during search
+
+Storage impact:
+- Storing N embeddings of dimension D with float32 requires approximately N × D × 4 bytes
+- Example: 1,000,000 chunks × 768 dims × 4 bytes ≈ 2.86 GB (plus graph/metadata)
+
+Converting an existing index (rebuild required):
+```bash
+# Rebuild in-place (ensure you still have original docs or can regenerate chunks)
+leann build my-index --force --no-recompute --no-compact
+```
+
+Python API usage:
+```python
+from leann import LeannSearcher
+
+searcher = LeannSearcher("/path/to/my-index.leann")
+results = searcher.search("your query", top_k=10, recompute_embeddings=False)
+```
+
+Trade-offs:
+- Lower latency and fewer network hops at query time
+- Significantly higher storage (10–100× vs selective recomputation)
+- Slightly larger memory footprint during build and search
+
+Quick benchmark results (`benchmarks/benchmark_no_recompute.py` with 5k texts, complexity=32):
+
+- HNSW
+
+  ```text
+  recompute=True:  search_time=0.818s, size=1.1MB
+  recompute=False: search_time=0.012s, size=16.6MB
+  ```
+
+- DiskANN
+
+  ```text
+  recompute=True:  search_time=0.041s, size=5.9MB
+  recompute=False: search_time=0.013s, size=24.6MB
+  ```
+
+Conclusion:
+- **HNSW**: `no-recompute` is significantly faster (no embedding recomputation) but requires much more storage (stores all embeddings)
+- **DiskANN**: `no-recompute` uses PQ + partial real distances during traversal (slower but higher accuracy), while `recompute=True` uses pure PQ traversal + final reranking (faster traversal, enables build-time partitioning for smaller storage)
+
 
-**Disable when**:
-- You have abundant storage and memory
-- Need extremely low latency (< 100ms)
-- Running a read-heavy workload where storage cost is acceptable
 
 ## Further Reading
 

docs/features.md

@@ -3,6 +3,7 @@
 ## 🔥 Core Features
 
 - **🔄 Real-time Embeddings** - Eliminate heavy embedding storage with dynamic computation using optimized ZMQ servers and highly optimized search paradigm (overlapping and batching) with highly optimized embedding engine
+- **🧠 AST-Aware Code Chunking** - Intelligent code chunking that preserves semantic boundaries (functions, classes, methods) for Python, Java, C#, and TypeScript files
 - **📈 Scalable Architecture** - Handles millions of documents on consumer hardware; the larger your dataset, the more LEANN can save
 - **🎯 Graph Pruning** - Advanced techniques to minimize the storage overhead of vector search to a limited footprint
 - **🏗️ Pluggable Backends** - HNSW/FAISS (default), with optional DiskANN for large-scale deployments

docs/grep_search.md

@@ -0,0 +1,149 @@
+# LEANN Grep Search Usage Guide
+
+## Overview
+
+LEANN's grep search functionality provides exact text matching for finding specific code patterns, error messages, function names, or exact phrases in your indexed documents.
+
+## Basic Usage
+
+### Simple Grep Search
+
+```python
+from leann.api import LeannSearcher
+
+searcher = LeannSearcher("your_index_path")
+
+# Exact text search
+results = searcher.search("def authenticate_user", use_grep=True, top_k=5)
+
+for result in results:
+    print(f"Score: {result.score}")
+    print(f"Text: {result.text[:100]}...")
+    print("-" * 40)
+```
+
+### Comparison: Semantic vs Grep Search
+
+```python
+# Semantic search - finds conceptually similar content
+semantic_results = searcher.search("machine learning algorithms", top_k=3)
+
+# Grep search - finds exact text matches
+grep_results = searcher.search("def train_model", use_grep=True, top_k=3)
+```
+
+## When to Use Grep Search
+
+### Use Cases
+
+- **Code Search**: Finding specific function definitions, class names, or variable references
+- **Error Debugging**: Locating exact error messages or stack traces
+- **Documentation**: Finding specific API endpoints or exact terminology
+
+### Examples
+
+```python
+# Find function definitions
+functions = searcher.search("def __init__", use_grep=True)
+
+# Find import statements
+imports = searcher.search("from sklearn import", use_grep=True)
+
+# Find specific error types
+errors = searcher.search("FileNotFoundError", use_grep=True)
+
+# Find TODO comments
+todos = searcher.search("TODO:", use_grep=True)
+
+# Find configuration entries
+configs = searcher.search("server_port=", use_grep=True)
+```
+
+## Technical Details
+
+### How It Works
+
+1. **File Location**: Grep search operates on the raw text stored in `.jsonl` files
+2. **Command Execution**: Uses the system `grep` command with case-insensitive search
+3. **Result Processing**: Parses JSON lines and extracts text and metadata
+4. **Scoring**: Simple frequency-based scoring based on query term occurrences
+
+### Search Process
+
+```
+Query: "def train_model"
+  ↓
+grep -i -n "def train_model" documents.leann.passages.jsonl
+  ↓
+Parse matching JSON lines
+  ↓
+Calculate scores based on term frequency
+  ↓
+Return top_k results
+```
+
+### Scoring Algorithm
+
+```python
+# Term frequency in document
+score = text.lower().count(query.lower())
+```
+
+Results are ranked by score (highest first), with higher scores indicating more occurrences of the search term.
+
+## Error Handling
+
+### Common Issues
+
+#### Grep Command Not Found
+```
+RuntimeError: grep command not found. Please install grep or use semantic search.
+```
+
+**Solution**: Install grep on your system:
+- **Ubuntu/Debian**: `sudo apt-get install grep`
+- **macOS**: grep is pre-installed
+- **Windows**: Use WSL or install grep via Git Bash/MSYS2
+
+#### No Results Found
+```python
+# Check if your query exists in the raw data
+results = searcher.search("your_query", use_grep=True)
+if not results:
+    print("No exact matches found. Try:")
+    print("1. Check spelling and case")
+    print("2. Use partial terms")
+    print("3. Switch to semantic search")
+```
+
+## Complete Example
+
+```python
+#!/usr/bin/env python3
+"""
+Grep Search Example
+Demonstrates grep search for exact text matching.
+"""
+
+from leann.api import LeannSearcher
+
+def demonstrate_grep_search():
+    # Initialize searcher
+    searcher = LeannSearcher("my_index")
+
+    print("=== Function Search ===")
+    functions = searcher.search("def __init__", use_grep=True, top_k=5)
+    for i, result in enumerate(functions, 1):
+        print(f"{i}. Score: {result.score}")
+        print(f"   Preview: {result.text[:60]}...")
+        print()
+
+    print("=== Error Search ===")
+    errors = searcher.search("FileNotFoundError", use_grep=True, top_k=3)
+    for result in errors:
+        print(f"Content: {result.text.strip()}")
+        print("-" * 40)
+
+if __name__ == "__main__":
+    demonstrate_grep_search()
+```

docs/metadata_filtering.md

@@ -0,0 +1,300 @@
+# LEANN Metadata Filtering Usage Guide
+
+## Overview
+
+Leann possesses metadata filtering capabilities that allow you to filter search results based on arbitrary metadata fields set during chunking. This feature enables use cases like spoiler-free book search, document filtering by date/type, code search by file type, and potentially much more.
+
+## Basic Usage
+
+### Adding Metadata to Your Documents
+
+When building your index, add metadata to each text chunk:
+
+```python
+from leann.api import LeannBuilder
+
+builder = LeannBuilder("hnsw")
+
+# Add text with metadata
+builder.add_text(
+    text="Chapter 1: Alice falls down the rabbit hole",
+    metadata={
+        "chapter": 1,
+        "character": "Alice",
+        "themes": ["adventure", "curiosity"],
+        "word_count": 150
+    }
+)
+
+builder.build_index("alice_in_wonderland_index")
+```
+
+### Searching with Metadata Filters
+
+Use the `metadata_filters` parameter in search calls:
+
+```python
+from leann.api import LeannSearcher
+
+searcher = LeannSearcher("alice_in_wonderland_index")
+
+# Search with filters
+results = searcher.search(
+    query="What happens to Alice?",
+    top_k=10,
+    metadata_filters={
+        "chapter": {"<=": 5},           # Only chapters 1-5
+        "spoiler_level": {"!=": "high"} # No high spoilers
+    }
+)
+```
+
+## Filter Syntax
+
+### Basic Structure
+
+```python
+metadata_filters = {
+    "field_name": {"operator": value},
+    "another_field": {"operator": value}
+}
+```
+
+### Supported Operators
+
+#### Comparison Operators
+- `"=="`: Equal to
+- `"!="`: Not equal to
+- `"<"`: Less than
+- `"<="`: Less than or equal
+- `">"`: Greater than
+- `">="`: Greater than or equal
+
+```python
+# Examples
+{"chapter": {"==": 1}}           # Exactly chapter 1
+{"page": {">": 100}}            # Pages after 100
+{"rating": {">=": 4.0}}         # Rating 4.0 or higher
+{"word_count": {"<": 500}}      # Short passages
+```
+
+#### Membership Operators
+- `"in"`: Value is in list
+- `"not_in"`: Value is not in list
+
+```python
+# Examples
+{"character": {"in": ["Alice", "Bob"]}}      # Alice OR Bob
+{"genre": {"not_in": ["horror", "thriller"]}} # Exclude genres
+{"tags": {"in": ["fiction", "adventure"]}}   # Any of these tags
+```
+
+#### String Operators
+- `"contains"`: String contains substring
+- `"starts_with"`: String starts with prefix
+- `"ends_with"`: String ends with suffix
+
+```python
+# Examples
+{"title": {"contains": "alice"}}        # Title contains "alice"
+{"filename": {"ends_with": ".py"}}      # Python files
+{"author": {"starts_with": "Dr."}}      # Authors with "Dr." prefix
+```
+
+#### Boolean Operators
+- `"is_true"`: Field is truthy
+- `"is_false"`: Field is falsy
+
+```python
+# Examples
+{"is_published": {"is_true": True}}     # Published content
+{"is_draft": {"is_false": False}}       # Not drafts
+```
+
+### Multiple Operators on Same Field
+
+You can apply multiple operators to the same field (AND logic):
+
+```python
+metadata_filters = {
+    "word_count": {
+        ">=": 100,    # At least 100 words
+        "<=": 500     # At most 500 words
+    }
+}
+```
+
+### Compound Filters
+
+Multiple fields are combined with AND logic:
+
+```python
+metadata_filters = {
+    "chapter": {"<=": 10},              # Up to chapter 10
+    "character": {"==": "Alice"},       # About Alice
+    "spoiler_level": {"!=": "high"}     # No major spoilers
+}
+```
+
+## Use Case Examples
+
+### 1. Spoiler-Free Book Search
+
+```python
+# Reader has only read up to chapter 5
+def search_spoiler_free(query, max_chapter):
+    return searcher.search(
+        query=query,
+        metadata_filters={
+            "chapter": {"<=": max_chapter},
+            "spoiler_level": {"in": ["none", "low"]}
+        }
+    )
+
+results = search_spoiler_free("What happens to Alice?", max_chapter=5)
+```
+
+### 2. Document Management by Date
+
+```python
+# Find recent documents
+recent_docs = searcher.search(
+    query="project updates",
+    metadata_filters={
+        "date": {">=": "2024-01-01"},
+        "document_type": {"==": "report"}
+    }
+)
+```
+
+### 3. Code Search by File Type
+
+```python
+# Search only Python files
+python_code = searcher.search(
+    query="authentication function",
+    metadata_filters={
+        "file_extension": {"==": ".py"},
+        "lines_of_code": {"<": 100}
+    }
+)
+```
+
+### 4. Content Filtering by Audience
+
+```python
+# Age-appropriate content
+family_content = searcher.search(
+    query="adventure stories",
+    metadata_filters={
+        "age_rating": {"in": ["G", "PG"]},
+        "content_warnings": {"not_in": ["violence", "adult_themes"]}
+    }
+)
+```
+
+### 5. Multi-Book Series Management
+
+```python
+# Search across first 3 books only
+early_series = searcher.search(
+    query="character development",
+    metadata_filters={
+        "series": {"==": "Harry Potter"},
+        "book_number": {"<=": 3}
+    }
+)
+```
+
+## Running the Example
+
+You can see metadata filtering in action with our spoiler-free book RAG example:
+
+```bash
+# Don't forget to set up the environment
+uv venv
+source .venv/bin/activate
+
+# Set your OpenAI API key (required for embeddings, but you can update the example locally and use ollama instead)
+export OPENAI_API_KEY="your-api-key-here"
+
+# Run the spoiler-free book RAG example
+uv run examples/spoiler_free_book_rag.py
+```
+
+This example demonstrates:
+- Building an index with metadata (chapter numbers, characters, themes, locations)
+- Searching with filters to avoid spoilers (e.g., only show results up to chapter 5)
+- Different scenarios for readers at various points in the book
+
+The example uses Alice's Adventures in Wonderland as sample data and shows how you can search for information without revealing plot points from later chapters.
+
+## Advanced Patterns
+
+### Custom Chunking with metadata
+
+```python
+def chunk_book_with_metadata(book_text, book_info):
+    chunks = []
+
+    for chapter_num, chapter_text in parse_chapters(book_text):
+        # Extract entities, themes, etc.
+        characters = extract_characters(chapter_text)
+        themes = classify_themes(chapter_text)
+        spoiler_level = assess_spoiler_level(chapter_text, chapter_num)
+
+        # Create chunks with rich metadata
+        for paragraph in split_paragraphs(chapter_text):
+            chunks.append({
+                "text": paragraph,
+                "metadata": {
+                    "book_title": book_info["title"],
+                    "chapter": chapter_num,
+                    "characters": characters,
+                    "themes": themes,
+                    "spoiler_level": spoiler_level,
+                    "word_count": len(paragraph.split()),
+                    "reading_level": calculate_reading_level(paragraph)
+                }
+            })
+
+    return chunks
+```
+
+## Performance Considerations
+
+### Efficient Filtering Strategies
+
+1. **Post-search filtering**: Applies filters after vector search, which should be efficient for typical result sets (10-100 results).
+
+2. **Metadata design**: Keep metadata fields simple and avoid deeply nested structures.
+
+### Best Practices
+
+1. **Consistent metadata schema**: Use consistent field names and value types across your documents.
+
+2. **Reasonable metadata size**: Keep metadata reasonably sized to avoid storage overhead.
+
+3. **Type consistency**: Use consistent data types for the same fields (e.g., always integers for chapter numbers).
+
+4. **Index multiple granularities**: Consider chunking at different levels (paragraph, section, chapter) with appropriate metadata.
+
+### Adding Metadata to Existing Indices
+
+To add metadata filtering to existing indices, you'll need to rebuild them with metadata:
+
+```python
+# Read existing passages and add metadata
+def add_metadata_to_existing_chunks(chunks):
+    for chunk in chunks:
+        # Extract or assign metadata based on content
+        chunk["metadata"] = extract_metadata(chunk["text"])
+    return chunks
+
+# Rebuild index with metadata
+enhanced_chunks = add_metadata_to_existing_chunks(existing_chunks)
+builder = LeannBuilder("hnsw")
+for chunk in enhanced_chunks:
+    builder.add_text(chunk["text"], chunk["metadata"])
+builder.build_index("enhanced_index")
+```

examples/dynamic_update_no_recompute.py

@@ -0,0 +1,404 @@
+"""Dynamic HNSW update demo without compact storage.
+
+This script reproduces the minimal scenario we used while debugging on-the-fly
+recompute:
+
+1. Build a non-compact HNSW index from the first few paragraphs of a text file.
+2. Print the top results with `recompute_embeddings=True`.
+3. Append additional paragraphs with :meth:`LeannBuilder.update_index`.
+4. Run the same query again to show the newly inserted passages.
+
+Run it with ``uv`` (optionally pointing LEANN_HNSW_LOG_PATH at a file to inspect
+ZMQ activity)::
+
+    LEANN_HNSW_LOG_PATH=embedding_fetch.log \
+    uv run -m examples.dynamic_update_no_recompute \
+      --index-path .leann/examples/leann-demo.leann
+
+By default the script builds an index from ``data/2501.14312v1 (1).pdf`` and
+then updates it with LEANN-related material from ``data/2506.08276v1.pdf``.
+It issues the query "What's LEANN?" before and after the update to show how the
+new passages become immediately searchable. The script uses the
+``sentence-transformers/all-MiniLM-L6-v2`` model with ``is_recompute=True`` so
+Faiss pulls existing vectors on demand via the ZMQ embedding server, while
+freshly added passages are embedded locally just like the initial build.
+
+To make storage comparisons easy, the script can also build a matching
+``is_recompute=False`` baseline (enabled by default) and report the index size
+delta after the update. Disable the baseline run with
+``--skip-compare-no-recompute`` if you only need the recompute flow.
+"""
+
+import argparse
+import json
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Any
+
+from leann.api import LeannBuilder, LeannSearcher
+from leann.registry import register_project_directory
+
+from apps.chunking import create_text_chunks
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+
+DEFAULT_QUERY = "What's LEANN?"
+DEFAULT_INITIAL_FILES = [REPO_ROOT / "data" / "2501.14312v1 (1).pdf"]
+DEFAULT_UPDATE_FILES = [REPO_ROOT / "data" / "2506.08276v1.pdf"]
+
+
+def load_chunks_from_files(paths: list[Path]) -> list[str]:
+    from llama_index.core import SimpleDirectoryReader
+
+    documents = []
+    for path in paths:
+        p = path.expanduser().resolve()
+        if not p.exists():
+            raise FileNotFoundError(f"Input path not found: {p}")
+        if p.is_dir():
+            reader = SimpleDirectoryReader(str(p), recursive=False)
+            documents.extend(reader.load_data(show_progress=True))
+        else:
+            reader = SimpleDirectoryReader(input_files=[str(p)])
+            documents.extend(reader.load_data(show_progress=True))
+
+    if not documents:
+        return []
+
+    chunks = create_text_chunks(
+        documents,
+        chunk_size=512,
+        chunk_overlap=128,
+        use_ast_chunking=False,
+    )
+    return [c for c in chunks if isinstance(c, str) and c.strip()]
+
+
+def run_search(index_path: Path, query: str, top_k: int, *, recompute_embeddings: bool) -> list:
+    searcher = LeannSearcher(str(index_path))
+    try:
+        return searcher.search(
+            query=query,
+            top_k=top_k,
+            recompute_embeddings=recompute_embeddings,
+            batch_size=16,
+        )
+    finally:
+        searcher.cleanup()
+
+
+def print_results(title: str, results: Iterable) -> None:
+    print(f"\n=== {title} ===")
+    res_list = list(results)
+    print(f"results count: {len(res_list)}")
+    print("passages:")
+    if not res_list:
+        print("  (no passages returned)")
+    for res in res_list:
+        snippet = res.text.replace("\n", " ")[:120]
+        print(f"  - {res.id}: {snippet}... (score={res.score:.4f})")
+
+
+def build_initial_index(
+    index_path: Path,
+    paragraphs: list[str],
+    model_name: str,
+    embedding_mode: str,
+    is_recompute: bool,
+) -> None:
+    builder = LeannBuilder(
+        backend_name="hnsw",
+        embedding_model=model_name,
+        embedding_mode=embedding_mode,
+        is_compact=False,
+        is_recompute=is_recompute,
+    )
+    for idx, passage in enumerate(paragraphs):
+        builder.add_text(passage, metadata={"id": str(idx)})
+    builder.build_index(str(index_path))
+
+
+def update_index(
+    index_path: Path,
+    start_id: int,
+    paragraphs: list[str],
+    model_name: str,
+    embedding_mode: str,
+    is_recompute: bool,
+) -> None:
+    updater = LeannBuilder(
+        backend_name="hnsw",
+        embedding_model=model_name,
+        embedding_mode=embedding_mode,
+        is_compact=False,
+        is_recompute=is_recompute,
+    )
+    for offset, passage in enumerate(paragraphs, start=start_id):
+        updater.add_text(passage, metadata={"id": str(offset)})
+    updater.update_index(str(index_path))
+
+
+def ensure_index_dir(index_path: Path) -> None:
+    index_path.parent.mkdir(parents=True, exist_ok=True)
+
+
+def cleanup_index_files(index_path: Path) -> None:
+    """Remove leftover index artifacts for a clean rebuild."""
+
+    parent = index_path.parent
+    if not parent.exists():
+        return
+    stem = index_path.stem
+    for file in parent.glob(f"{stem}*"):
+        if file.is_file():
+            file.unlink()
+
+
+def index_file_size(index_path: Path) -> int:
+    """Return the size of the primary .index file for the given index path."""
+
+    index_file = index_path.parent / f"{index_path.stem}.index"
+    return index_file.stat().st_size if index_file.exists() else 0
+
+
+def load_metadata_snapshot(index_path: Path) -> dict[str, Any] | None:
+    meta_path = index_path.parent / f"{index_path.name}.meta.json"
+    if not meta_path.exists():
+        return None
+    try:
+        return json.loads(meta_path.read_text())
+    except json.JSONDecodeError:
+        return None
+
+
+def run_workflow(
+    *,
+    label: str,
+    index_path: Path,
+    initial_paragraphs: list[str],
+    update_paragraphs: list[str],
+    model_name: str,
+    embedding_mode: str,
+    is_recompute: bool,
+    query: str,
+    top_k: int,
+) -> dict[str, Any]:
+    prefix = f"[{label}] " if label else ""
+
+    ensure_index_dir(index_path)
+    cleanup_index_files(index_path)
+
+    print(f"{prefix}Building initial index...")
+    build_initial_index(
+        index_path,
+        initial_paragraphs,
+        model_name,
+        embedding_mode,
+        is_recompute=is_recompute,
+    )
+
+    initial_size = index_file_size(index_path)
+    before_results = run_search(
+        index_path,
+        query,
+        top_k,
+        recompute_embeddings=is_recompute,
+    )
+
+    print(f"\n{prefix}Updating index with additional passages...")
+    update_index(
+        index_path,
+        start_id=len(initial_paragraphs),
+        paragraphs=update_paragraphs,
+        model_name=model_name,
+        embedding_mode=embedding_mode,
+        is_recompute=is_recompute,
+    )
+
+    after_results = run_search(
+        index_path,
+        query,
+        top_k,
+        recompute_embeddings=is_recompute,
+    )
+    updated_size = index_file_size(index_path)
+
+    return {
+        "initial_size": initial_size,
+        "updated_size": updated_size,
+        "delta": updated_size - initial_size,
+        "before_results": before_results,
+        "after_results": after_results,
+        "metadata": load_metadata_snapshot(index_path),
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--initial-files",
+        type=Path,
+        nargs="+",
+        default=DEFAULT_INITIAL_FILES,
+        help="Initial document files (PDF/TXT) used to build the base index",
+    )
+    parser.add_argument(
+        "--index-path",
+        type=Path,
+        default=Path(".leann/examples/leann-demo.leann"),
+        help="Destination index path (default: .leann/examples/leann-demo.leann)",
+    )
+    parser.add_argument(
+        "--initial-count",
+        type=int,
+        default=8,
+        help="Number of chunks to use from the initial documents (default: 8)",
+    )
+    parser.add_argument(
+        "--update-files",
+        type=Path,
+        nargs="*",
+        default=DEFAULT_UPDATE_FILES,
+        help="Additional documents to add during update (PDF/TXT)",
+    )
+    parser.add_argument(
+        "--update-count",
+        type=int,
+        default=4,
+        help="Number of chunks to append from update documents (default: 4)",
+    )
+    parser.add_argument(
+        "--update-text",
+        type=str,
+        default=(
+            "LEANN (Lightweight Embedding ANN) is an indexing toolkit focused on "
+            "recompute-aware HNSW graphs, allowing embeddings to be regenerated "
+            "on demand to keep disk usage minimal."
+        ),
+        help="Fallback text to append if --update-files is omitted",
+    )
+    parser.add_argument(
+        "--top-k",
+        type=int,
+        default=4,
+        help="Number of results to show for each search (default: 4)",
+    )
+    parser.add_argument(
+        "--query",
+        type=str,
+        default=DEFAULT_QUERY,
+        help="Query to run before/after the update",
+    )
+    parser.add_argument(
+        "--embedding-model",
+        type=str,
+        default="sentence-transformers/all-MiniLM-L6-v2",
+        help="Embedding model name",
+    )
+    parser.add_argument(
+        "--embedding-mode",
+        type=str,
+        default="sentence-transformers",
+        choices=["sentence-transformers", "openai", "mlx", "ollama"],
+        help="Embedding backend mode",
+    )
+    parser.add_argument(
+        "--compare-no-recompute",
+        dest="compare_no_recompute",
+        action="store_true",
+        help="Also run a baseline with is_recompute=False and report its index growth.",
+    )
+    parser.add_argument(
+        "--skip-compare-no-recompute",
+        dest="compare_no_recompute",
+        action="store_false",
+        help="Skip building the no-recompute baseline.",
+    )
+    parser.set_defaults(compare_no_recompute=True)
+    args = parser.parse_args()
+
+    ensure_index_dir(args.index_path)
+    register_project_directory(REPO_ROOT)
+
+    initial_chunks = load_chunks_from_files(list(args.initial_files))
+    if not initial_chunks:
+        raise ValueError("No text chunks extracted from the initial files.")
+
+    initial = initial_chunks[: args.initial_count]
+    if not initial:
+        raise ValueError("Initial chunk set is empty after applying --initial-count.")
+
+    if args.update_files:
+        update_chunks = load_chunks_from_files(list(args.update_files))
+        if not update_chunks:
+            raise ValueError("No text chunks extracted from the update files.")
+        to_add = update_chunks[: args.update_count]
+    else:
+        if not args.update_text:
+            raise ValueError("Provide --update-files or --update-text for the update step.")
+        to_add = [args.update_text]
+    if not to_add:
+        raise ValueError("Update chunk set is empty after applying --update-count.")
+
+    recompute_stats = run_workflow(
+        label="recompute",
+        index_path=args.index_path,
+        initial_paragraphs=initial,
+        update_paragraphs=to_add,
+        model_name=args.embedding_model,
+        embedding_mode=args.embedding_mode,
+        is_recompute=True,
+        query=args.query,
+        top_k=args.top_k,
+    )
+
+    print_results("initial search", recompute_stats["before_results"])
+    print_results("after update", recompute_stats["after_results"])
+    print(
+        f"\n[recompute] Index file size change: {recompute_stats['initial_size']} -> {recompute_stats['updated_size']} bytes"
+        f" (Δ {recompute_stats['delta']})"
+    )
+
+    if recompute_stats["metadata"]:
+        meta_view = {k: recompute_stats["metadata"].get(k) for k in ("is_compact", "is_pruned")}
+        print("[recompute] metadata snapshot:")
+        print(json.dumps(meta_view, indent=2))
+
+    if args.compare_no_recompute:
+        baseline_path = (
+            args.index_path.parent / f"{args.index_path.stem}-norecompute{args.index_path.suffix}"
+        )
+        baseline_stats = run_workflow(
+            label="no-recompute",
+            index_path=baseline_path,
+            initial_paragraphs=initial,
+            update_paragraphs=to_add,
+            model_name=args.embedding_model,
+            embedding_mode=args.embedding_mode,
+            is_recompute=False,
+            query=args.query,
+            top_k=args.top_k,
+        )
+
+        print(
+            f"\n[no-recompute] Index file size change: {baseline_stats['initial_size']} -> {baseline_stats['updated_size']} bytes"
+            f" (Δ {baseline_stats['delta']})"
+        )
+
+        after_texts = [res.text for res in recompute_stats["after_results"]]
+        baseline_after_texts = [res.text for res in baseline_stats["after_results"]]
+        if after_texts == baseline_after_texts:
+            print(
+                "[no-recompute] Search results match recompute baseline; see above for the shared output."
+            )
+        else:
+            print("[no-recompute] WARNING: search results differ from recompute baseline.")
+
+        if baseline_stats["metadata"]:
+            meta_view = {k: baseline_stats["metadata"].get(k) for k in ("is_compact", "is_pruned")}
+            print("[no-recompute] metadata snapshot:")
+            print(json.dumps(meta_view, indent=2))
+
+
+if __name__ == "__main__":
+    main()

examples/grep_search_example.py

@@ -0,0 +1,35 @@
+"""
+Grep Search Example
+
+Shows how to use grep-based text search instead of semantic search.
+Useful when you need exact text matches rather than meaning-based results.
+"""
+
+from leann import LeannSearcher
+
+# Load your index
+searcher = LeannSearcher("my-documents.leann")
+
+# Regular semantic search
+print("=== Semantic Search ===")
+results = searcher.search("machine learning algorithms", top_k=3)
+for result in results:
+    print(f"Score: {result.score:.3f}")
+    print(f"Text: {result.text[:80]}...")
+    print()
+
+# Grep-based search for exact text matches
+print("=== Grep Search ===")
+results = searcher.search("def train_model", top_k=3, use_grep=True)
+for result in results:
+    print(f"Score: {result.score}")
+    print(f"Text: {result.text[:80]}...")
+    print()
+
+# Find specific error messages
+error_results = searcher.search("FileNotFoundError", use_grep=True)
+print(f"Found {len(error_results)} files mentioning FileNotFoundError")
+
+# Search for function definitions
+func_results = searcher.search("class SearchResult", use_grep=True, top_k=5)
+print(f"Found {len(func_results)} class definitions")

examples/spoiler_free_book_rag.py

@@ -0,0 +1,250 @@
+#!/usr/bin/env python3
+"""
+Spoiler-Free Book RAG Example using LEANN Metadata Filtering
+
+This example demonstrates how to use LEANN's metadata filtering to create
+a spoiler-free book RAG system where users can search for information
+up to a specific chapter they've read.
+
+Usage:
+    python spoiler_free_book_rag.py
+"""
+
+import os
+import sys
+from typing import Any, Optional
+
+# Add LEANN to path (adjust path as needed)
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../packages/leann-core/src"))
+
+from leann.api import LeannBuilder, LeannSearcher
+
+
+def chunk_book_with_metadata(book_title: str = "Sample Book") -> list[dict[str, Any]]:
+    """
+    Create sample book chunks with metadata for demonstration.
+
+    In a real implementation, this would parse actual book files (epub, txt, etc.)
+    and extract chapter boundaries, character mentions, etc.
+
+    Args:
+        book_title: Title of the book
+
+    Returns:
+        List of chunk dictionaries with text and metadata
+    """
+    # Sample book chunks with metadata
+    # In practice, you'd use proper text processing libraries
+
+    sample_chunks = [
+        {
+            "text": "Alice was beginning to get very tired of sitting by her sister on the bank, and of having nothing to do.",
+            "metadata": {
+                "book": book_title,
+                "chapter": 1,
+                "page": 1,
+                "characters": ["Alice", "Sister"],
+                "themes": ["boredom", "curiosity"],
+                "location": "riverbank",
+            },
+        },
+        {
+            "text": "So she was considering in her own mind (as well as she could, for the hot day made her feel very sleepy and stupid), whether the pleasure of making a daisy-chain would be worth the trouble of getting up and picking the daisies, when suddenly a White Rabbit with pink eyes ran close by her.",
+            "metadata": {
+                "book": book_title,
+                "chapter": 1,
+                "page": 2,
+                "characters": ["Alice", "White Rabbit"],
+                "themes": ["decision", "surprise", "magic"],
+                "location": "riverbank",
+            },
+        },
+        {
+            "text": "Alice found herself falling down a very deep well. Either the well was very deep, or she fell very slowly, for she had plenty of time as she fell to look about her and to wonder what was going to happen next.",
+            "metadata": {
+                "book": book_title,
+                "chapter": 2,
+                "page": 15,
+                "characters": ["Alice"],
+                "themes": ["falling", "wonder", "transformation"],
+                "location": "rabbit hole",
+            },
+        },
+        {
+            "text": "Alice meets the Cheshire Cat, who tells her that everyone in Wonderland is mad, including Alice herself.",
+            "metadata": {
+                "book": book_title,
+                "chapter": 6,
+                "page": 85,
+                "characters": ["Alice", "Cheshire Cat"],
+                "themes": ["madness", "philosophy", "identity"],
+                "location": "Duchess's house",
+            },
+        },
+        {
+            "text": "At the Queen's croquet ground, Alice witnesses the absurd trial that reveals the arbitrary nature of Wonderland's justice system.",
+            "metadata": {
+                "book": book_title,
+                "chapter": 8,
+                "page": 120,
+                "characters": ["Alice", "Queen of Hearts", "King of Hearts"],
+                "themes": ["justice", "absurdity", "authority"],
+                "location": "Queen's court",
+            },
+        },
+        {
+            "text": "Alice realizes that Wonderland was all a dream, even the Rabbit, as she wakes up on the riverbank next to her sister.",
+            "metadata": {
+                "book": book_title,
+                "chapter": 12,
+                "page": 180,
+                "characters": ["Alice", "Sister", "Rabbit"],
+                "themes": ["revelation", "reality", "growth"],
+                "location": "riverbank",
+            },
+        },
+    ]
+
+    return sample_chunks
+
+
+def build_spoiler_free_index(book_chunks: list[dict[str, Any]], index_name: str) -> str:
+    """
+    Build a LEANN index with book chunks that include spoiler metadata.
+
+    Args:
+        book_chunks: List of book chunks with metadata
+        index_name: Name for the index
+
+    Returns:
+        Path to the built index
+    """
+    print(f"📚 Building spoiler-free book index: {index_name}")
+
+    # Initialize LEANN builder
+    builder = LeannBuilder(
+        backend_name="hnsw", embedding_model="text-embedding-3-small", embedding_mode="openai"
+    )
+
+    # Add each chunk with its metadata
+    for chunk in book_chunks:
+        builder.add_text(text=chunk["text"], metadata=chunk["metadata"])
+
+    # Build the index
+    index_path = f"{index_name}_book_index"
+    builder.build_index(index_path)
+
+    print(f"✅ Index built successfully: {index_path}")
+    return index_path
+
+
+def spoiler_free_search(
+    index_path: str,
+    query: str,
+    max_chapter: int,
+    character_filter: Optional[list[str]] = None,
+) -> list[dict[str, Any]]:
+    """
+    Perform a spoiler-free search on the book index.
+
+    Args:
+        index_path: Path to the LEANN index
+        query: Search query
+        max_chapter: Maximum chapter number to include
+        character_filter: Optional list of characters to focus on
+
+    Returns:
+        List of search results safe for the reader
+    """
+    print(f"🔍 Searching: '{query}' (up to chapter {max_chapter})")
+
+    searcher = LeannSearcher(index_path)
+
+    metadata_filters = {"chapter": {"<=": max_chapter}}
+
+    if character_filter:
+        metadata_filters["characters"] = {"contains": character_filter[0]}
+
+    results = searcher.search(query=query, top_k=10, metadata_filters=metadata_filters)
+
+    return results
+
+
+def demo_spoiler_free_rag():
+    """
+    Demonstrate the spoiler-free book RAG system.
+    """
+    print("🎭 Spoiler-Free Book RAG Demo")
+    print("=" * 40)
+
+    # Step 1: Prepare book data
+    book_title = "Alice's Adventures in Wonderland"
+    book_chunks = chunk_book_with_metadata(book_title)
+
+    print(f"📖 Loaded {len(book_chunks)} chunks from '{book_title}'")
+
+    # Step 2: Build the index (in practice, this would be done once)
+    try:
+        index_path = build_spoiler_free_index(book_chunks, "alice_wonderland")
+    except Exception as e:
+        print(f"❌ Failed to build index (likely missing dependencies): {e}")
+        print(
+            "💡 This demo shows the filtering logic - actual indexing requires LEANN dependencies"
+        )
+        return
+
+    # Step 3: Demonstrate various spoiler-free searches
+    search_scenarios = [
+        {
+            "description": "Reader who has only read Chapter 1",
+            "query": "What can you tell me about the rabbit?",
+            "max_chapter": 1,
+        },
+        {
+            "description": "Reader who has read up to Chapter 5",
+            "query": "Tell me about Alice's adventures",
+            "max_chapter": 5,
+        },
+        {
+            "description": "Reader who has read most of the book",
+            "query": "What does the Cheshire Cat represent?",
+            "max_chapter": 10,
+        },
+        {
+            "description": "Reader who has read the whole book",
+            "query": "What can you tell me about the rabbit?",
+            "max_chapter": 12,
+        },
+    ]
+
+    for scenario in search_scenarios:
+        print(f"\n📚 Scenario: {scenario['description']}")
+        print(f"   Query: {scenario['query']}")
+
+        try:
+            results = spoiler_free_search(
+                index_path=index_path,
+                query=scenario["query"],
+                max_chapter=scenario["max_chapter"],
+            )
+
+            print(f"   📄 Found {len(results)} results:")
+            for i, result in enumerate(results[:3], 1):  # Show top 3
+                chapter = result.metadata.get("chapter", "?")
+                location = result.metadata.get("location", "?")
+                print(f"      {i}. Chapter {chapter} ({location}): {result.text[:80]}...")
+
+        except Exception as e:
+            print(f"   ❌ Search failed: {e}")
+
+
+if __name__ == "__main__":
+    print("📚 LEANN Spoiler-Free Book RAG Example")
+    print("=====================================")
+
+    try:
+        demo_spoiler_free_rag()
+    except ImportError as e:
+        print(f"❌ Cannot run demo due to missing dependencies: {e}")
+    except Exception as e:
+        print(f"❌ Error running demo: {e}")

llms.txt

@@ -0,0 +1,28 @@
+# llms.txt — LEANN MCP and Agent Integration
+product: LEANN
+homepage: https://github.com/yichuan-w/LEANN
+contact: https://github.com/yichuan-w/LEANN/issues
+
+# Installation
+install: uv tool install leann-core --with leann
+
+# MCP Server Entry Point
+mcp.server: leann_mcp
+mcp.protocol_version: 2024-11-05
+
+# Tools
+mcp.tools: leann_list, leann_search
+
+mcp.tool.leann_list.description: List available LEANN indexes
+mcp.tool.leann_list.input: {}
+
+mcp.tool.leann_search.description: Semantic search across a named LEANN index
+mcp.tool.leann_search.input.index_name: string, required
+mcp.tool.leann_search.input.query: string, required
+mcp.tool.leann_search.input.top_k: integer, optional, default=5, min=1, max=20
+mcp.tool.leann_search.input.complexity: integer, optional, default=32, min=16, max=128
+
+# Notes
+note: Build indexes with `leann build <name> --docs <files...>` before searching.
+example.add: claude mcp add --scope user leann-server -- leann_mcp
+example.verify: claude mcp list | cat

packages/leann-backend-diskann/CMakeLists.txt

@@ -1,8 +0,0 @@
-# packages/leann-backend-diskann/CMakeLists.txt (simplified version)
-
-cmake_minimum_required(VERSION 3.20)
-project(leann_backend_diskann_wrapper)
-
-# Tell CMake to directly enter the DiskANN submodule and execute its own CMakeLists.txt
-# DiskANN will handle everything itself, including compiling Python bindings
-add_subdirectory(src/third_party/DiskANN)

packages/leann-backend-diskann/leann_backend_diskann/diskann_backend.py

@@ -22,6 +22,11 @@ logger = logging.getLogger(__name__)
 @contextlib.contextmanager
 def suppress_cpp_output_if_needed():
     """Suppress C++ stdout/stderr based on LEANN_LOG_LEVEL"""
+    # In CI we avoid fiddling with low-level file descriptors to prevent aborts
+    if os.getenv("CI") == "true":
+        yield
+        return
+
     log_level = os.getenv("LEANN_LOG_LEVEL", "WARNING").upper()
 
     # Only suppress if log level is WARNING or higher (ERROR, CRITICAL)
@@ -436,9 +441,14 @@ class DiskannSearcher(BaseSearcher):
         else:  # "global"
             use_global_pruning = True
 
-        # Perform search with suppressed C++ output based on log level
-        use_deferred_fetch = kwargs.get("USE_DEFERRED_FETCH", True)
-        recompute_neighors = False
+        # Strategy:
+        # - Traversal always uses PQ distances
+        # - If recompute_embeddings=True, do a single final rerank via deferred fetch
+        #   (fetch embeddings for the final candidate set only)
+        # - Do not recompute neighbor distances along the path
+        use_deferred_fetch = True if recompute_embeddings else False
+        recompute_neighors = False  # Expected typo. For backward compatibility.
+
         with suppress_cpp_output_if_needed():
             labels, distances = self._index.batch_search(
                 query,
@@ -459,25 +469,3 @@ class DiskannSearcher(BaseSearcher):
         string_labels = [[str(int_label) for int_label in batch_labels] for batch_labels in labels]
 
         return {"labels": string_labels, "distances": distances}
-
-    def cleanup(self):
-        """Cleanup DiskANN-specific resources including C++ index."""
-        # Call parent cleanup first
-        super().cleanup()
-
-        # Delete the C++ index to trigger destructors
-        try:
-            if hasattr(self, "_index") and self._index is not None:
-                del self._index
-                self._index = None
-                self._current_zmq_port = None
-        except Exception:
-            pass
-
-        # Force garbage collection to ensure C++ objects are destroyed
-        try:
-            import gc
-
-            gc.collect()
-        except Exception:
-            pass

packages/leann-backend-diskann/leann_backend_diskann/diskann_embedding_server.py

@@ -83,9 +83,7 @@ def create_diskann_embedding_server(
 
     logger.info(f"Loading PassageManager with metadata_file_path: {passages_file}")
     passages = PassageManager(meta["passage_sources"], metadata_file_path=passages_file)
-    logger.info(
-        f"Loaded PassageManager with {len(passages.global_offset_map)} passages from metadata"
-    )
+    logger.info(f"Loaded PassageManager with {len(passages)} passages from metadata")
 
     # Import protobuf after ensuring the path is correct
     try:
@@ -100,12 +98,12 @@ def create_diskann_embedding_server(
         socket = context.socket(
             zmq.REP
         )  # REP socket for both BaseSearcher and DiskANN C++ REQ clients
-        socket.setsockopt(zmq.LINGER, 0)  # Don't block on close
         socket.bind(f"tcp://*:{zmq_port}")
         logger.info(f"DiskANN ZMQ REP server listening on port {zmq_port}")
 
-        socket.setsockopt(zmq.RCVTIMEO, 300000)
-        socket.setsockopt(zmq.SNDTIMEO, 300000)
+        socket.setsockopt(zmq.RCVTIMEO, 1000)
+        socket.setsockopt(zmq.SNDTIMEO, 1000)
+        socket.setsockopt(zmq.LINGER, 0)
 
         while True:
             try:
@@ -222,30 +220,217 @@ def create_diskann_embedding_server(
                 traceback.print_exc()
                 raise
 
-    zmq_thread = threading.Thread(target=zmq_server_thread, daemon=True)
+    def zmq_server_thread_with_shutdown(shutdown_event):
+        """ZMQ server thread that respects shutdown signal.
+
+        This creates its own REP socket, binds to zmq_port, and periodically
+        checks shutdown_event using recv timeouts to exit cleanly.
+        """
+        logger.info("DiskANN ZMQ server thread started with shutdown support")
+
+        context = zmq.Context()
+        rep_socket = context.socket(zmq.REP)
+        rep_socket.bind(f"tcp://*:{zmq_port}")
+        logger.info(f"DiskANN ZMQ REP server listening on port {zmq_port}")
+
+        # Set receive timeout so we can check shutdown_event periodically
+        rep_socket.setsockopt(zmq.RCVTIMEO, 1000)  # 1 second timeout
+        rep_socket.setsockopt(zmq.SNDTIMEO, 1000)
+        rep_socket.setsockopt(zmq.LINGER, 0)
+
+        try:
+            while not shutdown_event.is_set():
+                try:
+                    e2e_start = time.time()
+                    # REP socket receives single-part messages
+                    message = rep_socket.recv()
+
+                    # Check for empty messages - REP socket requires response to every request
+                    if not message:
+                        logger.warning("Received empty message, sending empty response")
+                        rep_socket.send(b"")
+                        continue
+
+                    # Try protobuf first (same logic as original)
+                    texts = []
+                    is_text_request = False
+
+                    try:
+                        req_proto = embedding_pb2.NodeEmbeddingRequest()
+                        req_proto.ParseFromString(message)
+                        node_ids = list(req_proto.node_ids)
+
+                        # Look up texts by node IDs
+                        for nid in node_ids:
+                            try:
+                                passage_data = passages.get_passage(str(nid))
+                                txt = passage_data["text"]
+                                if not txt:
+                                    raise RuntimeError(f"FATAL: Empty text for passage ID {nid}")
+                                texts.append(txt)
+                            except KeyError:
+                                raise RuntimeError(f"FATAL: Passage with ID {nid} not found")
+
+                        logger.info(f"ZMQ received protobuf request for {len(node_ids)} node IDs")
+                    except Exception:
+                        # Fallback to msgpack for text requests
+                        try:
+                            import msgpack
+
+                            request = msgpack.unpackb(message)
+                            if isinstance(request, list) and all(
+                                isinstance(item, str) for item in request
+                            ):
+                                texts = request
+                                is_text_request = True
+                                logger.info(
+                                    f"ZMQ received msgpack text request for {len(texts)} texts"
+                                )
+                            else:
+                                raise ValueError("Not a valid msgpack text request")
+                        except Exception:
+                            logger.error("Both protobuf and msgpack parsing failed!")
+                            # Send error response
+                            resp_proto = embedding_pb2.NodeEmbeddingResponse()
+                            rep_socket.send(resp_proto.SerializeToString())
+                            continue
+
+                    # Process the request
+                    embeddings = compute_embeddings(texts, model_name, mode=embedding_mode)
+                    logger.info(f"Computed embeddings shape: {embeddings.shape}")
+
+                    # Validation
+                    if np.isnan(embeddings).any() or np.isinf(embeddings).any():
+                        logger.error("NaN or Inf detected in embeddings!")
+                        # Send error response
+                        if is_text_request:
+                            import msgpack
+
+                            response_data = msgpack.packb([])
+                        else:
+                            resp_proto = embedding_pb2.NodeEmbeddingResponse()
+                            response_data = resp_proto.SerializeToString()
+                        rep_socket.send(response_data)
+                        continue
+
+                    # Prepare response based on request type
+                    if is_text_request:
+                        # For direct text requests, return msgpack
+                        import msgpack
+
+                        response_data = msgpack.packb(embeddings.tolist())
+                    else:
+                        # For protobuf requests, return protobuf
+                        resp_proto = embedding_pb2.NodeEmbeddingResponse()
+                        hidden_contiguous = np.ascontiguousarray(embeddings, dtype=np.float32)
+
+                        resp_proto.embeddings_data = hidden_contiguous.tobytes()
+                        resp_proto.dimensions.append(hidden_contiguous.shape[0])
+                        resp_proto.dimensions.append(hidden_contiguous.shape[1])
+
+                        response_data = resp_proto.SerializeToString()
+
+                    # Send response back to the client
+                    rep_socket.send(response_data)
+
+                    e2e_end = time.time()
+                    logger.info(f"⏱️  ZMQ E2E time: {e2e_end - e2e_start:.6f}s")
+
+                except zmq.Again:
+                    # Timeout - check shutdown_event and continue
+                    continue
+                except Exception as e:
+                    if not shutdown_event.is_set():
+                        logger.error(f"Error in ZMQ server loop: {e}")
+                        try:
+                            # Send error response for REP socket
+                            resp_proto = embedding_pb2.NodeEmbeddingResponse()
+                            rep_socket.send(resp_proto.SerializeToString())
+                        except Exception:
+                            pass
+                    else:
+                        logger.info("Shutdown in progress, ignoring ZMQ error")
+                        break
+        finally:
+            try:
+                rep_socket.close(0)
+            except Exception:
+                pass
+            try:
+                context.term()
+            except Exception:
+                pass
+
+        logger.info("DiskANN ZMQ server thread exiting gracefully")
+
+    # Add shutdown coordination
+    shutdown_event = threading.Event()
+
+    def shutdown_zmq_server():
+        """Gracefully shutdown ZMQ server."""
+        logger.info("Initiating graceful shutdown...")
+        shutdown_event.set()
+
+        if zmq_thread.is_alive():
+            logger.info("Waiting for ZMQ thread to finish...")
+            zmq_thread.join(timeout=5)
+            if zmq_thread.is_alive():
+                logger.warning("ZMQ thread did not finish in time")
+
+        # Clean up ZMQ resources
+        try:
+            # Note: socket and context are cleaned up by thread exit
+            logger.info("ZMQ resources cleaned up")
+        except Exception as e:
+            logger.warning(f"Error cleaning ZMQ resources: {e}")
+
+        # Clean up other resources
+        try:
+            import gc
+
+            gc.collect()
+            logger.info("Additional resources cleaned up")
+        except Exception as e:
+            logger.warning(f"Error cleaning additional resources: {e}")
+
+        logger.info("Graceful shutdown completed")
+        sys.exit(0)
+
+    # Register signal handlers within this function scope
+    import signal
+
+    def signal_handler(sig, frame):
+        logger.info(f"Received signal {sig}, shutting down gracefully...")
+        shutdown_zmq_server()
+
+    signal.signal(signal.SIGTERM, signal_handler)
+    signal.signal(signal.SIGINT, signal_handler)
+
+    # Start ZMQ thread (NOT daemon!)
+    zmq_thread = threading.Thread(
+        target=lambda: zmq_server_thread_with_shutdown(shutdown_event),
+        daemon=False,  # Not daemon - we want to wait for it
+    )
     zmq_thread.start()
     logger.info(f"Started DiskANN ZMQ server thread on port {zmq_port}")
 
     # Keep the main thread alive
     try:
-        while True:
-            time.sleep(1)
+        while not shutdown_event.is_set():
+            time.sleep(0.1)  # Check shutdown more frequently
     except KeyboardInterrupt:
         logger.info("DiskANN Server shutting down...")
+        shutdown_zmq_server()
         return
 
+    # If we reach here, shutdown was triggered by signal
+    logger.info("Main loop exited, process should be shutting down")
+
 
 if __name__ == "__main__":
-    import signal
     import sys
 
-    def signal_handler(sig, frame):
-        logger.info(f"Received signal {sig}, shutting down gracefully...")
-        sys.exit(0)
-
-    # Register signal handlers for graceful shutdown
-    signal.signal(signal.SIGTERM, signal_handler)
-    signal.signal(signal.SIGINT, signal_handler)
+    # Signal handlers are now registered within create_diskann_embedding_server
 
     parser = argparse.ArgumentParser(description="DiskANN Embedding service")
     parser.add_argument("--zmq-port", type=int, default=5555, help="ZMQ port to run on")

packages/leann-backend-diskann/leann_backend_diskann/graph_partition_simple.py

@@ -1,137 +0,0 @@
-#!/usr/bin/env python3
-"""
-Simplified Graph Partition Module for LEANN DiskANN Backend
-
-This module provides a simple Python interface for graph partitioning
-that directly calls the existing executables.
-"""
-
-import os
-import subprocess
-import tempfile
-from pathlib import Path
-from typing import Optional
-
-
-def partition_graph_simple(
-    index_prefix_path: str, output_dir: Optional[str] = None, **kwargs
-) -> tuple[str, str]:
-    """
-    Simple function to partition a graph index.
-
-    Args:
-        index_prefix_path: Path to the index prefix (e.g., "/path/to/index")
-        output_dir: Output directory (defaults to parent of index_prefix_path)
-        **kwargs: Additional parameters for graph partitioning
-
-    Returns:
-        Tuple of (disk_graph_index_path, partition_bin_path)
-    """
-    # Set default parameters
-    params = {
-        "gp_times": 10,
-        "lock_nums": 10,
-        "cut": 100,
-        "scale_factor": 1,
-        "data_type": "float",
-        "thread_nums": 10,
-        **kwargs,
-    }
-
-    # Determine output directory
-    if output_dir is None:
-        output_dir = str(Path(index_prefix_path).parent)
-
-    # Find the graph_partition directory
-    current_file = Path(__file__)
-    graph_partition_dir = current_file.parent.parent / "third_party" / "DiskANN" / "graph_partition"
-
-    if not graph_partition_dir.exists():
-        raise RuntimeError(f"Graph partition directory not found: {graph_partition_dir}")
-
-    # Find input index file
-    old_index_file = f"{index_prefix_path}_disk_beam_search.index"
-    if not os.path.exists(old_index_file):
-        old_index_file = f"{index_prefix_path}_disk.index"
-
-    if not os.path.exists(old_index_file):
-        raise RuntimeError(f"Index file not found: {old_index_file}")
-
-    # Create temporary directory for processing
-    with tempfile.TemporaryDirectory() as temp_dir:
-        temp_data_dir = Path(temp_dir) / "data"
-        temp_data_dir.mkdir(parents=True, exist_ok=True)
-
-        # Set up paths for temporary files
-        graph_path = temp_data_dir / "starling" / "_M_R_L_B" / "GRAPH"
-        graph_gp_path = (
-            graph_path
-            / f"GP_TIMES_{params['gp_times']}_LOCK_{params['lock_nums']}_GP_USE_FREQ0_CUT{params['cut']}_SCALE{params['scale_factor']}"
-        )
-        graph_gp_path.mkdir(parents=True, exist_ok=True)
-
-        # Run the build script with our parameters
-        cmd = [str(graph_partition_dir / "build.sh"), "release", "split_graph", index_prefix_path]
-
-        # Set environment variables for parameters
-        env = os.environ.copy()
-        env.update(
-            {
-                "GP_TIMES": str(params["gp_times"]),
-                "GP_LOCK_NUMS": str(params["lock_nums"]),
-                "GP_CUT": str(params["cut"]),
-                "GP_SCALE_F": str(params["scale_factor"]),
-                "DATA_TYPE": params["data_type"],
-                "GP_T": str(params["thread_nums"]),
-            }
-        )
-
-        print(f"Running graph partition with command: {' '.join(cmd)}")
-        print(f"Working directory: {graph_partition_dir}")
-
-        # Run the command
-        result = subprocess.run(
-            cmd, env=env, capture_output=True, text=True, cwd=graph_partition_dir
-        )
-
-        if result.returncode != 0:
-            print(f"Command failed with return code {result.returncode}")
-            print(f"stdout: {result.stdout}")
-            print(f"stderr: {result.stderr}")
-            raise RuntimeError(
-                f"Graph partitioning failed with return code {result.returncode}.\n"
-                f"stdout: {result.stdout}\n"
-                f"stderr: {result.stderr}"
-            )
-
-        # Check if output files were created
-        disk_graph_path = Path(output_dir) / "_disk_graph.index"
-        partition_bin_path = Path(output_dir) / "_partition.bin"
-
-        if not disk_graph_path.exists():
-            raise RuntimeError(f"Expected output file not found: {disk_graph_path}")
-
-        if not partition_bin_path.exists():
-            raise RuntimeError(f"Expected output file not found: {partition_bin_path}")
-
-        print("✅ Partitioning completed successfully!")
-        print(f"   Disk graph index: {disk_graph_path}")
-        print(f"   Partition binary: {partition_bin_path}")
-
-        return str(disk_graph_path), str(partition_bin_path)
-
-
-# Example usage
-if __name__ == "__main__":
-    try:
-        disk_graph_path, partition_bin_path = partition_graph_simple(
-            "/Users/yichuan/Desktop/release2/leann/diskannbuild/test_doc_files",
-            gp_times=5,
-            lock_nums=5,
-            cut=50,
-        )
-        print("Success! Output files:")
-        print(f"  - {disk_graph_path}")
-        print(f"  - {partition_bin_path}")
-    except Exception as e:
-        print(f"Error: {e}")

packages/leann-backend-diskann/pyproject.toml

@@ -1,11 +1,11 @@
 [build-system]
-requires = ["scikit-build-core>=0.10", "pybind11>=2.12.0", "numpy"]
+requires = ["scikit-build-core>=0.10", "pybind11>=2.12.0", "numpy", "cmake>=3.30"]
 build-backend = "scikit_build_core.build"
 
 [project]
 name = "leann-backend-diskann"
-version = "0.2.7"
-dependencies = ["leann-core==0.2.7", "numpy", "protobuf>=3.19.0"]
+version = "0.3.4"
+dependencies = ["leann-core==0.3.4", "numpy", "protobuf>=3.19.0"]
 
 [tool.scikit-build]
 # Key: simplified CMake path
@@ -17,3 +17,5 @@ editable.mode = "redirect"
 cmake.build-type = "Release"
 build.verbose = true
 build.tool-args = ["-j8"]
+# Let CMake find packages via Homebrew prefix
+cmake.define = {CMAKE_PREFIX_PATH = {env = "CMAKE_PREFIX_PATH"}, OpenMP_ROOT = {env = "OpenMP_ROOT"}}

packages/leann-backend-hnsw/CMakeLists.txt

@@ -5,11 +5,20 @@ set(CMAKE_CXX_COMPILER_WORKS 1)
 
 # Set OpenMP path for macOS
 if(APPLE)
-    set(OpenMP_C_FLAGS "-Xpreprocessor -fopenmp -I/opt/homebrew/opt/libomp/include")
-    set(OpenMP_CXX_FLAGS "-Xpreprocessor -fopenmp -I/opt/homebrew/opt/libomp/include")
+    # Detect Homebrew installation path (Apple Silicon vs Intel)
+    if(EXISTS "/opt/homebrew/opt/libomp")
+        set(HOMEBREW_PREFIX "/opt/homebrew")
+    elseif(EXISTS "/usr/local/opt/libomp")
+        set(HOMEBREW_PREFIX "/usr/local")
+    else()
+        message(FATAL_ERROR "Could not find libomp installation. Please install with: brew install libomp")
+    endif()
+
+    set(OpenMP_C_FLAGS "-Xpreprocessor -fopenmp -I${HOMEBREW_PREFIX}/opt/libomp/include")
+    set(OpenMP_CXX_FLAGS "-Xpreprocessor -fopenmp -I${HOMEBREW_PREFIX}/opt/libomp/include")
     set(OpenMP_C_LIB_NAMES "omp")
     set(OpenMP_CXX_LIB_NAMES "omp")
-    set(OpenMP_omp_LIBRARY "/opt/homebrew/opt/libomp/lib/libomp.dylib")
+    set(OpenMP_omp_LIBRARY "${HOMEBREW_PREFIX}/opt/libomp/lib/libomp.dylib")
 
     # Force use of system libc++ to avoid version mismatch
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
@@ -40,9 +49,28 @@ set(BUILD_TESTING OFF CACHE BOOL "" FORCE)
 set(FAISS_ENABLE_C_API OFF CACHE BOOL "" FORCE)
 set(FAISS_OPT_LEVEL "generic" CACHE STRING "" FORCE)
 
-# Disable additional SIMD versions to speed up compilation
+# Disable x86-specific SIMD optimizations (important for ARM64 compatibility)
 set(FAISS_ENABLE_AVX2 OFF CACHE BOOL "" FORCE)
 set(FAISS_ENABLE_AVX512 OFF CACHE BOOL "" FORCE)
+set(FAISS_ENABLE_SSE4_1 OFF CACHE BOOL "" FORCE)
+
+# ARM64-specific configuration
+if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64")
+    message(STATUS "Configuring Faiss for ARM64 architecture")
+
+    if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+        # Use SVE optimization level for ARM64 Linux (as seen in Faiss conda build)
+        set(FAISS_OPT_LEVEL "sve" CACHE STRING "" FORCE)
+        message(STATUS "Setting FAISS_OPT_LEVEL to 'sve' for ARM64 Linux")
+    else()
+        # Use generic optimization for other ARM64 platforms (like macOS)
+        set(FAISS_OPT_LEVEL "generic" CACHE STRING "" FORCE)
+        message(STATUS "Setting FAISS_OPT_LEVEL to 'generic' for ARM64 ${CMAKE_SYSTEM_NAME}")
+    endif()
+
+    # ARM64 compatibility: Faiss submodule has been modified to fix x86 header inclusion
+    message(STATUS "Using ARM64-compatible Faiss submodule")
+endif()
 
 # Additional optimization options from INSTALL.md
 set(CMAKE_BUILD_TYPE "Release" CACHE STRING "" FORCE)

packages/leann-backend-hnsw/leann_backend_hnsw/convert_to_csr.py

@@ -5,6 +5,8 @@ import os
 import struct
 import sys
 import time
+from dataclasses import dataclass
+from typing import Any, Optional
 
 import numpy as np
 
@@ -237,6 +239,288 @@ def write_compact_format(
         f_out.write(storage_data)
 
 
+@dataclass
+class HNSWComponents:
+    original_hnsw_data: dict[str, Any]
+    assign_probas_np: np.ndarray
+    cum_nneighbor_per_level_np: np.ndarray
+    levels_np: np.ndarray
+    is_compact: bool
+    compact_level_ptr: Optional[np.ndarray] = None
+    compact_node_offsets_np: Optional[np.ndarray] = None
+    compact_neighbors_data: Optional[list[int]] = None
+    offsets_np: Optional[np.ndarray] = None
+    neighbors_np: Optional[np.ndarray] = None
+    storage_fourcc: int = NULL_INDEX_FOURCC
+    storage_data: bytes = b""
+
+
+def _read_hnsw_structure(f) -> HNSWComponents:
+    original_hnsw_data: dict[str, Any] = {}
+
+    hnsw_index_fourcc = read_struct(f, "<I")
+    if hnsw_index_fourcc not in EXPECTED_HNSW_FOURCCS:
+        raise ValueError(
+            f"Unexpected HNSW FourCC: {hnsw_index_fourcc:08x}. Expected one of {EXPECTED_HNSW_FOURCCS}."
+        )
+
+    original_hnsw_data["index_fourcc"] = hnsw_index_fourcc
+    original_hnsw_data["d"] = read_struct(f, "<i")
+    original_hnsw_data["ntotal"] = read_struct(f, "<q")
+    original_hnsw_data["dummy1"] = read_struct(f, "<q")
+    original_hnsw_data["dummy2"] = read_struct(f, "<q")
+    original_hnsw_data["is_trained"] = read_struct(f, "?")
+    original_hnsw_data["metric_type"] = read_struct(f, "<i")
+    original_hnsw_data["metric_arg"] = 0.0
+    if original_hnsw_data["metric_type"] > 1:
+        original_hnsw_data["metric_arg"] = read_struct(f, "<f")
+
+    assign_probas_np = read_numpy_vector(f, np.float64, "d")
+    cum_nneighbor_per_level_np = read_numpy_vector(f, np.int32, "i")
+    levels_np = read_numpy_vector(f, np.int32, "i")
+
+    ntotal = len(levels_np)
+    if ntotal != original_hnsw_data["ntotal"]:
+        original_hnsw_data["ntotal"] = ntotal
+
+    pos_before_compact = f.tell()
+    is_compact_flag = None
+    try:
+        is_compact_flag = read_struct(f, "<?")
+    except EOFError:
+        is_compact_flag = None
+
+    if is_compact_flag:
+        compact_level_ptr = read_numpy_vector(f, np.uint64, "Q")
+        compact_node_offsets_np = read_numpy_vector(f, np.uint64, "Q")
+
+        original_hnsw_data["entry_point"] = read_struct(f, "<i")
+        original_hnsw_data["max_level"] = read_struct(f, "<i")
+        original_hnsw_data["efConstruction"] = read_struct(f, "<i")
+        original_hnsw_data["efSearch"] = read_struct(f, "<i")
+        original_hnsw_data["dummy_upper_beam"] = read_struct(f, "<i")
+
+        storage_fourcc = read_struct(f, "<I")
+        compact_neighbors_data_np = read_numpy_vector(f, np.int32, "i")
+        compact_neighbors_data = compact_neighbors_data_np.tolist()
+        storage_data = f.read()
+
+        return HNSWComponents(
+            original_hnsw_data=original_hnsw_data,
+            assign_probas_np=assign_probas_np,
+            cum_nneighbor_per_level_np=cum_nneighbor_per_level_np,
+            levels_np=levels_np,
+            is_compact=True,
+            compact_level_ptr=compact_level_ptr,
+            compact_node_offsets_np=compact_node_offsets_np,
+            compact_neighbors_data=compact_neighbors_data,
+            storage_fourcc=storage_fourcc,
+            storage_data=storage_data,
+        )
+
+    # Non-compact case
+    f.seek(pos_before_compact)
+
+    pos_before_probe = f.tell()
+    try:
+        suspected_flag = read_struct(f, "<B")
+        if suspected_flag != 0x00:
+            f.seek(pos_before_probe)
+    except EOFError:
+        f.seek(pos_before_probe)
+
+    offsets_np = read_numpy_vector(f, np.uint64, "Q")
+    neighbors_np = read_numpy_vector(f, np.int32, "i")
+
+    original_hnsw_data["entry_point"] = read_struct(f, "<i")
+    original_hnsw_data["max_level"] = read_struct(f, "<i")
+    original_hnsw_data["efConstruction"] = read_struct(f, "<i")
+    original_hnsw_data["efSearch"] = read_struct(f, "<i")
+    original_hnsw_data["dummy_upper_beam"] = read_struct(f, "<i")
+
+    storage_fourcc = NULL_INDEX_FOURCC
+    storage_data = b""
+    try:
+        storage_fourcc = read_struct(f, "<I")
+        storage_data = f.read()
+    except EOFError:
+        storage_fourcc = NULL_INDEX_FOURCC
+
+    return HNSWComponents(
+        original_hnsw_data=original_hnsw_data,
+        assign_probas_np=assign_probas_np,
+        cum_nneighbor_per_level_np=cum_nneighbor_per_level_np,
+        levels_np=levels_np,
+        is_compact=False,
+        offsets_np=offsets_np,
+        neighbors_np=neighbors_np,
+        storage_fourcc=storage_fourcc,
+        storage_data=storage_data,
+    )
+
+
+def _read_hnsw_structure_from_file(path: str) -> HNSWComponents:
+    with open(path, "rb") as f:
+        return _read_hnsw_structure(f)
+
+
+def write_original_format(
+    f_out,
+    original_hnsw_data,
+    assign_probas_np,
+    cum_nneighbor_per_level_np,
+    levels_np,
+    offsets_np,
+    neighbors_np,
+    storage_fourcc,
+    storage_data,
+):
+    """Write non-compact HNSW data in original FAISS order."""
+
+    f_out.write(struct.pack("<I", original_hnsw_data["index_fourcc"]))
+    f_out.write(struct.pack("<i", original_hnsw_data["d"]))
+    f_out.write(struct.pack("<q", original_hnsw_data["ntotal"]))
+    f_out.write(struct.pack("<q", original_hnsw_data["dummy1"]))
+    f_out.write(struct.pack("<q", original_hnsw_data["dummy2"]))
+    f_out.write(struct.pack("<?", original_hnsw_data["is_trained"]))
+    f_out.write(struct.pack("<i", original_hnsw_data["metric_type"]))
+    if original_hnsw_data["metric_type"] > 1:
+        f_out.write(struct.pack("<f", original_hnsw_data["metric_arg"]))
+
+    write_numpy_vector(f_out, assign_probas_np, "d")
+    write_numpy_vector(f_out, cum_nneighbor_per_level_np, "i")
+    write_numpy_vector(f_out, levels_np, "i")
+
+    write_numpy_vector(f_out, offsets_np, "Q")
+    write_numpy_vector(f_out, neighbors_np, "i")
+
+    f_out.write(struct.pack("<i", original_hnsw_data["entry_point"]))
+    f_out.write(struct.pack("<i", original_hnsw_data["max_level"]))
+    f_out.write(struct.pack("<i", original_hnsw_data["efConstruction"]))
+    f_out.write(struct.pack("<i", original_hnsw_data["efSearch"]))
+    f_out.write(struct.pack("<i", original_hnsw_data["dummy_upper_beam"]))
+
+    f_out.write(struct.pack("<I", storage_fourcc))
+    if storage_fourcc != NULL_INDEX_FOURCC and storage_data:
+        f_out.write(storage_data)
+
+
+def prune_hnsw_embeddings(input_filename: str, output_filename: str) -> bool:
+    """Rewrite an HNSW index while dropping the embedded storage section."""
+
+    start_time = time.time()
+    try:
+        with open(input_filename, "rb") as f_in, open(output_filename, "wb") as f_out:
+            original_hnsw_data: dict[str, Any] = {}
+
+            hnsw_index_fourcc = read_struct(f_in, "<I")
+            if hnsw_index_fourcc not in EXPECTED_HNSW_FOURCCS:
+                print(
+                    f"Error: Expected HNSW Index FourCC ({list(EXPECTED_HNSW_FOURCCS)}), got {hnsw_index_fourcc:08x}.",
+                    file=sys.stderr,
+                )
+                return False
+
+            original_hnsw_data["index_fourcc"] = hnsw_index_fourcc
+            original_hnsw_data["d"] = read_struct(f_in, "<i")
+            original_hnsw_data["ntotal"] = read_struct(f_in, "<q")
+            original_hnsw_data["dummy1"] = read_struct(f_in, "<q")
+            original_hnsw_data["dummy2"] = read_struct(f_in, "<q")
+            original_hnsw_data["is_trained"] = read_struct(f_in, "?")
+            original_hnsw_data["metric_type"] = read_struct(f_in, "<i")
+            original_hnsw_data["metric_arg"] = 0.0
+            if original_hnsw_data["metric_type"] > 1:
+                original_hnsw_data["metric_arg"] = read_struct(f_in, "<f")
+
+            assign_probas_np = read_numpy_vector(f_in, np.float64, "d")
+            cum_nneighbor_per_level_np = read_numpy_vector(f_in, np.int32, "i")
+            levels_np = read_numpy_vector(f_in, np.int32, "i")
+
+            ntotal = len(levels_np)
+            if ntotal != original_hnsw_data["ntotal"]:
+                original_hnsw_data["ntotal"] = ntotal
+
+            pos_before_compact = f_in.tell()
+            is_compact_flag = None
+            try:
+                is_compact_flag = read_struct(f_in, "<?")
+            except EOFError:
+                is_compact_flag = None
+
+            if is_compact_flag:
+                compact_level_ptr = read_numpy_vector(f_in, np.uint64, "Q")
+                compact_node_offsets_np = read_numpy_vector(f_in, np.uint64, "Q")
+
+                original_hnsw_data["entry_point"] = read_struct(f_in, "<i")
+                original_hnsw_data["max_level"] = read_struct(f_in, "<i")
+                original_hnsw_data["efConstruction"] = read_struct(f_in, "<i")
+                original_hnsw_data["efSearch"] = read_struct(f_in, "<i")
+                original_hnsw_data["dummy_upper_beam"] = read_struct(f_in, "<i")
+
+                _storage_fourcc = read_struct(f_in, "<I")
+                compact_neighbors_data_np = read_numpy_vector(f_in, np.int32, "i")
+                compact_neighbors_data = compact_neighbors_data_np.tolist()
+                _storage_data = f_in.read()
+
+                write_compact_format(
+                    f_out,
+                    original_hnsw_data,
+                    assign_probas_np,
+                    cum_nneighbor_per_level_np,
+                    levels_np,
+                    compact_level_ptr,
+                    compact_node_offsets_np,
+                    compact_neighbors_data,
+                    NULL_INDEX_FOURCC,
+                    b"",
+                )
+            else:
+                f_in.seek(pos_before_compact)
+
+                pos_before_probe = f_in.tell()
+                try:
+                    suspected_flag = read_struct(f_in, "<B")
+                    if suspected_flag != 0x00:
+                        f_in.seek(pos_before_probe)
+                except EOFError:
+                    f_in.seek(pos_before_probe)
+
+                offsets_np = read_numpy_vector(f_in, np.uint64, "Q")
+                neighbors_np = read_numpy_vector(f_in, np.int32, "i")
+
+                original_hnsw_data["entry_point"] = read_struct(f_in, "<i")
+                original_hnsw_data["max_level"] = read_struct(f_in, "<i")
+                original_hnsw_data["efConstruction"] = read_struct(f_in, "<i")
+                original_hnsw_data["efSearch"] = read_struct(f_in, "<i")
+                original_hnsw_data["dummy_upper_beam"] = read_struct(f_in, "<i")
+
+                _storage_fourcc = None
+                _storage_data = b""
+                try:
+                    _storage_fourcc = read_struct(f_in, "<I")
+                    _storage_data = f_in.read()
+                except EOFError:
+                    _storage_fourcc = NULL_INDEX_FOURCC
+
+                write_original_format(
+                    f_out,
+                    original_hnsw_data,
+                    assign_probas_np,
+                    cum_nneighbor_per_level_np,
+                    levels_np,
+                    offsets_np,
+                    neighbors_np,
+                    NULL_INDEX_FOURCC,
+                    b"",
+                )
+
+        print(f"[{time.time() - start_time:.2f}s] Pruned embeddings from {input_filename}")
+        return True
+    except Exception as exc:
+        print(f"Failed to prune embeddings: {exc}", file=sys.stderr)
+        return False
+
+
 # --- Main Conversion Logic ---
 
 
@@ -250,11 +534,7 @@ def convert_hnsw_graph_to_csr(input_filename, output_filename, prune_embeddings=
         output_filename: Output CSR index file
         prune_embeddings: Whether to prune embedding storage (write NULL storage marker)
     """
-    # Disable buffering for print statements to avoid deadlock in CI/pytest
-    import functools
-
-    global print
-    print = functools.partial(print, flush=True)
+    # Keep prints simple; rely on CI runner to flush output as needed
 
     print(f"Starting conversion: {input_filename} -> {output_filename}")
     start_time = time.time()
@@ -704,6 +984,29 @@ def convert_hnsw_graph_to_csr(input_filename, output_filename, prune_embeddings=
             pass
 
 
+def prune_hnsw_embeddings_inplace(index_filename: str) -> bool:
+    """Convenience wrapper to prune embeddings in-place."""
+
+    temp_path = f"{index_filename}.prune.tmp"
+    success = prune_hnsw_embeddings(index_filename, temp_path)
+    if success:
+        try:
+            os.replace(temp_path, index_filename)
+        except Exception as exc:  # pragma: no cover - defensive
+            logger.error(f"Failed to replace original index with pruned version: {exc}")
+            try:
+                os.remove(temp_path)
+            except OSError:
+                pass
+            return False
+    else:
+        try:
+            os.remove(temp_path)
+        except OSError:
+            pass
+    return success
+
+
 # --- Script Execution ---
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(

packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_backend.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import shutil
+import time
 from pathlib import Path
 from typing import Any, Literal, Optional
 
@@ -13,7 +14,7 @@ from leann.interface import (
 from leann.registry import register_backend
 from leann.searcher_base import BaseSearcher
 
-from .convert_to_csr import convert_hnsw_graph_to_csr
+from .convert_to_csr import convert_hnsw_graph_to_csr, prune_hnsw_embeddings_inplace
 
 logger = logging.getLogger(__name__)
 
@@ -54,12 +55,13 @@ class HNSWBuilder(LeannBackendBuilderInterface):
         self.efConstruction = self.build_params.setdefault("efConstruction", 200)
         self.distance_metric = self.build_params.setdefault("distance_metric", "mips")
         self.dimensions = self.build_params.get("dimensions")
-        if not self.is_recompute:
-            if self.is_compact:
-                # TODO: support this case @andy
-                raise ValueError(
-                    "is_recompute is False, but is_compact is True. This is not compatible now. change is compact to False and you can use the original HNSW index."
-                )
+        if not self.is_recompute and self.is_compact:
+            # Auto-correct: non-recompute requires non-compact storage for HNSW
+            logger.warning(
+                "is_recompute=False requires non-compact HNSW. Forcing is_compact=False."
+            )
+            self.is_compact = False
+            self.build_params["is_compact"] = False
 
     def build(self, data: np.ndarray, ids: list[str], index_path: str, **kwargs):
         from . import faiss  # type: ignore
@@ -90,6 +92,8 @@ class HNSWBuilder(LeannBackendBuilderInterface):
 
         if self.is_compact:
             self._convert_to_csr(index_file)
+        elif self.is_recompute:
+            prune_hnsw_embeddings_inplace(str(index_file))
 
     def _convert_to_csr(self, index_file: Path):
         """Convert built index to CSR format"""
@@ -131,10 +135,10 @@ class HNSWSearcher(BaseSearcher):
         if metric_enum is None:
             raise ValueError(f"Unsupported distance_metric '{self.distance_metric}'.")
 
-        self.is_compact, self.is_pruned = (
-            self.meta.get("is_compact", True),
-            self.meta.get("is_pruned", True),
-        )
+        backend_meta_kwargs = self.meta.get("backend_kwargs", {})
+        self.is_compact = self.meta.get("is_compact", backend_meta_kwargs.get("is_compact", True))
+        default_pruned = backend_meta_kwargs.get("is_recompute", self.is_compact)
+        self.is_pruned = bool(self.meta.get("is_pruned", default_pruned))
 
         index_file = self.index_dir / f"{self.index_path.stem}.index"
         if not index_file.exists():
@@ -184,9 +188,11 @@ class HNSWSearcher(BaseSearcher):
         """
         from . import faiss  # type: ignore
 
-        if not recompute_embeddings:
-            if self.is_pruned:
-                raise RuntimeError("Recompute is required for pruned index.")
+        if not recompute_embeddings and self.is_pruned:
+            raise RuntimeError(
+                "Recompute is required for pruned/compact HNSW index. "
+                "Re-run search with --recompute, or rebuild with --no-recompute and --no-compact."
+            )
         if recompute_embeddings:
             if zmq_port is None:
                 raise ValueError("zmq_port must be provided if recompute_embeddings is True")
@@ -233,6 +239,7 @@ class HNSWSearcher(BaseSearcher):
         distances = np.empty((batch_size_query, top_k), dtype=np.float32)
         labels = np.empty((batch_size_query, top_k), dtype=np.int64)
 
+        search_time = time.time()
         self._index.search(
             query.shape[0],
             faiss.swig_ptr(query),
@@ -241,29 +248,8 @@ class HNSWSearcher(BaseSearcher):
             faiss.swig_ptr(labels),
             params,
         )
-
+        search_time = time.time() - search_time
+        logger.info(f"  Search time in HNSWSearcher.search() backend: {search_time} seconds")
         string_labels = [[str(int_label) for int_label in batch_labels] for batch_labels in labels]
 
         return {"labels": string_labels, "distances": distances}
-
-    def cleanup(self):
-        """Cleanup HNSW-specific resources including C++ ZMQ connections."""
-        # Call parent cleanup first
-        super().cleanup()
-
-        # Additional cleanup for C++ side ZMQ connections
-        # The ZmqDistanceComputer in C++ uses ZMQ connections that need cleanup
-        try:
-            # Delete the index to trigger C++ destructors
-            if hasattr(self, "index"):
-                del self.index
-        except Exception:
-            pass
-
-        # Force garbage collection to ensure C++ objects are destroyed
-        try:
-            import gc
-
-            gc.collect()
-        except Exception:
-            pass

packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_embedding_server.py

@@ -24,13 +24,26 @@ logger = logging.getLogger(__name__)
 log_level = getattr(logging, LOG_LEVEL, logging.WARNING)
 logger.setLevel(log_level)
 
-# Ensure we have a handler if none exists
+# Ensure we have handlers if none exist
 if not logger.handlers:
-    handler = logging.StreamHandler()
+    stream_handler = logging.StreamHandler()
     formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
-    handler.setFormatter(formatter)
-    logger.addHandler(handler)
-    logger.propagate = False
+    stream_handler.setFormatter(formatter)
+    logger.addHandler(stream_handler)
+
+log_path = os.getenv("LEANN_HNSW_LOG_PATH")
+if log_path:
+    try:
+        file_handler = logging.FileHandler(log_path, mode="a", encoding="utf-8")
+        file_formatter = logging.Formatter(
+            "%(asctime)s - %(levelname)s - [pid=%(process)d] %(message)s"
+        )
+        file_handler.setFormatter(file_formatter)
+        logger.addHandler(file_handler)
+    except Exception as exc:  # pragma: no cover - best effort logging
+        logger.warning(f"Failed to attach file handler for log path {log_path}: {exc}")
+
+logger.propagate = False
 
 
 def create_hnsw_embedding_server(
@@ -82,189 +95,315 @@ def create_hnsw_embedding_server(
     with open(passages_file) as f:
         meta = json.load(f)
 
-    # Let PassageManager handle path resolution uniformly
+    # Let PassageManager handle path resolution uniformly. It supports fallback order:
+    # 1) path/index_path; 2) *_relative; 3) standard siblings next to meta
     passages = PassageManager(meta["passage_sources"], metadata_file_path=passages_file)
-    logger.info(
-        f"Loaded PassageManager with {len(passages.global_offset_map)} passages from metadata"
-    )
+    # Dimension from metadata for shaping responses
+    try:
+        embedding_dim: int = int(meta.get("dimensions", 0))
+    except Exception:
+        embedding_dim = 0
+    logger.info(f"Loaded PassageManager with {len(passages)} passages from metadata")
+
+    # (legacy ZMQ thread removed; using shutdown-capable server only)
+
+    def zmq_server_thread_with_shutdown(shutdown_event):
+        """ZMQ server thread that respects shutdown signal.
+
+        Creates its own REP socket bound to zmq_port and polls with timeouts
+        to allow graceful shutdown.
+        """
+        logger.info("ZMQ server thread started with shutdown support")
 
-    def zmq_server_thread():
-        """ZMQ server thread"""
         context = zmq.Context()
-        socket = context.socket(zmq.REP)
-        socket.setsockopt(zmq.LINGER, 0)  # Don't block on close
-        socket.bind(f"tcp://*:{zmq_port}")
-        logger.info(f"HNSW ZMQ server listening on port {zmq_port}")
+        rep_socket = context.socket(zmq.REP)
+        rep_socket.bind(f"tcp://*:{zmq_port}")
+        logger.info(f"HNSW ZMQ REP server listening on port {zmq_port}")
+        rep_socket.setsockopt(zmq.RCVTIMEO, 1000)
+        # Keep sends from blocking during shutdown; fail fast and drop on close
+        rep_socket.setsockopt(zmq.SNDTIMEO, 1000)
+        rep_socket.setsockopt(zmq.LINGER, 0)
 
-        socket.setsockopt(zmq.RCVTIMEO, 300000)
-        socket.setsockopt(zmq.SNDTIMEO, 300000)
+        # Track last request type/length for shape-correct fallbacks
+        last_request_type = "unknown"  # 'text' | 'distance' | 'embedding' | 'unknown'
+        last_request_length = 0
 
-        while True:
-            try:
-                message_bytes = socket.recv()
-                logger.debug(f"Received ZMQ request of size {len(message_bytes)} bytes")
+        try:
+            while not shutdown_event.is_set():
+                try:
+                    e2e_start = time.time()
+                    logger.debug("🔍 Waiting for ZMQ message...")
+                    request_bytes = rep_socket.recv()
 
-                e2e_start = time.time()
-                request_payload = msgpack.unpackb(message_bytes)
+                    # Rest of the processing logic (same as original)
+                    request = msgpack.unpackb(request_bytes)
 
-                # Handle direct text embedding request
-                if isinstance(request_payload, list) and len(request_payload) > 0:
-                    # Check if this is a direct text request (list of strings)
-                    if all(isinstance(item, str) for item in request_payload):
-                        logger.info(
-                            f"Processing direct text embedding request for {len(request_payload)} texts in {embedding_mode} mode"
-                        )
+                    if len(request) == 1 and request[0] == "__QUERY_MODEL__":
+                        response_bytes = msgpack.packb([model_name])
+                        rep_socket.send(response_bytes)
+                        continue
 
-                        # Use unified embedding computation (now with model caching)
-                        embeddings = compute_embeddings(
-                            request_payload, model_name, mode=embedding_mode
-                        )
-
-                        response = embeddings.tolist()
-                        socket.send(msgpack.packb(response))
+                    # Handle direct text embedding request
+                    if (
+                        isinstance(request, list)
+                        and request
+                        and all(isinstance(item, str) for item in request)
+                    ):
+                        last_request_type = "text"
+                        last_request_length = len(request)
+                        embeddings = compute_embeddings(request, model_name, mode=embedding_mode)
+                        rep_socket.send(msgpack.packb(embeddings.tolist()))
                         e2e_end = time.time()
                         logger.info(f"⏱️  Text embedding E2E time: {e2e_end - e2e_start:.6f}s")
                         continue
 
-                # Handle distance calculation requests
-                if (
-                    isinstance(request_payload, list)
-                    and len(request_payload) == 2
-                    and isinstance(request_payload[0], list)
-                    and isinstance(request_payload[1], list)
-                ):
-                    node_ids = request_payload[0]
-                    query_vector = np.array(request_payload[1], dtype=np.float32)
+                    # Handle distance calculation request: [[ids], [query_vector]]
+                    if (
+                        isinstance(request, list)
+                        and len(request) == 2
+                        and isinstance(request[0], list)
+                        and isinstance(request[1], list)
+                    ):
+                        node_ids = request[0]
+                        # Handle nested [[ids]] shape defensively
+                        if len(node_ids) == 1 and isinstance(node_ids[0], list):
+                            node_ids = node_ids[0]
+                        query_vector = np.array(request[1], dtype=np.float32)
+                        last_request_type = "distance"
+                        last_request_length = len(node_ids)
 
-                    logger.debug("Distance calculation request received")
-                    logger.debug(f"    Node IDs: {node_ids}")
-                    logger.debug(f"    Query vector dim: {len(query_vector)}")
+                        logger.debug("Distance calculation request received")
+                        logger.debug(f"    Node IDs: {node_ids}")
+                        logger.debug(f"    Query vector dim: {len(query_vector)}")
 
-                    # Get embeddings for node IDs
-                    texts = []
-                    for nid in node_ids:
+                        # Gather texts for found ids
+                        texts: list[str] = []
+                        found_indices: list[int] = []
+                        for idx, nid in enumerate(node_ids):
+                            try:
+                                passage_data = passages.get_passage(str(nid))
+                                txt = passage_data.get("text", "")
+                                if isinstance(txt, str) and len(txt) > 0:
+                                    texts.append(txt)
+                                    found_indices.append(idx)
+                                else:
+                                    logger.error(f"Empty text for passage ID {nid}")
+                            except KeyError:
+                                logger.error(f"Passage ID {nid} not found")
+                            except Exception as e:
+                                logger.error(f"Exception looking up passage ID {nid}: {e}")
+
+                        # Prepare full-length response with large sentinel values
+                        large_distance = 1e9
+                        response_distances = [large_distance] * len(node_ids)
+
+                        if texts:
+                            try:
+                                embeddings = compute_embeddings(
+                                    texts, model_name, mode=embedding_mode
+                                )
+                                logger.info(
+                                    f"Computed embeddings for {len(texts)} texts, shape: {embeddings.shape}"
+                                )
+                                if distance_metric == "l2":
+                                    partial = np.sum(
+                                        np.square(embeddings - query_vector.reshape(1, -1)), axis=1
+                                    )
+                                else:  # mips or cosine
+                                    partial = -np.dot(embeddings, query_vector)
+
+                                for pos, dval in zip(found_indices, partial.flatten().tolist()):
+                                    response_distances[pos] = float(dval)
+                            except Exception as e:
+                                logger.error(f"Distance computation error, using sentinels: {e}")
+
+                        # Send response in expected shape [[distances]]
+                        rep_socket.send(msgpack.packb([response_distances], use_single_float=True))
+                        e2e_end = time.time()
+                        logger.info(f"⏱️  Distance calculation E2E time: {e2e_end - e2e_start:.6f}s")
+                        continue
+
+                    # Fallback: treat as embedding-by-id request
+                    if (
+                        isinstance(request, list)
+                        and len(request) == 1
+                        and isinstance(request[0], list)
+                    ):
+                        node_ids = request[0]
+                    elif isinstance(request, list):
+                        node_ids = request
+                    else:
+                        node_ids = []
+                    last_request_type = "embedding"
+                    last_request_length = len(node_ids)
+                    logger.info(f"ZMQ received {len(node_ids)} node IDs for embedding fetch")
+
+                    # Preallocate zero-filled flat data for robustness
+                    if embedding_dim <= 0:
+                        dims = [0, 0]
+                        flat_data: list[float] = []
+                    else:
+                        dims = [len(node_ids), embedding_dim]
+                        flat_data = [0.0] * (dims[0] * dims[1])
+
+                    # Collect texts for found ids
+                    texts: list[str] = []
+                    found_indices: list[int] = []
+                    for idx, nid in enumerate(node_ids):
                         try:
                             passage_data = passages.get_passage(str(nid))
-                            txt = passage_data["text"]
-                            texts.append(txt)
+                            txt = passage_data.get("text", "")
+                            if isinstance(txt, str) and len(txt) > 0:
+                                texts.append(txt)
+                                found_indices.append(idx)
+                            else:
+                                logger.error(f"Empty text for passage ID {nid}")
                         except KeyError:
-                            logger.error(f"Passage ID {nid} not found")
-                            raise RuntimeError(f"FATAL: Passage with ID {nid} not found")
+                            logger.error(f"Passage with ID {nid} not found")
                         except Exception as e:
                             logger.error(f"Exception looking up passage ID {nid}: {e}")
-                            raise
 
-                    # Process embeddings
-                    embeddings = compute_embeddings(texts, model_name, mode=embedding_mode)
-                    logger.info(
-                        f"Computed embeddings for {len(texts)} texts, shape: {embeddings.shape}"
-                    )
+                    if texts:
+                        try:
+                            embeddings = compute_embeddings(texts, model_name, mode=embedding_mode)
+                            logger.info(
+                                f"Computed embeddings for {len(texts)} texts, shape: {embeddings.shape}"
+                            )
 
-                    # Calculate distances
-                    if distance_metric == "l2":
-                        distances = np.sum(
-                            np.square(embeddings - query_vector.reshape(1, -1)), axis=1
-                        )
-                    else:  # mips or cosine
-                        distances = -np.dot(embeddings, query_vector)
+                            if np.isnan(embeddings).any() or np.isinf(embeddings).any():
+                                logger.error(
+                                    f"NaN or Inf detected in embeddings! Requested IDs: {node_ids[:5]}..."
+                                )
+                                dims = [0, embedding_dim]
+                                flat_data = []
+                            else:
+                                emb_f32 = np.ascontiguousarray(embeddings, dtype=np.float32)
+                                flat = emb_f32.flatten().tolist()
+                                for j, pos in enumerate(found_indices):
+                                    start = pos * embedding_dim
+                                    end = start + embedding_dim
+                                    if end <= len(flat_data):
+                                        flat_data[start:end] = flat[
+                                            j * embedding_dim : (j + 1) * embedding_dim
+                                        ]
+                        except Exception as e:
+                            logger.error(f"Embedding computation error, returning zeros: {e}")
 
-                    response_payload = distances.flatten().tolist()
-                    response_bytes = msgpack.packb([response_payload], use_single_float=True)
-                    logger.debug(f"Sending distance response with {len(distances)} distances")
+                    response_payload = [dims, flat_data]
+                    response_bytes = msgpack.packb(response_payload, use_single_float=True)
 
-                    socket.send(response_bytes)
+                    rep_socket.send(response_bytes)
                     e2e_end = time.time()
-                    logger.info(f"⏱️  Distance calculation E2E time: {e2e_end - e2e_start:.6f}s")
+                    logger.info(f"⏱️  ZMQ E2E time: {e2e_end - e2e_start:.6f}s")
+
+                except zmq.Again:
+                    # Timeout - check shutdown_event and continue
                     continue
+                except Exception as e:
+                    if not shutdown_event.is_set():
+                        logger.error(f"Error in ZMQ server loop: {e}")
+                        # Shape-correct fallback
+                        try:
+                            if last_request_type == "distance":
+                                large_distance = 1e9
+                                fallback_len = max(0, int(last_request_length))
+                                safe = [[large_distance] * fallback_len]
+                            elif last_request_type == "embedding":
+                                bsz = max(0, int(last_request_length))
+                                dim = max(0, int(embedding_dim))
+                                safe = (
+                                    [[bsz, dim], [0.0] * (bsz * dim)] if dim > 0 else [[0, 0], []]
+                                )
+                            elif last_request_type == "text":
+                                safe = []  # direct text embeddings expectation is a flat list
+                            else:
+                                safe = [[0, int(embedding_dim) if embedding_dim > 0 else 0], []]
+                            rep_socket.send(msgpack.packb(safe, use_single_float=True))
+                        except Exception:
+                            pass
+                    else:
+                        logger.info("Shutdown in progress, ignoring ZMQ error")
+                        break
+        finally:
+            try:
+                rep_socket.close(0)
+            except Exception:
+                pass
+            try:
+                context.term()
+            except Exception:
+                pass
 
-                # Standard embedding request (passage ID lookup)
-                if (
-                    not isinstance(request_payload, list)
-                    or len(request_payload) != 1
-                    or not isinstance(request_payload[0], list)
-                ):
-                    logger.error(
-                        f"Invalid MessagePack request format. Expected [[ids...]] or [texts...], got: {type(request_payload)}"
-                    )
-                    socket.send(msgpack.packb([[], []]))
-                    continue
+        logger.info("ZMQ server thread exiting gracefully")
 
-                node_ids = request_payload[0]
-                logger.debug(f"Request for {len(node_ids)} node embeddings")
+    # Add shutdown coordination
+    shutdown_event = threading.Event()
 
-                # Look up texts by node IDs
-                texts = []
-                for nid in node_ids:
-                    try:
-                        passage_data = passages.get_passage(str(nid))
-                        txt = passage_data["text"]
-                        if not txt:
-                            raise RuntimeError(f"FATAL: Empty text for passage ID {nid}")
-                        texts.append(txt)
-                    except KeyError:
-                        raise RuntimeError(f"FATAL: Passage with ID {nid} not found")
-                    except Exception as e:
-                        logger.error(f"Exception looking up passage ID {nid}: {e}")
-                        raise
+    def shutdown_zmq_server():
+        """Gracefully shutdown ZMQ server."""
+        logger.info("Initiating graceful shutdown...")
+        shutdown_event.set()
 
-                # Process embeddings
-                embeddings = compute_embeddings(texts, model_name, mode=embedding_mode)
-                logger.info(
-                    f"Computed embeddings for {len(texts)} texts, shape: {embeddings.shape}"
-                )
+        if zmq_thread.is_alive():
+            logger.info("Waiting for ZMQ thread to finish...")
+            zmq_thread.join(timeout=5)
+            if zmq_thread.is_alive():
+                logger.warning("ZMQ thread did not finish in time")
 
-                # Serialization and response
-                if np.isnan(embeddings).any() or np.isinf(embeddings).any():
-                    logger.error(
-                        f"NaN or Inf detected in embeddings! Requested IDs: {node_ids[:5]}..."
-                    )
-                    raise AssertionError()
+        # Clean up ZMQ resources
+        try:
+            # Note: socket and context are cleaned up by thread exit
+            logger.info("ZMQ resources cleaned up")
+        except Exception as e:
+            logger.warning(f"Error cleaning ZMQ resources: {e}")
 
-                hidden_contiguous_f32 = np.ascontiguousarray(embeddings, dtype=np.float32)
-                response_payload = [
-                    list(hidden_contiguous_f32.shape),
-                    hidden_contiguous_f32.flatten().tolist(),
-                ]
-                response_bytes = msgpack.packb(response_payload, use_single_float=True)
+        # Clean up other resources
+        try:
+            import gc
 
-                socket.send(response_bytes)
-                e2e_end = time.time()
-                logger.info(f"⏱️  ZMQ E2E time: {e2e_end - e2e_start:.6f}s")
+            gc.collect()
+            logger.info("Additional resources cleaned up")
+        except Exception as e:
+            logger.warning(f"Error cleaning additional resources: {e}")
 
-            except zmq.Again:
-                logger.debug("ZMQ socket timeout, continuing to listen")
-                continue
-            except Exception as e:
-                logger.error(f"Error in ZMQ server loop: {e}")
-                import traceback
+        logger.info("Graceful shutdown completed")
+        sys.exit(0)
 
-                traceback.print_exc()
-                socket.send(msgpack.packb([[], []]))
+    # Register signal handlers within this function scope
+    import signal
 
-    zmq_thread = threading.Thread(target=zmq_server_thread, daemon=True)
+    def signal_handler(sig, frame):
+        logger.info(f"Received signal {sig}, shutting down gracefully...")
+        shutdown_zmq_server()
+
+    signal.signal(signal.SIGTERM, signal_handler)
+    signal.signal(signal.SIGINT, signal_handler)
+
+    # Pass shutdown_event to ZMQ thread
+    zmq_thread = threading.Thread(
+        target=lambda: zmq_server_thread_with_shutdown(shutdown_event),
+        daemon=False,  # Not daemon - we want to wait for it
+    )
     zmq_thread.start()
     logger.info(f"Started HNSW ZMQ server thread on port {zmq_port}")
 
     # Keep the main thread alive
     try:
-        while True:
-            time.sleep(1)
+        while not shutdown_event.is_set():
+            time.sleep(0.1)  # Check shutdown more frequently
     except KeyboardInterrupt:
         logger.info("HNSW Server shutting down...")
+        shutdown_zmq_server()
         return
 
+    # If we reach here, shutdown was triggered by signal
+    logger.info("Main loop exited, process should be shutting down")
+
 
 if __name__ == "__main__":
-    import signal
     import sys
 
-    def signal_handler(sig, frame):
-        logger.info(f"Received signal {sig}, shutting down gracefully...")
-        sys.exit(0)
-
-    # Register signal handlers for graceful shutdown
-    signal.signal(signal.SIGTERM, signal_handler)
-    signal.signal(signal.SIGINT, signal_handler)
+    # Signal handlers are now registered within create_hnsw_embedding_server
 
     parser = argparse.ArgumentParser(description="HNSW Embedding service")
     parser.add_argument("--zmq-port", type=int, default=5555, help="ZMQ port to run on")

packages/leann-backend-hnsw/pyproject.toml

@@ -6,10 +6,10 @@ build-backend = "scikit_build_core.build"
 
 [project]
 name = "leann-backend-hnsw"
-version = "0.2.7"
+version = "0.3.4"
 description = "Custom-built HNSW (Faiss) backend for the Leann toolkit."
 dependencies = [
-    "leann-core==0.2.7",
+    "leann-core==0.3.4",
     "numpy",
     "pyzmq>=23.0.0",
     "msgpack>=1.0.0",
@@ -22,6 +22,8 @@ cmake.build-type = "Release"
 build.verbose = true
 build.tool-args = ["-j8"]
 
-# CMake definitions to optimize compilation
+# CMake definitions to optimize compilation and find Homebrew packages
 [tool.scikit-build.cmake.define]
 CMAKE_BUILD_PARALLEL_LEVEL = "8"
+CMAKE_PREFIX_PATH = {env = "CMAKE_PREFIX_PATH"}
+OpenMP_ROOT = {env = "OpenMP_ROOT"}

packages/leann-core/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "leann-core"
-version = "0.2.7"
+version = "0.3.4"
 description = "Core API and plugin system for LEANN"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -33,8 +33,8 @@ dependencies = [
     "pdfplumber>=0.10.0",
     "nbconvert>=7.0.0",  # For .ipynb file support
     "gitignore-parser>=0.1.12",  # For proper .gitignore handling
-    "mlx>=0.26.3; sys_platform == 'darwin'",
-    "mlx-lm>=0.26.0; sys_platform == 'darwin'",
+    "mlx>=0.26.3; sys_platform == 'darwin' and platform_machine == 'arm64'",
+    "mlx-lm>=0.26.0; sys_platform == 'darwin' and platform_machine == 'arm64'",
 ]
 
 [project.optional-dependencies]

packages/leann-core/src/leann/api.py

@@ -6,18 +6,22 @@ with the correct, original embedding logic from the user's reference code.
 import json
 import logging
 import pickle
+import re
+import subprocess
 import time
 import warnings
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Any, Literal, Optional
+from typing import Any, Literal, Optional, Union
 
 import numpy as np
+from leann_backend_hnsw.convert_to_csr import prune_hnsw_embeddings_inplace
 
 from leann.interface import LeannBackendSearcherInterface
 
 from .chat import get_llm
 from .interface import LeannBackendFactoryInterface
+from .metadata_filter import MetadataFilterEngine
 from .registry import BACKEND_REGISTRY
 
 logger = logging.getLogger(__name__)
@@ -46,6 +50,7 @@ def compute_embeddings(
             - "sentence-transformers": Use sentence-transformers library (default)
             - "mlx": Use MLX backend for Apple Silicon
             - "openai": Use OpenAI embedding API
+            - "gemini": Use Google Gemini embedding API
         use_server: Whether to use embedding server (True for search, False for build)
 
     Returns:
@@ -87,26 +92,21 @@ def compute_embeddings_via_server(chunks: list[str], model_name: str, port: int)
     # Connect to embedding server
     context = zmq.Context()
     socket = context.socket(zmq.REQ)
-    socket.setsockopt(zmq.LINGER, 0)  # Don't block on close
-    socket.setsockopt(zmq.RCVTIMEO, 1000)  # 1s timeout on receive
-    socket.setsockopt(zmq.SNDTIMEO, 1000)  # 1s timeout on send
-    socket.setsockopt(zmq.IMMEDIATE, 1)  # Don't wait for connection
     socket.connect(f"tcp://localhost:{port}")
 
-    try:
-        # Send chunks to server for embedding computation
-        request = chunks
-        socket.send(msgpack.packb(request))
+    # Send chunks to server for embedding computation
+    request = chunks
+    socket.send(msgpack.packb(request))
 
-        # Receive embeddings from server
-        response = socket.recv()
-        embeddings_list = msgpack.unpackb(response)
+    # Receive embeddings from server
+    response = socket.recv()
+    embeddings_list = msgpack.unpackb(response)
 
-        # Convert back to numpy array
-        embeddings = np.array(embeddings_list, dtype=np.float32)
-    finally:
-        socket.close(linger=0)
-        context.term()
+    # Convert back to numpy array
+    embeddings = np.array(embeddings_list, dtype=np.float32)
+
+    socket.close()
+    context.term()
 
     return embeddings
 
@@ -123,57 +123,153 @@ class PassageManager:
     def __init__(
         self, passage_sources: list[dict[str, Any]], metadata_file_path: Optional[str] = None
     ):
-        self.offset_maps = {}
-        self.passage_files = {}
-        self.global_offset_map = {}  # Combined map for fast lookup
+        self.offset_maps: dict[str, dict[str, int]] = {}
+        self.passage_files: dict[str, str] = {}
+        # Avoid materializing a single gigantic global map to reduce memory
+        # footprint on very large corpora (e.g., 60M+ passages). Instead, keep
+        # per-shard maps and do a lightweight per-shard lookup on demand.
+        self._total_count: int = 0
+        self.filter_engine = MetadataFilterEngine()  # Initialize filter engine
+
+        # Derive index base name for standard sibling fallbacks, e.g., <index_name>.passages.*
+        index_name_base = None
+        if metadata_file_path:
+            meta_name = Path(metadata_file_path).name
+            if meta_name.endswith(".meta.json"):
+                index_name_base = meta_name[: -len(".meta.json")]
 
         for source in passage_sources:
             assert source["type"] == "jsonl", "only jsonl is supported"
-            passage_file = source["path"]
-            index_file = source["index_path"]  # .idx file
+            passage_file = source.get("path", "")
+            index_file = source.get("index_path", "")  # .idx file
 
             # Fix path resolution - relative paths should be relative to metadata file directory
-            if not Path(index_file).is_absolute():
-                if metadata_file_path:
-                    # Resolve relative to metadata file directory
-                    metadata_dir = Path(metadata_file_path).parent
-                    logger.debug(
-                        f"PassageManager: Resolving relative paths from metadata_dir: {metadata_dir}"
-                    )
-                    index_file = str((metadata_dir / index_file).resolve())
-                    passage_file = str((metadata_dir / passage_file).resolve())
-                    logger.debug(f"PassageManager: Resolved index_file: {index_file}")
-                else:
-                    # Fallback to current directory resolution (legacy behavior)
-                    logger.warning(
-                        "PassageManager: No metadata_file_path provided, using fallback resolution from cwd"
-                    )
-                    logger.debug(f"PassageManager: Current working directory: {Path.cwd()}")
-                    index_file = str(Path(index_file).resolve())
-                    passage_file = str(Path(passage_file).resolve())
-                    logger.debug(f"PassageManager: Fallback resolved index_file: {index_file}")
+            def _resolve_candidates(
+                primary: str,
+                relative_key: str,
+                default_name: Optional[str],
+                source_dict: dict[str, Any],
+            ) -> list[Path]:
+                """
+                Build an ordered list of candidate paths. For relative paths specified in
+                metadata, prefer resolution relative to the metadata file directory first,
+                then fall back to CWD-based resolution, and finally to conventional
+                sibling defaults (e.g., <index_base>.passages.idx / .jsonl).
+                """
+                candidates: list[Path] = []
+                # 1) Primary path
+                if primary:
+                    p = Path(primary)
+                    if p.is_absolute():
+                        candidates.append(p)
+                    else:
+                        # Prefer metadata-relative resolution for relative paths
+                        if metadata_file_path:
+                            candidates.append(Path(metadata_file_path).parent / p)
+                        # Also consider CWD-relative as a fallback for legacy layouts
+                        candidates.append(Path.cwd() / p)
+                # 2) metadata-relative explicit relative key (if present)
+                if metadata_file_path and source_dict.get(relative_key):
+                    candidates.append(Path(metadata_file_path).parent / source_dict[relative_key])
+                # 3) metadata-relative standard sibling filename
+                if metadata_file_path and default_name:
+                    candidates.append(Path(metadata_file_path).parent / default_name)
+                return candidates
+
+            # Build candidate lists and pick first existing; otherwise keep last candidate for error message
+            idx_default = f"{index_name_base}.passages.idx" if index_name_base else None
+            idx_candidates = _resolve_candidates(
+                index_file, "index_path_relative", idx_default, source
+            )
+            pas_default = f"{index_name_base}.passages.jsonl" if index_name_base else None
+            pas_candidates = _resolve_candidates(passage_file, "path_relative", pas_default, source)
+
+            def _pick_existing(cands: list[Path]) -> str:
+                for c in cands:
+                    if c.exists():
+                        return str(c.resolve())
+                # Fallback to last candidate (best guess) even if not exists; will error below
+                return str(cands[-1].resolve()) if cands else ""
+
+            index_file = _pick_existing(idx_candidates)
+            passage_file = _pick_existing(pas_candidates)
 
             if not Path(index_file).exists():
                 raise FileNotFoundError(f"Passage index file not found: {index_file}")
 
             with open(index_file, "rb") as f:
-                offset_map = pickle.load(f)
+                offset_map: dict[str, int] = pickle.load(f)
                 self.offset_maps[passage_file] = offset_map
                 self.passage_files[passage_file] = passage_file
-
-                # Build global map for O(1) lookup
-                for passage_id, offset in offset_map.items():
-                    self.global_offset_map[passage_id] = (passage_file, offset)
+                self._total_count += len(offset_map)
 
     def get_passage(self, passage_id: str) -> dict[str, Any]:
-        if passage_id in self.global_offset_map:
-            passage_file, offset = self.global_offset_map[passage_id]
-            # Lazy file opening - only open when needed
-            with open(passage_file, encoding="utf-8") as f:
-                f.seek(offset)
-                return json.loads(f.readline())
+        # Fast path: check each shard map (there are typically few shards).
+        # This avoids building a massive combined dict while keeping lookups
+        # bounded by the number of shards.
+        for passage_file, offset_map in self.offset_maps.items():
+            try:
+                offset = offset_map[passage_id]
+                with open(passage_file, encoding="utf-8") as f:
+                    f.seek(offset)
+                    return json.loads(f.readline())
+            except KeyError:
+                continue
         raise KeyError(f"Passage ID not found: {passage_id}")
 
+    def filter_search_results(
+        self,
+        search_results: list[SearchResult],
+        metadata_filters: Optional[dict[str, dict[str, Union[str, int, float, bool, list]]]],
+    ) -> list[SearchResult]:
+        """
+        Apply metadata filters to search results.
+
+        Args:
+            search_results: List of SearchResult objects
+            metadata_filters: Filter specifications to apply
+
+        Returns:
+            Filtered list of SearchResult objects
+        """
+        if not metadata_filters:
+            return search_results
+
+        logger.debug(f"Applying metadata filters to {len(search_results)} results")
+
+        # Convert SearchResult objects to dictionaries for the filter engine
+        result_dicts = []
+        for result in search_results:
+            result_dicts.append(
+                {
+                    "id": result.id,
+                    "score": result.score,
+                    "text": result.text,
+                    "metadata": result.metadata,
+                }
+            )
+
+        # Apply filters using the filter engine
+        filtered_dicts = self.filter_engine.apply_filters(result_dicts, metadata_filters)
+
+        # Convert back to SearchResult objects
+        filtered_results = []
+        for result_dict in filtered_dicts:
+            filtered_results.append(
+                SearchResult(
+                    id=result_dict["id"],
+                    score=result_dict["score"],
+                    text=result_dict["text"],
+                    metadata=result_dict["metadata"],
+                )
+            )
+
+        logger.debug(f"Filtered results: {len(filtered_results)} remaining")
+        return filtered_results
+
+    def __len__(self) -> int:
+        return self._total_count
+
 
 class LeannBuilder:
     def __init__(
@@ -185,6 +281,18 @@ class LeannBuilder:
         **backend_kwargs,
     ):
         self.backend_name = backend_name
+        # Normalize incompatible combinations early (for consistent metadata)
+        if backend_name == "hnsw":
+            is_recompute = backend_kwargs.get("is_recompute", True)
+            is_compact = backend_kwargs.get("is_compact", True)
+            if is_recompute is False and is_compact is True:
+                warnings.warn(
+                    "HNSW with is_recompute=False requires non-compact storage. Forcing is_compact=False.",
+                    UserWarning,
+                    stacklevel=2,
+                )
+                backend_kwargs["is_compact"] = False
+
         backend_factory: Optional[LeannBackendFactoryInterface] = BACKEND_REGISTRY.get(backend_name)
         if backend_factory is None:
             raise ValueError(f"Backend '{backend_name}' not found or not registered.")
@@ -275,6 +383,23 @@ class LeannBuilder:
     def build_index(self, index_path: str):
         if not self.chunks:
             raise ValueError("No chunks added.")
+
+        # Filter out invalid/empty text chunks early to keep passage and embedding counts aligned
+        valid_chunks: list[dict[str, Any]] = []
+        skipped = 0
+        for chunk in self.chunks:
+            text = chunk.get("text", "")
+            if isinstance(text, str) and text.strip():
+                valid_chunks.append(chunk)
+            else:
+                skipped += 1
+        if skipped > 0:
+            print(
+                f"Warning: Skipping {skipped} empty/invalid text chunk(s). Processing {len(valid_chunks)} valid chunks"
+            )
+            self.chunks = valid_chunks
+            if not self.chunks:
+                raise ValueError("All provided chunks are empty or invalid. Nothing to index.")
         if self.dimensions is None:
             self.dimensions = len(
                 compute_embeddings(
@@ -337,8 +462,12 @@ class LeannBuilder:
             "passage_sources": [
                 {
                     "type": "jsonl",
-                    "path": passages_file.name,  # Use relative path (just filename)
-                    "index_path": offset_file.name,  # Use relative path (just filename)
+                    # Preserve existing relative file names (backward-compatible)
+                    "path": passages_file.name,
+                    "index_path": offset_file.name,
+                    # Add optional redundant relative keys for remote build portability (non-breaking)
+                    "path_relative": passages_file.name,
+                    "index_path_relative": offset_file.name,
                 }
             ],
         }
@@ -348,9 +477,7 @@ class LeannBuilder:
             is_compact = self.backend_kwargs.get("is_compact", True)
             is_recompute = self.backend_kwargs.get("is_recompute", True)
             meta_data["is_compact"] = is_compact
-            meta_data["is_pruned"] = (
-                is_compact and is_recompute
-            )  # Pruned only if compact and recompute
+            meta_data["is_pruned"] = bool(is_recompute)
         with open(leann_meta_path, "w", encoding="utf-8") as f:
             json.dump(meta_data, f, indent=2)
 
@@ -453,8 +580,12 @@ class LeannBuilder:
             "passage_sources": [
                 {
                     "type": "jsonl",
-                    "path": passages_file.name,  # Use relative path (just filename)
-                    "index_path": offset_file.name,  # Use relative path (just filename)
+                    # Preserve existing relative file names (backward-compatible)
+                    "path": passages_file.name,
+                    "index_path": offset_file.name,
+                    # Add optional redundant relative keys for remote build portability (non-breaking)
+                    "path_relative": passages_file.name,
+                    "index_path_relative": offset_file.name,
                 }
             ],
             "built_from_precomputed_embeddings": True,
@@ -466,13 +597,157 @@ class LeannBuilder:
             is_compact = self.backend_kwargs.get("is_compact", True)
             is_recompute = self.backend_kwargs.get("is_recompute", True)
             meta_data["is_compact"] = is_compact
-            meta_data["is_pruned"] = is_compact and is_recompute
+            meta_data["is_pruned"] = bool(is_recompute)
 
         with open(leann_meta_path, "w", encoding="utf-8") as f:
             json.dump(meta_data, f, indent=2)
 
         logger.info(f"Index built successfully from precomputed embeddings: {index_path}")
 
+    def update_index(self, index_path: str):
+        """Append new passages and vectors to an existing HNSW index."""
+        if not self.chunks:
+            raise ValueError("No new chunks provided for update.")
+
+        path = Path(index_path)
+        index_dir = path.parent
+        index_name = path.name
+        index_prefix = path.stem
+
+        meta_path = index_dir / f"{index_name}.meta.json"
+        passages_file = index_dir / f"{index_name}.passages.jsonl"
+        offset_file = index_dir / f"{index_name}.passages.idx"
+        index_file = index_dir / f"{index_prefix}.index"
+
+        if not meta_path.exists() or not passages_file.exists() or not offset_file.exists():
+            raise FileNotFoundError("Index metadata or passage files are missing; cannot update.")
+        if not index_file.exists():
+            raise FileNotFoundError(f"HNSW index file not found: {index_file}")
+
+        with open(meta_path, encoding="utf-8") as f:
+            meta = json.load(f)
+        backend_name = meta.get("backend_name")
+        if backend_name != self.backend_name:
+            raise ValueError(
+                f"Index was built with backend '{backend_name}', cannot update with '{self.backend_name}'."
+            )
+
+        meta_backend_kwargs = meta.get("backend_kwargs", {})
+        index_is_compact = meta.get("is_compact", meta_backend_kwargs.get("is_compact", True))
+        if index_is_compact:
+            raise ValueError(
+                "Compact HNSW indices do not support in-place updates. Rebuild required."
+            )
+
+        distance_metric = meta_backend_kwargs.get(
+            "distance_metric", self.backend_kwargs.get("distance_metric", "mips")
+        ).lower()
+        needs_recompute = bool(
+            meta.get("is_pruned")
+            or meta_backend_kwargs.get("is_recompute")
+            or self.backend_kwargs.get("is_recompute")
+        )
+
+        with open(offset_file, "rb") as f:
+            offset_map: dict[str, int] = pickle.load(f)
+        existing_ids = set(offset_map.keys())
+
+        valid_chunks: list[dict[str, Any]] = []
+        for chunk in self.chunks:
+            text = chunk.get("text", "")
+            if not isinstance(text, str) or not text.strip():
+                continue
+            metadata = chunk.setdefault("metadata", {})
+            passage_id = chunk.get("id") or metadata.get("id")
+            if passage_id and passage_id in existing_ids:
+                raise ValueError(f"Passage ID '{passage_id}' already exists in the index.")
+            valid_chunks.append(chunk)
+
+        if not valid_chunks:
+            raise ValueError("No valid chunks to append.")
+
+        texts_to_embed = [chunk["text"] for chunk in valid_chunks]
+        embeddings = compute_embeddings(
+            texts_to_embed,
+            self.embedding_model,
+            self.embedding_mode,
+            use_server=False,
+            is_build=True,
+        )
+
+        embedding_dim = embeddings.shape[1]
+        expected_dim = meta.get("dimensions")
+        if expected_dim is not None and expected_dim != embedding_dim:
+            raise ValueError(
+                f"Dimension mismatch during update: existing index uses {expected_dim}, got {embedding_dim}."
+            )
+
+        from leann_backend_hnsw import faiss  # type: ignore
+
+        embeddings = np.ascontiguousarray(embeddings, dtype=np.float32)
+        if distance_metric == "cosine":
+            norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
+            norms[norms == 0] = 1
+            embeddings = embeddings / norms
+
+        index = faiss.read_index(str(index_file))
+        if hasattr(index, "is_recompute"):
+            index.is_recompute = needs_recompute
+        if getattr(index, "storage", None) is None:
+            if index.metric_type == faiss.METRIC_INNER_PRODUCT:
+                storage_index = faiss.IndexFlatIP(index.d)
+            else:
+                storage_index = faiss.IndexFlatL2(index.d)
+            index.storage = storage_index
+            index.own_fields = True
+        if index.d != embedding_dim:
+            raise ValueError(
+                f"Existing index dimension ({index.d}) does not match new embeddings ({embedding_dim})."
+            )
+
+        base_id = index.ntotal
+        for offset, chunk in enumerate(valid_chunks):
+            new_id = str(base_id + offset)
+            chunk.setdefault("metadata", {})["id"] = new_id
+            chunk["id"] = new_id
+
+        index.add(embeddings.shape[0], faiss.swig_ptr(embeddings))
+        faiss.write_index(index, str(index_file))
+
+        with open(passages_file, "a", encoding="utf-8") as f:
+            for chunk in valid_chunks:
+                offset = f.tell()
+                json.dump(
+                    {
+                        "id": chunk["id"],
+                        "text": chunk["text"],
+                        "metadata": chunk.get("metadata", {}),
+                    },
+                    f,
+                    ensure_ascii=False,
+                )
+                f.write("\n")
+                offset_map[chunk["id"]] = offset
+
+        with open(offset_file, "wb") as f:
+            pickle.dump(offset_map, f)
+
+        meta["total_passages"] = len(offset_map)
+        with open(meta_path, "w", encoding="utf-8") as f:
+            json.dump(meta, f, indent=2)
+
+        logger.info(
+            "Appended %d passages to index '%s'. New total: %d",
+            len(valid_chunks),
+            index_path,
+            len(offset_map),
+        )
+
+        self.chunks.clear()
+
+        if needs_recompute:
+            prune_hnsw_embeddings_inplace(str(index_file))
+
 
 class LeannSearcher:
     def __init__(self, index_path: str, enable_warmup: bool = False, **backend_kwargs):
@@ -496,9 +771,12 @@ class LeannSearcher:
         self.embedding_model = self.meta_data["embedding_model"]
         # Support both old and new format
         self.embedding_mode = self.meta_data.get("embedding_mode", "sentence-transformers")
+        # Delegate portability handling to PassageManager
         self.passage_manager = PassageManager(
             self.meta_data.get("passage_sources", []), metadata_file_path=self.meta_path_str
         )
+        # Preserve backend name for conditional parameter forwarding
+        self.backend_name = backend_name
         backend_factory = BACKEND_REGISTRY.get(backend_name)
         if backend_factory is None:
             raise ValueError(f"Backend '{backend_name}' not found.")
@@ -518,15 +796,49 @@ class LeannSearcher:
         recompute_embeddings: bool = True,
         pruning_strategy: Literal["global", "local", "proportional"] = "global",
         expected_zmq_port: int = 5557,
+        metadata_filters: Optional[dict[str, dict[str, Union[str, int, float, bool, list]]]] = None,
+        batch_size: int = 0,
+        use_grep: bool = False,
         **kwargs,
     ) -> list[SearchResult]:
+        """
+        Search for nearest neighbors with optional metadata filtering.
+
+        Args:
+            query: Text query to search for
+            top_k: Number of nearest neighbors to return
+            complexity: Search complexity/candidate list size, higher = more accurate but slower
+            beam_width: Number of parallel search paths/IO requests per iteration
+            prune_ratio: Ratio of neighbors to prune via approximate distance (0.0-1.0)
+            recompute_embeddings: Whether to fetch fresh embeddings from server vs use stored codes
+            pruning_strategy: Candidate selection strategy - "global" (default), "local", or "proportional"
+            expected_zmq_port: ZMQ port for embedding server communication
+            metadata_filters: Optional filters to apply to search results based on metadata.
+                Format: {"field_name": {"operator": value}}
+                Supported operators:
+                - Comparison: "==", "!=", "<", "<=", ">", ">="
+                - Membership: "in", "not_in"
+                - String: "contains", "starts_with", "ends_with"
+                Example: {"chapter": {"<=": 5}, "tags": {"in": ["fiction", "drama"]}}
+            **kwargs: Backend-specific parameters
+
+        Returns:
+            List of SearchResult objects with text, metadata, and similarity scores
+        """
+        # Handle grep search
+        if use_grep:
+            return self._grep_search(query, top_k)
+
         logger.info("🔍 LeannSearcher.search() called:")
         logger.info(f"  Query: '{query}'")
         logger.info(f"  Top_k: {top_k}")
+        logger.info(f"  Metadata filters: {metadata_filters}")
         logger.info(f"  Additional kwargs: {kwargs}")
 
         # Smart top_k detection and adjustment
-        total_docs = len(self.passage_manager.global_offset_map)
+        # Use PassageManager length (sum of shard sizes) to avoid
+        # depending on a massive combined map
+        total_docs = len(self.passage_manager)
         original_top_k = top_k
         if top_k > total_docs:
             top_k = total_docs
@@ -555,23 +867,33 @@ class LeannSearcher:
             use_server_if_available=recompute_embeddings,
             zmq_port=zmq_port,
         )
-        # logger.info(f"  Generated embedding shape: {query_embedding.shape}")
-        time.time() - start_time
-        # logger.info(f"  Embedding time: {embedding_time} seconds")
+        logger.info(f"  Generated embedding shape: {query_embedding.shape}")
+        embedding_time = time.time() - start_time
+        logger.info(f"  Embedding time: {embedding_time} seconds")
 
         start_time = time.time()
+        backend_search_kwargs: dict[str, Any] = {
+            "complexity": complexity,
+            "beam_width": beam_width,
+            "prune_ratio": prune_ratio,
+            "recompute_embeddings": recompute_embeddings,
+            "pruning_strategy": pruning_strategy,
+            "zmq_port": zmq_port,
+        }
+        # Only HNSW supports batching; forward conditionally
+        if self.backend_name == "hnsw":
+            backend_search_kwargs["batch_size"] = batch_size
+
+        # Merge any extra kwargs last
+        backend_search_kwargs.update(kwargs)
+
         results = self.backend_impl.search(
             query_embedding,
             top_k,
-            complexity=complexity,
-            beam_width=beam_width,
-            prune_ratio=prune_ratio,
-            recompute_embeddings=recompute_embeddings,
-            pruning_strategy=pruning_strategy,
-            zmq_port=zmq_port,
-            **kwargs,
+            **backend_search_kwargs,
         )
-        # logger.info(f"  Search time: {search_time} seconds")
+        search_time = time.time() - start_time
+        logger.info(f"  Search time in search() LEANN searcher: {search_time} seconds")
         logger.info(f"  Backend returned: labels={len(results.get('labels', [[]])[0])} results")
 
         enriched_results = []
@@ -610,34 +932,133 @@ class LeannSearcher:
                         f"   {RED}✗{RESET} [{i + 1:2d}] ID: '{string_id}' -> {RED}ERROR: Passage not found!{RESET}"
                     )
 
+        # Apply metadata filters if specified
+        if metadata_filters:
+            logger.info(f"  🔍 Applying metadata filters: {metadata_filters}")
+            enriched_results = self.passage_manager.filter_search_results(
+                enriched_results, metadata_filters
+            )
+
         # Define color codes outside the loop for final message
         GREEN = "\033[92m"
         RESET = "\033[0m"
         logger.info(f"  {GREEN}✓ Final enriched results: {len(enriched_results)} passages{RESET}")
         return enriched_results
 
-    def cleanup(self):
-        """Explicitly cleanup embedding server and ZMQ resources.
+    def _find_jsonl_file(self) -> Optional[str]:
+        """Find the .jsonl file containing raw passages for grep search"""
+        index_path = Path(self.meta_path_str).parent
+        potential_files = [
+            index_path / "documents.leann.passages.jsonl",
+            index_path.parent / "documents.leann.passages.jsonl",
+        ]
 
+        for file_path in potential_files:
+            if file_path.exists():
+                return str(file_path)
+        return None
+
+    def _grep_search(self, query: str, top_k: int = 5) -> list[SearchResult]:
+        """Perform grep-based search on raw passages"""
+        jsonl_file = self._find_jsonl_file()
+        if not jsonl_file:
+            raise FileNotFoundError("No .jsonl passages file found for grep search")
+
+        try:
+            cmd = ["grep", "-i", "-n", query, jsonl_file]
+            result = subprocess.run(cmd, capture_output=True, text=True, check=False)
+
+            if result.returncode == 1:
+                return []
+            elif result.returncode != 0:
+                raise RuntimeError(f"Grep failed: {result.stderr}")
+
+            matches = []
+            for line in result.stdout.strip().split("\n"):
+                if not line:
+                    continue
+                parts = line.split(":", 1)
+                if len(parts) != 2:
+                    continue
+
+                try:
+                    data = json.loads(parts[1])
+                    text = data.get("text", "")
+                    score = text.lower().count(query.lower())
+
+                    matches.append(
+                        SearchResult(
+                            id=data.get("id", parts[0]),
+                            text=text,
+                            metadata=data.get("metadata", {}),
+                            score=float(score),
+                        )
+                    )
+                except json.JSONDecodeError:
+                    continue
+
+            matches.sort(key=lambda x: x.score, reverse=True)
+            return matches[:top_k]
+
+        except FileNotFoundError:
+            raise RuntimeError(
+                "grep command not found. Please install grep or use semantic search."
+            )
+
+    def _python_regex_search(self, query: str, top_k: int = 5) -> list[SearchResult]:
+        """Fallback regex search"""
+        jsonl_file = self._find_jsonl_file()
+        if not jsonl_file:
+            raise FileNotFoundError("No .jsonl file found")
+
+        pattern = re.compile(re.escape(query), re.IGNORECASE)
+        matches = []
+
+        with open(jsonl_file, encoding="utf-8") as f:
+            for line_num, line in enumerate(f, 1):
+                if pattern.search(line):
+                    try:
+                        data = json.loads(line.strip())
+                        matches.append(
+                            SearchResult(
+                                id=data.get("id", str(line_num)),
+                                text=data.get("text", ""),
+                                metadata=data.get("metadata", {}),
+                                score=float(len(pattern.findall(data.get("text", "")))),
+                            )
+                        )
+                    except json.JSONDecodeError:
+                        continue
+
+        matches.sort(key=lambda x: x.score, reverse=True)
+        return matches[:top_k]
+
+    def cleanup(self):
+        """Explicitly cleanup embedding server resources.
         This method should be called after you're done using the searcher,
         especially in test environments or batch processing scenarios.
         """
-        # Stop embedding server
-        if hasattr(self.backend_impl, "embedding_server_manager"):
-            self.backend_impl.embedding_server_manager.stop_server()
+        backend = getattr(self.backend_impl, "embedding_server_manager", None)
+        if backend is not None:
+            backend.stop_server()
 
-        # Set ZMQ linger but don't terminate global context
+    # Enable automatic cleanup patterns
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc, tb):
         try:
-            import zmq
-
-            # Just set linger on the global instance
-            ctx = zmq.Context.instance()
-            ctx.linger = 0
-            # NEVER call ctx.term() or destroy() on the global instance
-            # That would block waiting for all sockets to close
+            self.cleanup()
         except Exception:
             pass
 
+    def __del__(self):
+        try:
+            self.cleanup()
+        except Exception:
+            # Avoid noisy errors during interpreter shutdown
+            pass
+
 
 class LeannChat:
     def __init__(
@@ -645,9 +1066,15 @@ class LeannChat:
         index_path: str,
         llm_config: Optional[dict[str, Any]] = None,
         enable_warmup: bool = False,
+        searcher: Optional[LeannSearcher] = None,
         **kwargs,
     ):
-        self.searcher = LeannSearcher(index_path, enable_warmup=enable_warmup, **kwargs)
+        if searcher is None:
+            self.searcher = LeannSearcher(index_path, enable_warmup=enable_warmup, **kwargs)
+            self._owns_searcher = True
+        else:
+            self.searcher = searcher
+            self._owns_searcher = False
         self.llm = get_llm(llm_config)
 
     def ask(
@@ -661,6 +1088,9 @@ class LeannChat:
         pruning_strategy: Literal["global", "local", "proportional"] = "global",
         llm_kwargs: Optional[dict[str, Any]] = None,
         expected_zmq_port: int = 5557,
+        metadata_filters: Optional[dict[str, dict[str, Union[str, int, float, bool, list]]]] = None,
+        batch_size: int = 0,
+        use_grep: bool = False,
         **search_kwargs,
     ):
         if llm_kwargs is None:
@@ -675,10 +1105,12 @@ class LeannChat:
             recompute_embeddings=recompute_embeddings,
             pruning_strategy=pruning_strategy,
             expected_zmq_port=expected_zmq_port,
+            metadata_filters=metadata_filters,
+            batch_size=batch_size,
             **search_kwargs,
         )
         search_time = time.time() - search_time
-        # logger.info(f"  Search time: {search_time} seconds")
+        logger.info(f"  Search time: {search_time} seconds")
         context = "\n\n".join([r.text for r in results])
         prompt = (
             "Here is some retrieved context that might help answer your question:\n\n"
@@ -714,5 +1146,23 @@ class LeannChat:
         This method should be called after you're done using the chat interface,
         especially in test environments or batch processing scenarios.
         """
-        if hasattr(self.searcher, "cleanup"):
+        # Only stop the embedding server if this LeannChat instance created the searcher.
+        # When a shared searcher is passed in, avoid shutting down the server to enable reuse.
+        if getattr(self, "_owns_searcher", False) and hasattr(self.searcher, "cleanup"):
             self.searcher.cleanup()
+
+    # Enable automatic cleanup patterns
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc, tb):
+        try:
+            self.cleanup()
+        except Exception:
+            pass
+
+    def __del__(self):
+        try:
+            self.cleanup()
+        except Exception:
+            pass

packages/leann-core/src/leann/chat.py

@@ -422,7 +422,6 @@ class LLMInterface(ABC):
                 top_k=10,
                 complexity=64,
                 beam_width=8,
-                USE_DEFERRED_FETCH=True,
                 skip_search_reorder=True,
                 recompute_beighbor_embeddings=True,
                 dedup_node_dis=True,
@@ -434,7 +433,6 @@ class LLMInterface(ABC):
         Supported kwargs:
             - complexity (int): Search complexity parameter (default: 32)
             - beam_width (int): Beam width for search (default: 4)
-            - USE_DEFERRED_FETCH (bool): Enable deferred fetch mode (default: False)
             - skip_search_reorder (bool): Skip search reorder step (default: False)
             - recompute_beighbor_embeddings (bool): Enable ZMQ embedding server for neighbor recomputation (default: False)
             - dedup_node_dis (bool): Deduplicate nodes by distance (default: False)
@@ -682,6 +680,60 @@ class HFChat(LLMInterface):
         return response.strip()
 
 
+class GeminiChat(LLMInterface):
+    """LLM interface for Google Gemini models."""
+
+    def __init__(self, model: str = "gemini-2.5-flash", api_key: Optional[str] = None):
+        self.model = model
+        self.api_key = api_key or os.getenv("GEMINI_API_KEY")
+
+        if not self.api_key:
+            raise ValueError(
+                "Gemini API key is required. Set GEMINI_API_KEY environment variable or pass api_key parameter."
+            )
+
+        logger.info(f"Initializing Gemini Chat with model='{model}'")
+
+        try:
+            import google.genai as genai
+
+            self.client = genai.Client(api_key=self.api_key)
+        except ImportError:
+            raise ImportError(
+                "The 'google-genai' library is required for Gemini models. Please install it with 'uv pip install google-genai'."
+            )
+
+    def ask(self, prompt: str, **kwargs) -> str:
+        logger.info(f"Sending request to Gemini with model {self.model}")
+
+        try:
+            from google.genai.types import GenerateContentConfig
+
+            generation_config = GenerateContentConfig(
+                temperature=kwargs.get("temperature", 0.7),
+                max_output_tokens=kwargs.get("max_tokens", 1000),
+            )
+
+            # Handle top_p parameter
+            if "top_p" in kwargs:
+                generation_config.top_p = kwargs["top_p"]
+
+            response = self.client.models.generate_content(
+                model=self.model,
+                contents=prompt,
+                config=generation_config,
+            )
+            # Handle potential None response text
+            response_text = response.text
+            if response_text is None:
+                logger.warning("Gemini returned None response text")
+                return ""
+            return response_text.strip()
+        except Exception as e:
+            logger.error(f"Error communicating with Gemini: {e}")
+            return f"Error: Could not get a response from Gemini. Details: {e}"
+
+
 class OpenAIChat(LLMInterface):
     """LLM interface for OpenAI models."""
 
@@ -795,6 +847,8 @@ def get_llm(llm_config: Optional[dict[str, Any]] = None) -> LLMInterface:
         return HFChat(model_name=model or "deepseek-ai/deepseek-llm-7b-chat")
     elif llm_type == "openai":
         return OpenAIChat(model=model or "gpt-4o", api_key=llm_config.get("api_key"))
+    elif llm_type == "gemini":
+        return GeminiChat(model=model or "gemini-2.5-flash", api_key=llm_config.get("api_key"))
     elif llm_type == "simulated":
         return SimulatedChat()
     else:

packages/leann-core/src/leann/chunking_utils.py

@@ -0,0 +1,220 @@
+"""
+Enhanced chunking utilities with AST-aware code chunking support.
+Packaged within leann-core so installed wheels can import it reliably.
+"""
+
+import logging
+from pathlib import Path
+from typing import Optional
+
+from llama_index.core.node_parser import SentenceSplitter
+
+logger = logging.getLogger(__name__)
+
+# Code file extensions supported by astchunk
+CODE_EXTENSIONS = {
+    ".py": "python",
+    ".java": "java",
+    ".cs": "csharp",
+    ".ts": "typescript",
+    ".tsx": "typescript",
+    ".js": "typescript",
+    ".jsx": "typescript",
+}
+
+
+def detect_code_files(documents, code_extensions=None) -> tuple[list, list]:
+    """Separate documents into code files and regular text files."""
+    if code_extensions is None:
+        code_extensions = CODE_EXTENSIONS
+
+    code_docs = []
+    text_docs = []
+
+    for doc in documents:
+        file_path = doc.metadata.get("file_path", "") or doc.metadata.get("file_name", "")
+        if file_path:
+            file_ext = Path(file_path).suffix.lower()
+            if file_ext in code_extensions:
+                doc.metadata["language"] = code_extensions[file_ext]
+                doc.metadata["is_code"] = True
+                code_docs.append(doc)
+            else:
+                doc.metadata["is_code"] = False
+                text_docs.append(doc)
+        else:
+            doc.metadata["is_code"] = False
+            text_docs.append(doc)
+
+    logger.info(f"Detected {len(code_docs)} code files and {len(text_docs)} text files")
+    return code_docs, text_docs
+
+
+def get_language_from_extension(file_path: str) -> Optional[str]:
+    """Return language string from a filename/extension using CODE_EXTENSIONS."""
+    ext = Path(file_path).suffix.lower()
+    return CODE_EXTENSIONS.get(ext)
+
+
+def create_ast_chunks(
+    documents,
+    max_chunk_size: int = 512,
+    chunk_overlap: int = 64,
+    metadata_template: str = "default",
+) -> list[str]:
+    """Create AST-aware chunks from code documents using astchunk.
+
+    Falls back to traditional chunking if astchunk is unavailable.
+    """
+    try:
+        from astchunk import ASTChunkBuilder  # optional dependency
+    except ImportError as e:
+        logger.error(f"astchunk not available: {e}")
+        logger.info("Falling back to traditional chunking for code files")
+        return create_traditional_chunks(documents, max_chunk_size, chunk_overlap)
+
+    all_chunks = []
+    for doc in documents:
+        language = doc.metadata.get("language")
+        if not language:
+            logger.warning("No language detected; falling back to traditional chunking")
+            all_chunks.extend(create_traditional_chunks([doc], max_chunk_size, chunk_overlap))
+            continue
+
+        try:
+            configs = {
+                "max_chunk_size": max_chunk_size,
+                "language": language,
+                "metadata_template": metadata_template,
+                "chunk_overlap": chunk_overlap if chunk_overlap > 0 else 0,
+            }
+
+            repo_metadata = {
+                "file_path": doc.metadata.get("file_path", ""),
+                "file_name": doc.metadata.get("file_name", ""),
+                "creation_date": doc.metadata.get("creation_date", ""),
+                "last_modified_date": doc.metadata.get("last_modified_date", ""),
+            }
+            configs["repo_level_metadata"] = repo_metadata
+
+            chunk_builder = ASTChunkBuilder(**configs)
+            code_content = doc.get_content()
+            if not code_content or not code_content.strip():
+                logger.warning("Empty code content, skipping")
+                continue
+
+            chunks = chunk_builder.chunkify(code_content)
+            for chunk in chunks:
+                if hasattr(chunk, "text"):
+                    chunk_text = chunk.text
+                elif isinstance(chunk, dict) and "text" in chunk:
+                    chunk_text = chunk["text"]
+                elif isinstance(chunk, str):
+                    chunk_text = chunk
+                else:
+                    chunk_text = str(chunk)
+
+                if chunk_text and chunk_text.strip():
+                    all_chunks.append(chunk_text.strip())
+
+            logger.info(
+                f"Created {len(chunks)} AST chunks from {language} file: {doc.metadata.get('file_name', 'unknown')}"
+            )
+        except Exception as e:
+            logger.warning(f"AST chunking failed for {language} file: {e}")
+            logger.info("Falling back to traditional chunking")
+            all_chunks.extend(create_traditional_chunks([doc], max_chunk_size, chunk_overlap))
+
+    return all_chunks
+
+
+def create_traditional_chunks(
+    documents, chunk_size: int = 256, chunk_overlap: int = 128
+) -> list[str]:
+    """Create traditional text chunks using LlamaIndex SentenceSplitter."""
+    if chunk_size <= 0:
+        logger.warning(f"Invalid chunk_size={chunk_size}, using default value of 256")
+        chunk_size = 256
+    if chunk_overlap < 0:
+        chunk_overlap = 0
+    if chunk_overlap >= chunk_size:
+        chunk_overlap = chunk_size // 2
+
+    node_parser = SentenceSplitter(
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+        separator=" ",
+        paragraph_separator="\n\n",
+    )
+
+    all_texts = []
+    for doc in documents:
+        try:
+            nodes = node_parser.get_nodes_from_documents([doc])
+            if nodes:
+                all_texts.extend(node.get_content() for node in nodes)
+        except Exception as e:
+            logger.error(f"Traditional chunking failed for document: {e}")
+            content = doc.get_content()
+            if content and content.strip():
+                all_texts.append(content.strip())
+
+    return all_texts
+
+
+def create_text_chunks(
+    documents,
+    chunk_size: int = 256,
+    chunk_overlap: int = 128,
+    use_ast_chunking: bool = False,
+    ast_chunk_size: int = 512,
+    ast_chunk_overlap: int = 64,
+    code_file_extensions: Optional[list[str]] = None,
+    ast_fallback_traditional: bool = True,
+) -> list[str]:
+    """Create text chunks from documents with optional AST support for code files."""
+    if not documents:
+        logger.warning("No documents provided for chunking")
+        return []
+
+    local_code_extensions = CODE_EXTENSIONS.copy()
+    if code_file_extensions:
+        ext_mapping = {
+            ".py": "python",
+            ".java": "java",
+            ".cs": "c_sharp",
+            ".ts": "typescript",
+            ".tsx": "typescript",
+        }
+        for ext in code_file_extensions:
+            if ext.lower() not in local_code_extensions:
+                if ext.lower() in ext_mapping:
+                    local_code_extensions[ext.lower()] = ext_mapping[ext.lower()]
+                else:
+                    logger.warning(f"Unsupported extension {ext}, will use traditional chunking")
+
+    all_chunks = []
+    if use_ast_chunking:
+        code_docs, text_docs = detect_code_files(documents, local_code_extensions)
+        if code_docs:
+            try:
+                all_chunks.extend(
+                    create_ast_chunks(
+                        code_docs, max_chunk_size=ast_chunk_size, chunk_overlap=ast_chunk_overlap
+                    )
+                )
+            except Exception as e:
+                logger.error(f"AST chunking failed: {e}")
+                if ast_fallback_traditional:
+                    all_chunks.extend(
+                        create_traditional_chunks(code_docs, chunk_size, chunk_overlap)
+                    )
+                else:
+                    raise
+        if text_docs:
+            all_chunks.extend(create_traditional_chunks(text_docs, chunk_size, chunk_overlap))
+    else:
+        all_chunks = create_traditional_chunks(documents, chunk_size, chunk_overlap)
+
+    logger.info(f"Total chunks created: {len(all_chunks)}")
+    return all_chunks

packages/leann-core/src/leann/embedding_compute.py

@@ -6,7 +6,7 @@ Preserves all optimization parameters to ensure performance
 
 import logging
 import os
-from concurrent.futures import ThreadPoolExecutor, as_completed
+import time
 from typing import Any
 
 import numpy as np
@@ -29,6 +29,8 @@ def compute_embeddings(
     is_build: bool = False,
     batch_size: int = 32,
     adaptive_optimization: bool = True,
+    manual_tokenize: bool = False,
+    max_length: int = 512,
 ) -> np.ndarray:
     """
     Unified embedding computation entry point
@@ -51,6 +53,8 @@ def compute_embeddings(
             is_build=is_build,
             batch_size=batch_size,
             adaptive_optimization=adaptive_optimization,
+            manual_tokenize=manual_tokenize,
+            max_length=max_length,
         )
     elif mode == "openai":
         return compute_embeddings_openai(texts, model_name)
@@ -58,6 +62,8 @@ def compute_embeddings(
         return compute_embeddings_mlx(texts, model_name)
     elif mode == "ollama":
         return compute_embeddings_ollama(texts, model_name, is_build=is_build)
+    elif mode == "gemini":
+        return compute_embeddings_gemini(texts, model_name, is_build=is_build)
     else:
         raise ValueError(f"Unsupported embedding mode: {mode}")
 
@@ -70,6 +76,8 @@ def compute_embeddings_sentence_transformers(
     batch_size: int = 32,
     is_build: bool = False,
     adaptive_optimization: bool = True,
+    manual_tokenize: bool = False,
+    max_length: int = 512,
 ) -> np.ndarray:
     """
     Compute embeddings using SentenceTransformer with model caching and adaptive optimization
@@ -213,20 +221,130 @@ def compute_embeddings_sentence_transformers(
         logger.info(f"Model cached: {cache_key}")
 
     # Compute embeddings with optimized inference mode
-    logger.info(f"Starting embedding computation... (batch_size: {batch_size})")
+    logger.info(
+        f"Starting embedding computation... (batch_size: {batch_size}, manual_tokenize={manual_tokenize})"
+    )
 
-    # Use torch.inference_mode for optimal performance
-    with torch.inference_mode():
-        embeddings = model.encode(
-            texts,
-            batch_size=batch_size,
-            show_progress_bar=is_build,  # Don't show progress bar in server environment
-            convert_to_numpy=True,
-            normalize_embeddings=False,
-            device=device,
-        )
+    start_time = time.time()
+    if not manual_tokenize:
+        # Use SentenceTransformer's optimized encode path (default)
+        with torch.inference_mode():
+            embeddings = model.encode(
+                texts,
+                batch_size=batch_size,
+                show_progress_bar=is_build,  # Don't show progress bar in server environment
+                convert_to_numpy=True,
+                normalize_embeddings=False,
+                device=device,
+            )
+        # Synchronize if CUDA to measure accurate wall time
+        try:
+            if torch.cuda.is_available():
+                torch.cuda.synchronize()
+        except Exception:
+            pass
+    else:
+        # Manual tokenization + forward pass using HF AutoTokenizer/AutoModel
+        try:
+            from transformers import AutoModel, AutoTokenizer  # type: ignore
+        except Exception as e:
+            raise ImportError(f"transformers is required for manual_tokenize=True: {e}")
 
+        # Cache tokenizer and model
+        tok_cache_key = f"hf_tokenizer_{model_name}"
+        mdl_cache_key = f"hf_model_{model_name}_{device}_{use_fp16}"
+        if tok_cache_key in _model_cache and mdl_cache_key in _model_cache:
+            hf_tokenizer = _model_cache[tok_cache_key]
+            hf_model = _model_cache[mdl_cache_key]
+            logger.info("Using cached HF tokenizer/model for manual path")
+        else:
+            logger.info("Loading HF tokenizer/model for manual tokenization path")
+            hf_tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)
+            torch_dtype = torch.float16 if (use_fp16 and device == "cuda") else torch.float32
+            hf_model = AutoModel.from_pretrained(model_name, torch_dtype=torch_dtype)
+            hf_model.to(device)
+            hf_model.eval()
+            # Optional compile on supported devices
+            if device in ["cuda", "mps"]:
+                try:
+                    hf_model = torch.compile(hf_model, mode="reduce-overhead", dynamic=True)  # type: ignore
+                except Exception:
+                    pass
+            _model_cache[tok_cache_key] = hf_tokenizer
+            _model_cache[mdl_cache_key] = hf_model
+
+        all_embeddings: list[np.ndarray] = []
+        # Progress bar when building or for large inputs
+        show_progress = is_build or len(texts) > 32
+        try:
+            if show_progress:
+                from tqdm import tqdm  # type: ignore
+
+                batch_iter = tqdm(
+                    range(0, len(texts), batch_size),
+                    desc="Embedding (manual)",
+                    unit="batch",
+                )
+            else:
+                batch_iter = range(0, len(texts), batch_size)
+        except Exception:
+            batch_iter = range(0, len(texts), batch_size)
+
+        start_time_manual = time.time()
+        with torch.inference_mode():
+            for start_index in batch_iter:
+                end_index = min(start_index + batch_size, len(texts))
+                batch_texts = texts[start_index:end_index]
+                tokenize_start_time = time.time()
+                inputs = hf_tokenizer(
+                    batch_texts,
+                    padding=True,
+                    truncation=True,
+                    max_length=max_length,
+                    return_tensors="pt",
+                )
+                tokenize_end_time = time.time()
+                logger.info(
+                    f"Tokenize time taken: {tokenize_end_time - tokenize_start_time} seconds"
+                )
+                # Print shapes of all input tensors for debugging
+                for k, v in inputs.items():
+                    print(f"inputs[{k!r}] shape: {getattr(v, 'shape', type(v))}")
+                to_device_start_time = time.time()
+                inputs = {k: v.to(device) for k, v in inputs.items()}
+                to_device_end_time = time.time()
+                logger.info(
+                    f"To device time taken: {to_device_end_time - to_device_start_time} seconds"
+                )
+                forward_start_time = time.time()
+                outputs = hf_model(**inputs)
+                forward_end_time = time.time()
+                logger.info(f"Forward time taken: {forward_end_time - forward_start_time} seconds")
+                last_hidden_state = outputs.last_hidden_state  # (B, L, H)
+                attention_mask = inputs.get("attention_mask")
+                if attention_mask is None:
+                    # Fallback: assume all tokens are valid
+                    pooled = last_hidden_state.mean(dim=1)
+                else:
+                    mask = attention_mask.unsqueeze(-1).to(last_hidden_state.dtype)
+                    masked = last_hidden_state * mask
+                    lengths = mask.sum(dim=1).clamp(min=1)
+                    pooled = masked.sum(dim=1) / lengths
+                # Move to CPU float32
+                batch_embeddings = pooled.detach().to("cpu").float().numpy()
+                all_embeddings.append(batch_embeddings)
+
+        embeddings = np.vstack(all_embeddings).astype(np.float32, copy=False)
+        try:
+            if torch.cuda.is_available():
+                torch.cuda.synchronize()
+        except Exception:
+            pass
+        end_time = time.time()
+        logger.info(f"Manual tokenize time taken: {end_time - start_time_manual} seconds")
+    end_time = time.time()
     logger.info(f"Generated {len(embeddings)} embeddings, dimension: {embeddings.shape[1]}")
+    logger.info(f"Time taken: {end_time - start_time} seconds")
 
     # Validate results
     if np.isnan(embeddings).any() or np.isinf(embeddings).any():
@@ -245,6 +363,16 @@ def compute_embeddings_openai(texts: list[str], model_name: str) -> np.ndarray:
     except ImportError as e:
         raise ImportError(f"OpenAI package not installed: {e}")
 
+    # Validate input list
+    if not texts:
+        raise ValueError("Cannot compute embeddings for empty text list")
+    # Extra validation: abort early if any item is empty/whitespace
+    invalid_count = sum(1 for t in texts if not isinstance(t, str) or not t.strip())
+    if invalid_count > 0:
+        raise ValueError(
+            f"Found {invalid_count} empty/invalid text(s) in input. Upstream should filter before calling OpenAI."
+        )
+
     api_key = os.getenv("OPENAI_API_KEY")
     if not api_key:
         raise RuntimeError("OPENAI_API_KEY environment variable not set")
@@ -264,8 +392,16 @@ def compute_embeddings_openai(texts: list[str], model_name: str) -> np.ndarray:
     print(f"len of texts: {len(texts)}")
 
     # OpenAI has limits on batch size and input length
-    max_batch_size = 1000  # Conservative batch size
+    max_batch_size = 800  # Conservative batch size because the token limit is 300K
     all_embeddings = []
+    # get the avg len of texts
+    avg_len = sum(len(text) for text in texts) / len(texts)
+    print(f"avg len of texts: {avg_len}")
+    # if avg len is less than 1000, use the max batch size
+    if avg_len > 300:
+        max_batch_size = 500
+
+    # if avg len is less than 1000, use the max batch size
 
     try:
         from tqdm import tqdm
@@ -374,7 +510,9 @@ def compute_embeddings_ollama(
     texts: list[str], model_name: str, is_build: bool = False, host: str = "http://localhost:11434"
 ) -> np.ndarray:
     """
-    Compute embeddings using Ollama API.
+    Compute embeddings using Ollama API with simplified batch processing.
+
+    Uses batch size of 32 for MPS/CPU and 128 for CUDA to optimize performance.
 
     Args:
         texts: List of texts to compute embeddings for
@@ -438,12 +576,19 @@ def compute_embeddings_ollama(
             if any(emb in base_name for emb in ["embed", "bge", "minilm", "e5"]):
                 embedding_models.append(model)
 
-        # Check if model exists (handle versioned names)
-        model_found = any(
-            model_name == name.split(":")[0] or model_name == name for name in model_names
-        )
+        # Check if model exists (handle versioned names) and resolve to full name
+        resolved_model_name = None
+        for name in model_names:
+            # Exact match
+            if model_name == name:
+                resolved_model_name = name
+                break
+            # Match without version tag (use the versioned name)
+            elif model_name == name.split(":")[0]:
+                resolved_model_name = name
+                break
 
-        if not model_found:
+        if not resolved_model_name:
             error_msg = f"❌ Model '{model_name}' not found in local Ollama.\n\n"
 
             # Suggest pulling the model
@@ -465,6 +610,11 @@ def compute_embeddings_ollama(
             error_msg += "\n📚 Browse more: https://ollama.com/library"
             raise ValueError(error_msg)
 
+        # Use the resolved model name for all subsequent operations
+        if resolved_model_name != model_name:
+            logger.info(f"Resolved model name '{model_name}' to '{resolved_model_name}'")
+        model_name = resolved_model_name
+
         # Verify the model supports embeddings by testing it
         try:
             test_response = requests.post(
@@ -485,138 +635,148 @@ def compute_embeddings_ollama(
     except requests.exceptions.RequestException as e:
         logger.warning(f"Could not verify model existence: {e}")
 
-    # Process embeddings with optimized concurrent processing
-    import requests
+    # Determine batch size based on device availability
+    # Check for CUDA/MPS availability using torch if available
+    batch_size = 32  # Default for MPS/CPU
+    try:
+        import torch
 
-    def get_single_embedding(text_idx_tuple):
-        """Helper function to get embedding for a single text."""
-        text, idx = text_idx_tuple
-        max_retries = 3
-        retry_count = 0
+        if torch.cuda.is_available():
+            batch_size = 128  # CUDA gets larger batch size
+        elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
+            batch_size = 32  # MPS gets smaller batch size
+    except ImportError:
+        # If torch is not available, use conservative batch size
+        batch_size = 32
 
-        # Truncate very long texts to avoid API issues
-        truncated_text = text[:8000] if len(text) > 8000 else text
+    logger.info(f"Using batch size: {batch_size}")
 
-        while retry_count < max_retries:
-            try:
-                response = requests.post(
-                    f"{host}/api/embeddings",
-                    json={"model": model_name, "prompt": truncated_text},
-                    timeout=30,
-                )
-                response.raise_for_status()
+    def get_batch_embeddings(batch_texts):
+        """Get embeddings for a batch of texts."""
+        all_embeddings = []
+        failed_indices = []
 
-                result = response.json()
-                embedding = result.get("embedding")
+        for i, text in enumerate(batch_texts):
+            max_retries = 3
+            retry_count = 0
 
-                if embedding is None:
-                    raise ValueError(f"No embedding returned for text {idx}")
-
-                return idx, embedding
-
-            except requests.exceptions.Timeout:
-                retry_count += 1
-                if retry_count >= max_retries:
-                    logger.warning(f"Timeout for text {idx} after {max_retries} retries")
-                    return idx, None
-
-            except Exception as e:
-                if retry_count >= max_retries - 1:
-                    logger.error(f"Failed to get embedding for text {idx}: {e}")
-                    return idx, None
-                retry_count += 1
-
-        return idx, None
-
-    # Determine if we should use concurrent processing
-    use_concurrent = (
-        len(texts) > 5 and not is_build
-    )  # Don't use concurrent in build mode to avoid overwhelming
-    max_workers = min(4, len(texts))  # Limit concurrent requests to avoid overwhelming Ollama
-
-    all_embeddings = [None] * len(texts)  # Pre-allocate list to maintain order
-    failed_indices = []
-
-    if use_concurrent:
-        logger.info(
-            f"Using concurrent processing with {max_workers} workers for {len(texts)} texts"
-        )
-
-        with ThreadPoolExecutor(max_workers=max_workers) as executor:
-            # Submit all tasks
-            future_to_idx = {
-                executor.submit(get_single_embedding, (text, idx)): idx
-                for idx, text in enumerate(texts)
-            }
-
-            # Add progress bar for concurrent processing
-            try:
-                if is_build or len(texts) > 10:
-                    from tqdm import tqdm
-
-                    futures_iterator = tqdm(
-                        as_completed(future_to_idx),
-                        total=len(texts),
-                        desc="Computing Ollama embeddings",
-                    )
-                else:
-                    futures_iterator = as_completed(future_to_idx)
-            except ImportError:
-                futures_iterator = as_completed(future_to_idx)
-
-            # Collect results as they complete
-            for future in futures_iterator:
+            # Truncate very long texts to avoid API issues
+            truncated_text = text[:8000] if len(text) > 8000 else text
+            while retry_count < max_retries:
                 try:
-                    idx, embedding = future.result()
-                    if embedding is not None:
-                        all_embeddings[idx] = embedding
-                    else:
-                        failed_indices.append(idx)
+                    response = requests.post(
+                        f"{host}/api/embeddings",
+                        json={"model": model_name, "prompt": truncated_text},
+                        timeout=30,
+                    )
+                    response.raise_for_status()
+
+                    result = response.json()
+                    embedding = result.get("embedding")
+
+                    if embedding is None:
+                        raise ValueError(f"No embedding returned for text {i}")
+
+                    if not isinstance(embedding, list) or len(embedding) == 0:
+                        raise ValueError(f"Invalid embedding format for text {i}")
+
+                    all_embeddings.append(embedding)
+                    break
+
+                except requests.exceptions.Timeout:
+                    retry_count += 1
+                    if retry_count >= max_retries:
+                        logger.warning(f"Timeout for text {i} after {max_retries} retries")
+                        failed_indices.append(i)
+                        all_embeddings.append(None)
+                        break
+
                 except Exception as e:
-                    idx = future_to_idx[future]
-                    logger.error(f"Exception for text {idx}: {e}")
-                    failed_indices.append(idx)
+                    retry_count += 1
+                    if retry_count >= max_retries:
+                        logger.error(f"Failed to get embedding for text {i}: {e}")
+                        failed_indices.append(i)
+                        all_embeddings.append(None)
+                        break
+        return all_embeddings, failed_indices
 
+    # Process texts in batches
+    all_embeddings = []
+    all_failed_indices = []
+
+    # Setup progress bar if needed
+    show_progress = is_build or len(texts) > 10
+    try:
+        if show_progress:
+            from tqdm import tqdm
+    except ImportError:
+        show_progress = False
+
+    # Process batches
+    num_batches = (len(texts) + batch_size - 1) // batch_size
+
+    if show_progress:
+        batch_iterator = tqdm(range(num_batches), desc="Computing Ollama embeddings")
     else:
-        # Sequential processing with progress bar
-        show_progress = is_build or len(texts) > 10
+        batch_iterator = range(num_batches)
 
-        try:
-            if show_progress:
-                from tqdm import tqdm
+    for batch_idx in batch_iterator:
+        start_idx = batch_idx * batch_size
+        end_idx = min(start_idx + batch_size, len(texts))
+        batch_texts = texts[start_idx:end_idx]
 
-                iterator = tqdm(
-                    enumerate(texts), total=len(texts), desc="Computing Ollama embeddings"
-                )
-            else:
-                iterator = enumerate(texts)
-        except ImportError:
-            iterator = enumerate(texts)
+        batch_embeddings, batch_failed = get_batch_embeddings(batch_texts)
 
-        for idx, text in iterator:
-            result_idx, embedding = get_single_embedding((text, idx))
-            if embedding is not None:
-                all_embeddings[idx] = embedding
-            else:
-                failed_indices.append(idx)
+        # Adjust failed indices to global indices
+        global_failed = [start_idx + idx for idx in batch_failed]
+        all_failed_indices.extend(global_failed)
+        all_embeddings.extend(batch_embeddings)
 
     # Handle failed embeddings
-    if failed_indices:
-        if len(failed_indices) == len(texts):
+    if all_failed_indices:
+        if len(all_failed_indices) == len(texts):
             raise RuntimeError("Failed to compute any embeddings")
 
-        logger.warning(f"Failed to compute embeddings for {len(failed_indices)}/{len(texts)} texts")
+        logger.warning(
+            f"Failed to compute embeddings for {len(all_failed_indices)}/{len(texts)} texts"
+        )
 
         # Use zero embeddings as fallback for failed ones
         valid_embedding = next((e for e in all_embeddings if e is not None), None)
         if valid_embedding:
             embedding_dim = len(valid_embedding)
-            for idx in failed_indices:
-                all_embeddings[idx] = [0.0] * embedding_dim
+            for i, embedding in enumerate(all_embeddings):
+                if embedding is None:
+                    all_embeddings[i] = [0.0] * embedding_dim
 
-    # Remove None values and convert to numpy array
+    # Remove None values
     all_embeddings = [e for e in all_embeddings if e is not None]
 
+    if not all_embeddings:
+        raise RuntimeError("No valid embeddings were computed")
+
+    # Validate embedding dimensions
+    expected_dim = len(all_embeddings[0])
+    inconsistent_dims = []
+    for i, embedding in enumerate(all_embeddings):
+        if len(embedding) != expected_dim:
+            inconsistent_dims.append((i, len(embedding)))
+
+    if inconsistent_dims:
+        error_msg = f"Ollama returned inconsistent embedding dimensions. Expected {expected_dim}, but got:\n"
+        for idx, dim in inconsistent_dims[:10]:  # Show first 10 inconsistent ones
+            error_msg += f"  - Text {idx}: {dim} dimensions\n"
+        if len(inconsistent_dims) > 10:
+            error_msg += f"  ... and {len(inconsistent_dims) - 10} more\n"
+        error_msg += f"\nThis is likely an Ollama API bug with model '{model_name}'. Please try:\n"
+        error_msg += "1. Restart Ollama service: 'ollama serve'\n"
+        error_msg += f"2. Re-pull the model: 'ollama pull {model_name}'\n"
+        error_msg += (
+            "3. Use sentence-transformers instead: --embedding-mode sentence-transformers\n"
+        )
+        error_msg += "4. Report this issue to Ollama: https://github.com/ollama/ollama/issues"
+        raise ValueError(error_msg)
+
     # Convert to numpy array and normalize
     embeddings = np.array(all_embeddings, dtype=np.float32)
 
@@ -627,3 +787,83 @@ def compute_embeddings_ollama(
     logger.info(f"Generated {len(embeddings)} embeddings, dimension: {embeddings.shape[1]}")
 
     return embeddings
+
+
+def compute_embeddings_gemini(
+    texts: list[str], model_name: str = "text-embedding-004", is_build: bool = False
+) -> np.ndarray:
+    """
+    Compute embeddings using Google Gemini API.
+
+    Args:
+        texts: List of texts to compute embeddings for
+        model_name: Gemini model name (default: "text-embedding-004")
+        is_build: Whether this is a build operation (shows progress bar)
+
+    Returns:
+        Embeddings array, shape: (len(texts), embedding_dim)
+    """
+    try:
+        import os
+
+        import google.genai as genai
+    except ImportError as e:
+        raise ImportError(f"Google GenAI package not installed: {e}")
+
+    api_key = os.getenv("GEMINI_API_KEY")
+    if not api_key:
+        raise RuntimeError("GEMINI_API_KEY environment variable not set")
+
+    # Cache Gemini client
+    cache_key = "gemini_client"
+    if cache_key in _model_cache:
+        client = _model_cache[cache_key]
+    else:
+        client = genai.Client(api_key=api_key)
+        _model_cache[cache_key] = client
+        logger.info("Gemini client cached")
+
+    logger.info(
+        f"Computing embeddings for {len(texts)} texts using Gemini API, model: '{model_name}'"
+    )
+
+    # Gemini supports batch embedding
+    max_batch_size = 100  # Conservative batch size for Gemini
+    all_embeddings = []
+
+    try:
+        from tqdm import tqdm
+
+        total_batches = (len(texts) + max_batch_size - 1) // max_batch_size
+        batch_range = range(0, len(texts), max_batch_size)
+        batch_iterator = tqdm(
+            batch_range, desc="Computing embeddings", unit="batch", total=total_batches
+        )
+    except ImportError:
+        # Fallback when tqdm is not available
+        batch_iterator = range(0, len(texts), max_batch_size)
+
+    for i in batch_iterator:
+        batch_texts = texts[i : i + max_batch_size]
+
+        try:
+            # Use the embed_content method from the new Google GenAI SDK
+            response = client.models.embed_content(
+                model=model_name,
+                contents=batch_texts,
+                config=genai.types.EmbedContentConfig(
+                    task_type="RETRIEVAL_DOCUMENT"  # For document embedding
+                ),
+            )
+
+            # Extract embeddings from response
+            for embedding_data in response.embeddings:
+                all_embeddings.append(embedding_data.values)
+        except Exception as e:
+            logger.error(f"Batch {i} failed: {e}")
+            raise
+
+    embeddings = np.array(all_embeddings, dtype=np.float32)
+    logger.info(f"Generated {len(embeddings)} embeddings, dimension: {embeddings.shape[1]}")
+
+    return embeddings

packages/leann-core/src/leann/embedding_server_manager.py

@@ -1,7 +1,6 @@
 import atexit
 import logging
 import os
-import signal
 import socket
 import subprocess
 import sys
@@ -9,7 +8,7 @@ import time
 from pathlib import Path
 from typing import Optional
 
-import psutil
+# Lightweight, self-contained server manager with no cross-process inspection
 
 # Set up logging based on environment variable
 LOG_LEVEL = os.getenv("LEANN_LOG_LEVEL", "WARNING").upper()
@@ -44,130 +43,7 @@ def _check_port(port: int) -> bool:
         return s.connect_ex(("localhost", port)) == 0
 
 
-def _check_process_matches_config(
-    port: int, expected_model: str, expected_passages_file: str
-) -> bool:
-    """
-    Check if the process using the port matches our expected model and passages file.
-    Returns True if matches, False otherwise.
-    """
-    try:
-        for proc in psutil.process_iter(["pid", "cmdline"]):
-            if not _is_process_listening_on_port(proc, port):
-                continue
-
-            cmdline = proc.info["cmdline"]
-            if not cmdline:
-                continue
-
-            return _check_cmdline_matches_config(
-                cmdline, port, expected_model, expected_passages_file
-            )
-
-        logger.debug(f"No process found listening on port {port}")
-        return False
-
-    except Exception as e:
-        logger.warning(f"Could not check process on port {port}: {e}")
-        return False
-
-
-def _is_process_listening_on_port(proc, port: int) -> bool:
-    """Check if a process is listening on the given port."""
-    try:
-        connections = proc.net_connections()
-        for conn in connections:
-            if conn.laddr.port == port and conn.status == psutil.CONN_LISTEN:
-                return True
-        return False
-    except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
-        return False
-
-
-def _check_cmdline_matches_config(
-    cmdline: list, port: int, expected_model: str, expected_passages_file: str
-) -> bool:
-    """Check if command line matches our expected configuration."""
-    cmdline_str = " ".join(cmdline)
-    logger.debug(f"Found process on port {port}: {cmdline_str}")
-
-    # Check if it's our embedding server
-    is_embedding_server = any(
-        server_type in cmdline_str
-        for server_type in [
-            "embedding_server",
-            "leann_backend_diskann.embedding_server",
-            "leann_backend_hnsw.hnsw_embedding_server",
-        ]
-    )
-
-    if not is_embedding_server:
-        logger.debug(f"Process on port {port} is not our embedding server")
-        return False
-
-    # Check model name
-    model_matches = _check_model_in_cmdline(cmdline, expected_model)
-
-    # Check passages file if provided
-    passages_matches = _check_passages_in_cmdline(cmdline, expected_passages_file)
-
-    result = model_matches and passages_matches
-    logger.debug(
-        f"model_matches: {model_matches}, passages_matches: {passages_matches}, overall: {result}"
-    )
-    return result
-
-
-def _check_model_in_cmdline(cmdline: list, expected_model: str) -> bool:
-    """Check if the command line contains the expected model."""
-    if "--model-name" not in cmdline:
-        return False
-
-    model_idx = cmdline.index("--model-name")
-    if model_idx + 1 >= len(cmdline):
-        return False
-
-    actual_model = cmdline[model_idx + 1]
-    return actual_model == expected_model
-
-
-def _check_passages_in_cmdline(cmdline: list, expected_passages_file: str) -> bool:
-    """Check if the command line contains the expected passages file."""
-    if "--passages-file" not in cmdline:
-        return False  # Expected but not found
-
-    passages_idx = cmdline.index("--passages-file")
-    if passages_idx + 1 >= len(cmdline):
-        return False
-
-    actual_passages = cmdline[passages_idx + 1]
-    expected_path = Path(expected_passages_file).resolve()
-    actual_path = Path(actual_passages).resolve()
-    return actual_path == expected_path
-
-
-def _find_compatible_port_or_next_available(
-    start_port: int, model_name: str, passages_file: str, max_attempts: int = 100
-) -> tuple[int, bool]:
-    """
-    Find a port that either has a compatible server or is available.
-    Returns (port, is_compatible) where is_compatible indicates if we found a matching server.
-    """
-    for port in range(start_port, start_port + max_attempts):
-        if not _check_port(port):
-            # Port is available
-            return port, False
-
-        # Port is in use, check if it's compatible
-        if _check_process_matches_config(port, model_name, passages_file):
-            logger.info(f"Found compatible server on port {port}")
-            return port, True
-        else:
-            logger.info(f"Port {port} has incompatible server, trying next port...")
-
-    raise RuntimeError(
-        f"Could not find compatible or available port in range {start_port}-{start_port + max_attempts}"
-    )
+# Note: All cross-process scanning helpers removed for simplicity
 
 
 class EmbeddingServerManager:
@@ -186,7 +62,16 @@ class EmbeddingServerManager:
         self.backend_module_name = backend_module_name
         self.server_process: Optional[subprocess.Popen] = None
         self.server_port: Optional[int] = None
+        # Track last-started config for in-process reuse only
+        self._server_config: Optional[dict] = None
         self._atexit_registered = False
+        # Also register a weakref finalizer to ensure cleanup when manager is GC'ed
+        try:
+            import weakref
+
+            self._finalizer = weakref.finalize(self, self._finalize_process)
+        except Exception:
+            self._finalizer = None
 
     def start_server(
         self,
@@ -196,26 +81,24 @@ class EmbeddingServerManager:
         **kwargs,
     ) -> tuple[bool, int]:
         """Start the embedding server."""
-        passages_file = kwargs.get("passages_file")
+        # passages_file may be present in kwargs for server CLI, but we don't need it here
 
-        # Check if we have a compatible server already running
-        if self._has_compatible_running_server(model_name, passages_file):
-            logger.info("Found compatible running server!")
-            return True, port
+        # If this manager already has a live server, just reuse it
+        if self.server_process and self.server_process.poll() is None and self.server_port:
+            logger.info("Reusing in-process server")
+            return True, self.server_port
 
         # For Colab environment, use a different strategy
         if _is_colab_environment():
             logger.info("Detected Colab environment, using alternative startup strategy")
             return self._start_server_colab(port, model_name, embedding_mode, **kwargs)
 
-        # Find a compatible port or next available
-        actual_port, is_compatible = _find_compatible_port_or_next_available(
-            port, model_name, passages_file
-        )
-
-        if is_compatible:
-            logger.info(f"Found compatible server on port {actual_port}")
-            return True, actual_port
+        # Always pick a fresh available port
+        try:
+            actual_port = _get_available_port(port)
+        except RuntimeError:
+            logger.error("No available ports found")
+            return False, port
 
         # Start a new server
         return self._start_new_server(actual_port, model_name, embedding_mode, **kwargs)
@@ -248,17 +131,7 @@ class EmbeddingServerManager:
             logger.error(f"Failed to start embedding server in Colab: {e}")
             return False, actual_port
 
-    def _has_compatible_running_server(self, model_name: str, passages_file: str) -> bool:
-        """Check if we have a compatible running server."""
-        if not (self.server_process and self.server_process.poll() is None and self.server_port):
-            return False
-
-        if _check_process_matches_config(self.server_port, model_name, passages_file):
-            logger.info(f"Existing server process (PID {self.server_process.pid}) is compatible")
-            return True
-
-        logger.info("Existing server process is incompatible. Should start a new server.")
-        return False
+    # Note: No compatibility check needed; manager is per-searcher and configs are stable per instance
 
     def _start_new_server(
         self, port: int, model_name: str, embedding_mode: str, **kwargs
@@ -305,33 +178,62 @@ class EmbeddingServerManager:
         project_root = Path(__file__).parent.parent.parent.parent.parent
         logger.info(f"Command: {' '.join(command)}")
 
-        # In CI environment, redirect output to avoid buffer deadlock
-        # Embedding servers use many print statements that can fill buffers
+        # In CI environment, redirect stdout to avoid buffer deadlock but keep stderr for debugging
+        # Embedding servers use many print statements that can fill stdout buffers
         is_ci = os.environ.get("CI") == "true"
         if is_ci:
             stdout_target = subprocess.DEVNULL
-            stderr_target = subprocess.DEVNULL
-            logger.info("CI environment detected, redirecting embedding server output to DEVNULL")
+            stderr_target = None  # Keep stderr for error debugging in CI
+            logger.info(
+                "CI environment detected, redirecting embedding server stdout to DEVNULL, keeping stderr"
+            )
         else:
             stdout_target = None  # Direct to console for visible logs
             stderr_target = None  # Direct to console for visible logs
 
-        # IMPORTANT: Use a new session so we can manage the whole process group reliably
+        # Start embedding server subprocess
+        logger.info(f"Starting server process with command: {' '.join(command)}")
         self.server_process = subprocess.Popen(
             command,
             cwd=project_root,
             stdout=stdout_target,
             stderr=stderr_target,
-            start_new_session=True,
         )
         self.server_port = port
+        # Record config for in-process reuse
+        try:
+            self._server_config = {
+                "model_name": command[command.index("--model-name") + 1]
+                if "--model-name" in command
+                else "",
+                "passages_file": command[command.index("--passages-file") + 1]
+                if "--passages-file" in command
+                else "",
+                "embedding_mode": command[command.index("--embedding-mode") + 1]
+                if "--embedding-mode" in command
+                else "sentence-transformers",
+            }
+        except Exception:
+            self._server_config = {
+                "model_name": "",
+                "passages_file": "",
+                "embedding_mode": "sentence-transformers",
+            }
         logger.info(f"Server process started with PID: {self.server_process.pid}")
 
         # Register atexit callback only when we actually start a process
         if not self._atexit_registered:
-            # Use a lambda to avoid issues with bound methods
-            atexit.register(lambda: self.stop_server() if self.server_process else None)
+            # Always attempt best-effort finalize at interpreter exit
+            atexit.register(self._finalize_process)
             self._atexit_registered = True
+        # Touch finalizer so it knows there is a live process
+        if getattr(self, "_finalizer", None) is not None and not self._finalizer.alive:
+            try:
+                import weakref
+
+                self._finalizer = weakref.finalize(self, self._finalize_process)
+            except Exception:
+                pass
 
     def _wait_for_server_ready(self, port: int) -> tuple[bool, int]:
         """Wait for the server to be ready."""
@@ -356,34 +258,35 @@ class EmbeddingServerManager:
         if not self.server_process:
             return
 
-        if self.server_process.poll() is not None:
+        if self.server_process and self.server_process.poll() is not None:
             # Process already terminated
             self.server_process = None
+            self.server_port = None
+            self._server_config = None
             return
 
         logger.info(
             f"Terminating server process (PID: {self.server_process.pid}) for backend {self.backend_module_name}..."
         )
-        # Try terminating the whole process group first (POSIX)
+
+        # Use simple termination first; if the server installed signal handlers,
+        # it will exit cleanly. Otherwise escalate to kill after a short wait.
         try:
-            pgid = os.getpgid(self.server_process.pid)
-            os.killpg(pgid, signal.SIGTERM)
-        except Exception:
-            # Fallback to terminating just the process
             self.server_process.terminate()
+        except Exception:
+            pass
 
         try:
-            self.server_process.wait(timeout=3)
-            logger.info(f"Server process {self.server_process.pid} terminated.")
+            self.server_process.wait(timeout=5)  # Give more time for graceful shutdown
+            logger.info(f"Server process {self.server_process.pid} terminated gracefully.")
         except subprocess.TimeoutExpired:
             logger.warning(
-                f"Server process {self.server_process.pid} did not terminate gracefully within 3 seconds, killing it."
+                f"Server process {self.server_process.pid} did not terminate within 5 seconds, force killing..."
             )
             try:
-                pgid = os.getpgid(self.server_process.pid)
-                os.killpg(pgid, signal.SIGKILL)
-            except Exception:
                 self.server_process.kill()
+            except Exception:
+                pass
             try:
                 self.server_process.wait(timeout=2)
                 logger.info(f"Server process {self.server_process.pid} killed successfully.")
@@ -391,32 +294,58 @@ class EmbeddingServerManager:
                 logger.error(
                     f"Failed to kill server process {self.server_process.pid} - it may be hung"
                 )
-                # Don't hang indefinitely
 
-        # Clean up process resources without waiting
-        # The process should already be terminated/killed above
-        # Don't wait here as it can hang CI indefinitely
-        self.server_process = None
+        # Clean up process resources with timeout to avoid CI hang
+        try:
+            # Use shorter timeout in CI environments
+            is_ci = os.environ.get("CI") == "true"
+            timeout = 3 if is_ci else 10
+            self.server_process.wait(timeout=timeout)
+            logger.info(f"Server process {self.server_process.pid} cleanup completed")
+        except subprocess.TimeoutExpired:
+            logger.warning(f"Process cleanup timeout after {timeout}s, proceeding anyway")
+        except Exception as e:
+            logger.warning(f"Error during process cleanup: {e}")
+        finally:
+            self.server_process = None
+            self.server_port = None
+            self._server_config = None
+
+    def _finalize_process(self) -> None:
+        """Best-effort cleanup used by weakref.finalize/atexit."""
+        try:
+            self.stop_server()
+        except Exception:
+            pass
+
+    def _adopt_existing_server(self, *args, **kwargs) -> None:
+        # Removed: cross-process adoption no longer supported
+        return
 
     def _launch_server_process_colab(self, command: list, port: int) -> None:
         """Launch the server process with Colab-specific settings."""
         logger.info(f"Colab Command: {' '.join(command)}")
 
-        # In Colab, redirect to DEVNULL to avoid pipe blocking
-        # PIPE without reading can cause hangs
+        # In Colab, we need to be more careful about process management
         self.server_process = subprocess.Popen(
             command,
-            stdout=subprocess.DEVNULL,
-            stderr=subprocess.DEVNULL,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
             text=True,
         )
         self.server_port = port
         logger.info(f"Colab server process started with PID: {self.server_process.pid}")
 
-        # Register atexit callback
+        # Register atexit callback (unified)
         if not self._atexit_registered:
-            atexit.register(lambda: self.stop_server() if self.server_process else None)
+            atexit.register(self._finalize_process)
             self._atexit_registered = True
+        # Record config for in-process reuse is best-effort in Colab mode
+        self._server_config = {
+            "model_name": "",
+            "passages_file": "",
+            "embedding_mode": "sentence-transformers",
+        }
 
     def _wait_for_server_ready_colab(self, port: int) -> tuple[bool, int]:
         """Wait for the server to be ready with Colab-specific timeout."""

packages/leann-core/src/leann/mcp.py

@@ -64,19 +64,6 @@ def handle_request(request):
                             "required": ["index_name", "query"],
                         },
                     },
-                    {
-                        "name": "leann_status",
-                        "description": "📊 Check the health and stats of your code indexes - like a medical checkup for your codebase knowledge!",
-                        "inputSchema": {
-                            "type": "object",
-                            "properties": {
-                                "index_name": {
-                                    "type": "string",
-                                    "description": "Optional: Name of specific index to check. If not provided, shows status of all indexes.",
-                                }
-                            },
-                        },
-                    },
                     {
                         "name": "leann_list",
                         "description": "📋 Show all your indexed codebases - your personal code library! Use this to see what's available for search.",
@@ -107,7 +94,7 @@ def handle_request(request):
                         },
                     }
 
-                # Build simplified command
+                # Build simplified command with non-interactive flag for MCP compatibility
                 cmd = [
                     "leann",
                     "search",
@@ -115,19 +102,10 @@ def handle_request(request):
                     args["query"],
                     f"--top-k={args.get('top_k', 5)}",
                     f"--complexity={args.get('complexity', 32)}",
+                    "--non-interactive",
                 ]
-
                 result = subprocess.run(cmd, capture_output=True, text=True)
 
-            elif tool_name == "leann_status":
-                if args.get("index_name"):
-                    # Check specific index status - for now, we'll use leann list and filter
-                    result = subprocess.run(["leann", "list"], capture_output=True, text=True)
-                    # We could enhance this to show more detailed status per index
-                else:
-                    # Show all indexes status
-                    result = subprocess.run(["leann", "list"], capture_output=True, text=True)
-
             elif tool_name == "leann_list":
                 result = subprocess.run(["leann", "list"], capture_output=True, text=True)
 

packages/leann-core/src/leann/metadata_filter.py

@@ -0,0 +1,240 @@
+"""
+Metadata filtering engine for LEANN search results.
+
+This module provides generic metadata filtering capabilities that can be applied
+to search results from any LEANN backend. The filtering supports various
+operators for different data types including numbers, strings, booleans, and lists.
+"""
+
+import logging
+from typing import Any, Union
+
+logger = logging.getLogger(__name__)
+
+# Type alias for filter specifications
+FilterValue = Union[str, int, float, bool, list]
+FilterSpec = dict[str, FilterValue]
+MetadataFilters = dict[str, FilterSpec]
+
+
+class MetadataFilterEngine:
+    """
+    Engine for evaluating metadata filters against search results.
+
+    Supports various operators for filtering based on metadata fields:
+    - Comparison: ==, !=, <, <=, >, >=
+    - Membership: in, not_in
+    - String operations: contains, starts_with, ends_with
+    - Boolean operations: is_true, is_false
+    """
+
+    def __init__(self):
+        """Initialize the filter engine with supported operators."""
+        self.operators = {
+            "==": self._equals,
+            "!=": self._not_equals,
+            "<": self._less_than,
+            "<=": self._less_than_or_equal,
+            ">": self._greater_than,
+            ">=": self._greater_than_or_equal,
+            "in": self._in,
+            "not_in": self._not_in,
+            "contains": self._contains,
+            "starts_with": self._starts_with,
+            "ends_with": self._ends_with,
+            "is_true": self._is_true,
+            "is_false": self._is_false,
+        }
+
+    def apply_filters(
+        self, search_results: list[dict[str, Any]], metadata_filters: MetadataFilters
+    ) -> list[dict[str, Any]]:
+        """
+        Apply metadata filters to a list of search results.
+
+        Args:
+            search_results: List of result dictionaries, each containing 'metadata' field
+            metadata_filters: Dictionary of filter specifications
+                Format: {"field_name": {"operator": value}}
+
+        Returns:
+            Filtered list of search results
+        """
+        if not metadata_filters:
+            return search_results
+
+        logger.debug(f"Applying filters: {metadata_filters}")
+        logger.debug(f"Input results count: {len(search_results)}")
+
+        filtered_results = []
+        for result in search_results:
+            if self._evaluate_filters(result, metadata_filters):
+                filtered_results.append(result)
+
+        logger.debug(f"Filtered results count: {len(filtered_results)}")
+        return filtered_results
+
+    def _evaluate_filters(self, result: dict[str, Any], filters: MetadataFilters) -> bool:
+        """
+        Evaluate all filters against a single search result.
+
+        All filters must pass (AND logic) for the result to be included.
+
+        Args:
+            result: Full search result dictionary (including metadata, text, etc.)
+            filters: Filter specifications to evaluate
+
+        Returns:
+            True if all filters pass, False otherwise
+        """
+        for field_name, filter_spec in filters.items():
+            if not self._evaluate_field_filter(result, field_name, filter_spec):
+                return False
+        return True
+
+    def _evaluate_field_filter(
+        self, result: dict[str, Any], field_name: str, filter_spec: FilterSpec
+    ) -> bool:
+        """
+        Evaluate a single field filter against a search result.
+
+        Args:
+            result: Full search result dictionary
+            field_name: Name of the field to filter on
+            filter_spec: Filter specification for this field
+
+        Returns:
+            True if the filter passes, False otherwise
+        """
+        # First check top-level fields, then check metadata
+        field_value = result.get(field_name)
+        if field_value is None:
+            # Try to get from metadata if not found at top level
+            metadata = result.get("metadata", {})
+            field_value = metadata.get(field_name)
+
+        # Handle missing fields - they fail all filters except existence checks
+        if field_value is None:
+            logger.debug(f"Field '{field_name}' not found in result or metadata")
+            return False
+
+        # Evaluate each operator in the filter spec
+        for operator, expected_value in filter_spec.items():
+            if operator not in self.operators:
+                logger.warning(f"Unsupported operator: {operator}")
+                return False
+
+            try:
+                if not self.operators[operator](field_value, expected_value):
+                    logger.debug(
+                        f"Filter failed: {field_name} {operator} {expected_value} "
+                        f"(actual: {field_value})"
+                    )
+                    return False
+            except Exception as e:
+                logger.warning(
+                    f"Error evaluating filter {field_name} {operator} {expected_value}: {e}"
+                )
+                return False
+
+        return True
+
+    # Comparison operators
+    def _equals(self, field_value: Any, expected_value: Any) -> bool:
+        """Check if field value equals expected value."""
+        return field_value == expected_value
+
+    def _not_equals(self, field_value: Any, expected_value: Any) -> bool:
+        """Check if field value does not equal expected value."""
+        return field_value != expected_value
+
+    def _less_than(self, field_value: Any, expected_value: Any) -> bool:
+        """Check if field value is less than expected value."""
+        return self._numeric_compare(field_value, expected_value, lambda a, b: a < b)
+
+    def _less_than_or_equal(self, field_value: Any, expected_value: Any) -> bool:
+        """Check if field value is less than or equal to expected value."""
+        return self._numeric_compare(field_value, expected_value, lambda a, b: a <= b)
+
+    def _greater_than(self, field_value: Any, expected_value: Any) -> bool:
+        """Check if field value is greater than expected value."""
+        return self._numeric_compare(field_value, expected_value, lambda a, b: a > b)
+
+    def _greater_than_or_equal(self, field_value: Any, expected_value: Any) -> bool:
+        """Check if field value is greater than or equal to expected value."""
+        return self._numeric_compare(field_value, expected_value, lambda a, b: a >= b)
+
+    # Membership operators
+    def _in(self, field_value: Any, expected_value: Any) -> bool:
+        """Check if field value is in the expected list/collection."""
+        if not isinstance(expected_value, (list, tuple, set)):
+            raise ValueError("'in' operator requires a list, tuple, or set")
+        return field_value in expected_value
+
+    def _not_in(self, field_value: Any, expected_value: Any) -> bool:
+        """Check if field value is not in the expected list/collection."""
+        if not isinstance(expected_value, (list, tuple, set)):
+            raise ValueError("'not_in' operator requires a list, tuple, or set")
+        return field_value not in expected_value
+
+    # String operators
+    def _contains(self, field_value: Any, expected_value: Any) -> bool:
+        """Check if field value contains the expected substring."""
+        field_str = str(field_value)
+        expected_str = str(expected_value)
+        return expected_str in field_str
+
+    def _starts_with(self, field_value: Any, expected_value: Any) -> bool:
+        """Check if field value starts with the expected prefix."""
+        field_str = str(field_value)
+        expected_str = str(expected_value)
+        return field_str.startswith(expected_str)
+
+    def _ends_with(self, field_value: Any, expected_value: Any) -> bool:
+        """Check if field value ends with the expected suffix."""
+        field_str = str(field_value)
+        expected_str = str(expected_value)
+        return field_str.endswith(expected_str)
+
+    # Boolean operators
+    def _is_true(self, field_value: Any, expected_value: Any) -> bool:
+        """Check if field value is truthy."""
+        return bool(field_value)
+
+    def _is_false(self, field_value: Any, expected_value: Any) -> bool:
+        """Check if field value is falsy."""
+        return not bool(field_value)
+
+    # Helper methods
+    def _numeric_compare(self, field_value: Any, expected_value: Any, compare_func) -> bool:
+        """
+        Helper for numeric comparisons with type coercion.
+
+        Args:
+            field_value: Value from metadata
+            expected_value: Value to compare against
+            compare_func: Comparison function to apply
+
+        Returns:
+            Result of comparison
+        """
+        try:
+            # Try to convert both values to numbers for comparison
+            if isinstance(field_value, str) and isinstance(expected_value, str):
+                # String comparison if both are strings
+                return compare_func(field_value, expected_value)
+
+            # Numeric comparison - attempt to convert to float
+            field_num = (
+                float(field_value) if not isinstance(field_value, (int, float)) else field_value
+            )
+            expected_num = (
+                float(expected_value)
+                if not isinstance(expected_value, (int, float))
+                else expected_value
+            )
+
+            return compare_func(field_num, expected_num)
+        except (ValueError, TypeError):
+            # Fall back to string comparison if numeric conversion fails
+            return compare_func(str(field_value), str(expected_value))

packages/leann-core/src/leann/registry.py

@@ -2,11 +2,17 @@
 
 import importlib
 import importlib.metadata
-from typing import TYPE_CHECKING
+import json
+import logging
+from pathlib import Path
+from typing import TYPE_CHECKING, Optional, Union
 
 if TYPE_CHECKING:
     from leann.interface import LeannBackendFactoryInterface
 
+# Set up logger for this module
+logger = logging.getLogger(__name__)
+
 BACKEND_REGISTRY: dict[str, "LeannBackendFactoryInterface"] = {}
 
 
@@ -14,7 +20,7 @@ def register_backend(name: str):
     """A decorator to register a new backend class."""
 
     def decorator(cls):
-        print(f"INFO: Registering backend '{name}'")
+        logger.debug(f"Registering backend '{name}'")
         BACKEND_REGISTRY[name] = cls
         return cls
 
@@ -39,3 +45,54 @@ def autodiscover_backends():
             # print(f"WARN: Could not import backend module '{backend_module_name}': {e}")
             pass
     # print("INFO: Backend auto-discovery finished.")
+
+
+def register_project_directory(project_dir: Optional[Union[str, Path]] = None):
+    """
+    Register a project directory in the global LEANN registry.
+
+    This allows `leann list` to discover indexes created by apps or other tools.
+
+    Args:
+        project_dir: Directory to register. If None, uses current working directory.
+    """
+    if project_dir is None:
+        project_dir = Path.cwd()
+    else:
+        project_dir = Path(project_dir)
+
+    # Only register directories that have some kind of LEANN content
+    # Either .leann/indexes/ (CLI format) or *.leann.meta.json files (apps format)
+    has_cli_indexes = (project_dir / ".leann" / "indexes").exists()
+    has_app_indexes = any(project_dir.rglob("*.leann.meta.json"))
+
+    if not (has_cli_indexes or has_app_indexes):
+        # Don't register if there are no LEANN indexes
+        return
+
+    global_registry = Path.home() / ".leann" / "projects.json"
+    global_registry.parent.mkdir(exist_ok=True)
+
+    project_str = str(project_dir.resolve())
+
+    # Load existing registry
+    projects = []
+    if global_registry.exists():
+        try:
+            with open(global_registry) as f:
+                projects = json.load(f)
+        except Exception:
+            logger.debug("Could not load existing project registry")
+            projects = []
+
+    # Add project if not already present
+    if project_str not in projects:
+        projects.append(project_str)
+
+        # Save updated registry
+        try:
+            with open(global_registry, "w") as f:
+                json.dump(projects, f, indent=2)
+            logger.debug(f"Registered project directory: {project_str}")
+        except Exception as e:
+            logger.warning(f"Could not save project registry: {e}")

packages/leann-core/src/leann/searcher_base.py

@@ -132,15 +132,10 @@ class BaseSearcher(LeannBackendSearcherInterface, ABC):
         import msgpack
         import zmq
 
-        context = None
-        socket = None
         try:
             context = zmq.Context()
             socket = context.socket(zmq.REQ)
-            socket.setsockopt(zmq.LINGER, 0)  # Don't block on close
-            socket.setsockopt(zmq.RCVTIMEO, 5000)  # 5 second timeout
-            socket.setsockopt(zmq.SNDTIMEO, 5000)  # 5 second timeout
-            socket.setsockopt(zmq.IMMEDIATE, 1)  # Don't wait for connection
+            socket.setsockopt(zmq.RCVTIMEO, 30000)  # 30 second timeout
             socket.connect(f"tcp://localhost:{zmq_port}")
 
             # Send embedding request
@@ -152,6 +147,9 @@ class BaseSearcher(LeannBackendSearcherInterface, ABC):
             response_bytes = socket.recv()
             response = msgpack.unpackb(response_bytes)
 
+            socket.close()
+            context.term()
+
             # Convert response to numpy array
             if isinstance(response, list) and len(response) > 0:
                 return np.array(response, dtype=np.float32)
@@ -160,11 +158,6 @@ class BaseSearcher(LeannBackendSearcherInterface, ABC):
 
         except Exception as e:
             raise RuntimeError(f"Failed to compute embeddings via server: {e}")
-        finally:
-            if socket:
-                socket.close(linger=0)
-            if context:
-                context.term()
 
     @abstractmethod
     def search(
@@ -198,27 +191,7 @@ class BaseSearcher(LeannBackendSearcherInterface, ABC):
         """
         pass
 
-    def cleanup(self):
-        """Cleanup resources including embedding server and ZMQ connections."""
-        # Stop embedding server
+    def __del__(self):
+        """Ensures the embedding server is stopped when the searcher is destroyed."""
         if hasattr(self, "embedding_server_manager"):
             self.embedding_server_manager.stop_server()
-
-        # Set ZMQ linger but don't terminate global context
-        try:
-            import zmq
-
-            # Just set linger on the global instance
-            ctx = zmq.Context.instance()
-            ctx.linger = 0
-            # NEVER call ctx.term() on the global instance
-        except Exception:
-            pass
-
-    def __del__(self):
-        """Ensures resources are cleaned up when the searcher is destroyed."""
-        try:
-            self.cleanup()
-        except Exception:
-            # Ignore errors during destruction
-            pass

packages/leann-mcp/README.md

@@ -2,29 +2,33 @@
 
 Transform your development workflow with intelligent code assistance using LEANN's semantic search directly in Claude Code.
 
+For agent-facing discovery details, see `llms.txt` in the repository root.
+
 ## Prerequisites
 
-**Step 1:** First, complete the basic LEANN installation following the [📦 Installation guide](../../README.md#installation) in the root README:
+Install LEANN globally for MCP integration (with default backend):
 
 ```bash
-uv venv
-source .venv/bin/activate
-uv pip install leann
+uv tool install leann-core --with leann
 ```
-
-**Step 2:** Install LEANN globally for MCP integration:
-```bash
-uv tool install leann-core
-```
-
-This makes the `leann` command available system-wide, which `leann_mcp` requires.
+This installs the `leann` CLI into an isolated tool environment and includes both backends so `leann build` works out-of-the-box.
 
 ## 🚀 Quick Setup
 
-Add the LEANN MCP server to Claude Code:
+Add the LEANN MCP server to Claude Code. Choose the scope based on how widely you want it available. Below is the command to install it globally; if you prefer a local install, skip this step:
 
 ```bash
-claude mcp add leann-server -- leann_mcp
+# Global (recommended): available in all projects for your user
+claude mcp add --scope user leann-server -- leann_mcp
+```
+
+- `leann-server`: the display name of the MCP server in Claude Code (you can change it).
+- `leann_mcp`: the Python entry point installed with LEANN that starts the MCP server.
+
+Verify it is registered globally:
+
+```bash
+claude mcp list | cat
 ```
 
 ## 🛠️ Available Tools
@@ -33,19 +37,64 @@ Once connected, you'll have access to these powerful semantic search tools in Cl
 
 - **`leann_list`** - List all available indexes across your projects
 - **`leann_search`** - Perform semantic searches across code and documents
-- **`leann_ask`** - Ask natural language questions and get AI-powered answers from your codebase
+
 
 ## 🎯 Quick Start Example
 
 ```bash
+# Add locally if you did not add it globally (current folder only; default if --scope is omitted)
+claude mcp add leann-server -- leann_mcp
+
 # Build an index for your project (change to your actual path)
-leann build my-project --docs ./
+# See the advanced examples below for more ways to configure indexing
+# Set the index name (replace 'my-project' with your own)
+leann build my-project --docs $(git ls-files)
 
 # Start Claude Code
 claude
 ```
 
-**Try this in Claude Code:**
+## 🚀 Advanced Usage Examples to build the index
+
+### Index Entire Git Repository
+```bash
+# Index all tracked files in your Git repository.
+# Note: submodules are currently skipped; we can add them back if needed.
+leann build my-repo --docs $(git ls-files) --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw
+
+# Index only tracked Python files from Git.
+leann build my-python-code --docs $(git ls-files "*.py") --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw
+
+# If you encounter empty requests caused by empty files (e.g., __init__.py), exclude zero-byte files. Thanks @ww2283 for pointing [that](https://github.com/yichuan-w/LEANN/issues/48) out
+leann build leann-prospec-lig --docs $(find ./src -name "*.py" -not -empty) --embedding-mode openai --embedding-model text-embedding-3-small
+```
+
+### Multiple Directories and Files
+```bash
+# Index multiple directories
+leann build my-codebase --docs ./src ./tests ./docs ./config --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw
+
+# Mix files and directories
+leann build my-project --docs ./README.md ./src/ ./package.json ./docs/ --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw
+
+# Specific files only
+leann build my-configs --docs ./tsconfig.json ./package.json ./webpack.config.js --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw
+```
+
+### Advanced Git Integration
+```bash
+# Index recently modified files
+leann build recent-changes --docs $(git diff --name-only HEAD~10..HEAD) --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw
+
+# Index files matching pattern
+leann build frontend --docs $(git ls-files "*.tsx" "*.ts" "*.jsx" "*.js") --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw
+
+# Index documentation and config files
+leann build docs-and-configs --docs $(git ls-files "*.md" "*.yml" "*.yaml" "*.json" "*.toml") --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw
+```
+
+
+## **Try this in Claude Code:**
 ```
 Help me understand this codebase. List available indexes and search for authentication patterns.
 ```
@@ -54,6 +103,7 @@ Help me understand this codebase. List available indexes and search for authenti
   <img src="../../assets/claude_code_leann.png" alt="LEANN in Claude Code" width="80%">
 </p>
 
+If you see a prompt asking whether to proceed with LEANN, you can now use it in your chat!
 
 ## 🧠 How It Works
 
@@ -89,3 +139,11 @@ To remove LEANN
 ```
 uv pip uninstall leann leann-backend-hnsw leann-core
 ```
+
+To globally remove LEANN (for version update)
+```
+uv tool list | cat
+uv tool uninstall leann-core
+command -v leann || echo "leann gone"
+command -v leann_mcp || echo "leann_mcp gone"
+```

packages/leann/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "leann"
-version = "0.2.7"
+version = "0.3.4"
 description = "LEANN - The smallest vector index in the world. RAG Everything with LEANN!"
 readme = "README.md"
 requires-python = ">=3.9"

packages/wechat-exporter/__init__.py

@@ -0,0 +1 @@
+__all__ = []

packages/wechat-exporter/main.py

@@ -136,5 +136,9 @@ def export_sqlite(
     connection.commit()
 
 
-if __name__ == "__main__":
+def main():
     app()
+
+
+if __name__ == "__main__":
+    main()

pyproject.toml

@@ -10,11 +10,10 @@ requires-python = ">=3.9"
 dependencies = [
     "leann-core",
     "leann-backend-hnsw",
+    "typer>=0.12.3",
     "numpy>=1.26.0",
     "torch",
     "tqdm",
-    "flask",
-    "flask_compress",
     "datasets>=2.15.0",
     "evaluate",
     "colorama",
@@ -40,20 +39,27 @@ dependencies = [
     # Other dependencies
     "ipykernel==6.29.5",
     "msgpack>=1.1.1",
-    "mlx>=0.26.3; sys_platform == 'darwin'",
-    "mlx-lm>=0.26.0; sys_platform == 'darwin'",
+    "mlx>=0.26.3; sys_platform == 'darwin' and platform_machine == 'arm64'",
+    "mlx-lm>=0.26.0; sys_platform == 'darwin' and platform_machine == 'arm64'",
     "psutil>=5.8.0",
     "pybind11>=3.0.0",
     "pathspec>=0.12.1",
     "nbconvert>=7.16.6",
     "gitignore-parser>=0.1.12",
+    # AST-aware code chunking dependencies
+    "astchunk>=0.1.0",
+    "tree-sitter>=0.20.0",
+    "tree-sitter-python>=0.20.0",
+    "tree-sitter-java>=0.20.0",
+    "tree-sitter-c-sharp>=0.20.0",
+    "tree-sitter-typescript>=0.20.0",
 ]
 
 [project.optional-dependencies]
 dev = [
-    "pytest>=8.3.0",  # Minimum version for Python 3.13 support
-"pytest-cov>=5.0",
-"pytest-xdist>=3.5",  # For parallel test execution
+    "pytest>=7.0",
+    "pytest-cov>=4.0",
+    "pytest-xdist>=3.0",  # For parallel test execution
     "black>=23.0",
     "ruff==0.12.7",  # Fixed version to ensure consistent formatting across all environments
     "matplotlib",
@@ -62,14 +68,10 @@ dev = [
 ]
 
 test = [
-    "pytest>=8.3.0",  # Minimum version for Python 3.13 support
-    "pytest-timeout>=2.3",
-    "anyio>=4.0",  # For async test support (includes pytest plugin)
-    "psutil>=5.9.0",  # For process cleanup in tests
+    "pytest>=7.0",
+    "pytest-timeout>=2.0",
     "llama-index-core>=0.12.0",
-    "llama-index-readers-file>=0.4.0",
     "python-dotenv>=1.0.0",
-    "sentence-transformers>=2.2.0",
 ]
 
 diskann = [
@@ -86,23 +88,24 @@ documents = [
 
 [tool.setuptools]
 py-modules = []
+packages = ["wechat_exporter"]
+package-dir = { "wechat_exporter" = "packages/wechat-exporter" }
+
+[project.scripts]
+wechat-exporter = "wechat_exporter.main:main"
 
 
 [tool.uv.sources]
 leann-core = { path = "packages/leann-core", editable = true }
 leann-backend-diskann = { path = "packages/leann-backend-diskann", editable = true }
 leann-backend-hnsw = { path = "packages/leann-backend-hnsw", editable = true }
+astchunk = { path = "packages/astchunk-leann", editable = true }
 
 [tool.ruff]
 target-version = "py39"
 line-length = 100
-extend-exclude = [
-    "third_party",
-    "*.egg-info",
-    "__pycache__",
-    ".git",
-    ".venv",
-]
+extend-exclude = ["third_party"]
+
 
 [tool.ruff.lint]
 select = [
@@ -125,21 +128,12 @@ ignore = [
     "RUF012", # mutable class attributes should be annotated with typing.ClassVar
 ]
 
-[tool.ruff.lint.per-file-ignores]
-"test/**/*.py" = ["E402"]      # module level import not at top of file (common in tests)
-"examples/**/*.py" = ["E402"]  # module level import not at top of file (common in examples)
-
 [tool.ruff.format]
 quote-style = "double"
 indent-style = "space"
 skip-magic-trailing-comma = false
 line-ending = "auto"
 
-[dependency-groups]
-dev = [
-    "ruff>=0.12.4",
-]
-
 [tool.lychee]
 accept = ["200", "403", "429", "503"]
 timeout = 20
@@ -158,7 +152,6 @@ markers = [
     "openai: marks tests that require OpenAI API key",
 ]
 timeout = 300  # Reduced from 600s (10min) to 300s (5min) for CI safety
-timeout_method = "thread"  # Use thread method to avoid non-daemon thread issues
 addopts = [
     "-v",
     "--tb=short",

scripts/diagnose_hang.sh

@@ -1,103 +0,0 @@
-#!/bin/bash
-# Diagnostic script for debugging CI hangs
-
-echo "========================================="
-echo "      CI HANG DIAGNOSTIC SCRIPT"
-echo "========================================="
-echo ""
-
-echo "📅 Current time: $(date)"
-echo "🖥️  Hostname: $(hostname)"
-echo "👤 User: $(whoami)"
-echo "📂 Working directory: $(pwd)"
-echo ""
-
-echo "=== PYTHON ENVIRONMENT ==="
-python --version 2>&1 || echo "Python not found"
-pip list 2>&1 | head -20 || echo "pip not available"
-echo ""
-
-echo "=== PROCESS INFORMATION ==="
-echo "Current shell PID: $$"
-echo "Parent PID: $PPID"
-echo ""
-
-echo "All Python processes:"
-ps aux | grep -E "[p]ython" || echo "No Python processes"
-echo ""
-
-echo "All pytest processes:"
-ps aux | grep -E "[p]ytest" || echo "No pytest processes"
-echo ""
-
-echo "Embedding server processes:"
-ps aux | grep -E "[e]mbedding_server" || echo "No embedding server processes"
-echo ""
-
-echo "Zombie processes:"
-ps aux | grep "<defunct>" || echo "No zombie processes"
-echo ""
-
-echo "=== NETWORK INFORMATION ==="
-echo "Network listeners on typical embedding server ports:"
-ss -ltn 2>/dev/null | grep -E ":555[0-9]|:556[0-9]" || netstat -ltn 2>/dev/null | grep -E ":555[0-9]|:556[0-9]" || echo "No listeners on embedding ports"
-echo ""
-
-echo "All network listeners:"
-ss -ltn 2>/dev/null | head -20 || netstat -ltn 2>/dev/null | head -20 || echo "Cannot get network info"
-echo ""
-
-echo "=== FILE DESCRIPTORS ==="
-echo "Open files for current shell:"
-lsof -p $$ 2>/dev/null | head -20 || echo "lsof not available"
-echo ""
-
-if [ -d "/proc/$$" ]; then
-    echo "File descriptors for current shell (/proc/$$/fd):"
-    ls -la /proc/$$/fd 2>/dev/null | head -20 || echo "Cannot access /proc/$$/fd"
-    echo ""
-fi
-
-echo "=== SYSTEM RESOURCES ==="
-echo "Memory usage:"
-free -h 2>/dev/null || vm_stat 2>/dev/null || echo "Cannot get memory info"
-echo ""
-
-echo "Disk usage:"
-df -h . 2>/dev/null || echo "Cannot get disk info"
-echo ""
-
-echo "CPU info:"
-nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo "Cannot get CPU info"
-echo ""
-
-echo "=== PYTHON SPECIFIC CHECKS ==="
-python -c "
-import sys
-import os
-print(f'Python executable: {sys.executable}')
-print(f'Python path: {sys.path[:3]}...')
-print(f'Environment PYTHONPATH: {os.environ.get(\"PYTHONPATH\", \"Not set\")}')
-print(f'Site packages: {[p for p in sys.path if \"site-packages\" in p][:2]}')
-" 2>&1 || echo "Cannot run Python diagnostics"
-echo ""
-
-echo "=== ZMQ SPECIFIC CHECKS ==="
-python -c "
-try:
-    import zmq
-    print(f'ZMQ version: {zmq.zmq_version()}')
-    print(f'PyZMQ version: {zmq.pyzmq_version()}')
-    ctx = zmq.Context.instance()
-    print(f'ZMQ context instance: {ctx}')
-except Exception as e:
-    print(f'ZMQ check failed: {e}')
-" 2>&1 || echo "Cannot check ZMQ"
-echo ""
-
-echo "=== PYTEST CHECK ==="
-pytest --version 2>&1 || echo "pytest not found"
-echo ""
-
-echo "=== END OF DIAGNOSTICS ==="
-echo "Generated at: $(date)"

sky/leann-build.yaml

@@ -0,0 +1,76 @@
+name: leann-build
+
+resources:
+  # Choose a GPU for fast embeddings (examples: L4, A10G, A100). CPU also works but is slower.
+  accelerators: L4:1
+  # Optionally pin a cloud, otherwise SkyPilot will auto-select
+  # cloud: aws
+  disk_size: 100
+
+envs:
+  # Build parameters (override with: sky launch -c leann-gpu sky/leann-build.yaml -e key=value)
+  index_name: my-index
+  docs: ./data
+  backend: hnsw               # hnsw | diskann
+  complexity: 64
+  graph_degree: 32
+  num_threads: 8
+  # Embedding selection
+  embedding_mode: sentence-transformers   # sentence-transformers | openai | mlx | ollama
+  embedding_model: facebook/contriever
+  # Storage/latency knobs
+  recompute: true             # true => selective recomputation (recommended)
+  compact: true               # for HNSW only
+  # Optional pass-through
+  extra_args: ""
+  # Rebuild control
+  force: true
+
+# Sync local paths to the remote VM. Adjust as needed.
+file_mounts:
+  # Example: mount your local data directory used for building
+  ~/leann-data: ${docs}
+
+setup: |
+  set -e
+  # Install uv (package manager)
+  curl -LsSf https://astral.sh/uv/install.sh | sh
+  export PATH="$HOME/.local/bin:$PATH"
+
+  # Ensure modern libstdc++ for FAISS (GLIBCXX >= 3.4.30)
+  sudo apt-get update -y
+  sudo apt-get install -y libstdc++6 libgomp1
+  # Also upgrade conda's libstdc++ in base env (Skypilot images include conda)
+  if command -v conda >/dev/null 2>&1; then
+    conda install -y -n base -c conda-forge libstdcxx-ng
+  fi
+
+  # Install LEANN CLI and backends into the user environment
+  uv pip install --upgrade pip
+  uv pip install leann-core leann-backend-hnsw leann-backend-diskann
+
+run: |
+  export PATH="$HOME/.local/bin:$PATH"
+  # Derive flags from env
+  recompute_flag=""
+  if [ "${recompute}" = "false" ] || [ "${recompute}" = "0" ]; then
+    recompute_flag="--no-recompute"
+  fi
+  force_flag=""
+  if [ "${force}" = "true" ] || [ "${force}" = "1" ]; then
+    force_flag="--force"
+  fi
+
+  # Build command
+  python -m leann.cli build ${index_name} \
+    --docs ~/leann-data \
+    --backend ${backend} \
+    --complexity ${complexity} \
+    --graph-degree ${graph_degree} \
+    --num-threads ${num_threads} \
+    --embedding-mode ${embedding_mode} \
+    --embedding-model ${embedding_model} \
+    ${recompute_flag} ${force_flag} ${extra_args}
+
+  # Print where the index is stored for downstream rsync
+  echo "INDEX_OUT_DIR=~/.leann/indexes/${index_name}"

tests/conftest.py

@@ -1,301 +0,0 @@
-"""Global test configuration and cleanup fixtures."""
-
-import faulthandler
-import os
-import signal
-import time
-from collections.abc import Generator
-
-import pytest
-
-# Enable faulthandler to dump stack traces
-faulthandler.enable()
-
-
-@pytest.fixture(scope="session", autouse=True)
-def _ci_backtraces():
-    """Dump stack traces before CI timeout to diagnose hanging."""
-    if os.getenv("CI") == "true":
-        # Dump stack traces 10s before the 180s timeout
-        faulthandler.dump_traceback_later(170, repeat=True)
-    yield
-    faulthandler.cancel_dump_traceback_later()
-
-
-@pytest.fixture(scope="session", autouse=True)
-def global_test_cleanup() -> Generator:
-    """Global cleanup fixture that runs after all tests.
-
-    This ensures all ZMQ connections and child processes are properly cleaned up,
-    preventing the test runner from hanging on exit.
-    """
-    yield
-
-    # Cleanup after all tests
-    print("\n🧹 Running global test cleanup...")
-
-    # 1. Force cleanup of any LeannSearcher instances
-    try:
-        import gc
-
-        # Force garbage collection to trigger __del__ methods
-        gc.collect()
-        time.sleep(0.2)
-    except Exception:
-        pass
-
-    # 2. Set ZMQ linger but DON'T term Context.instance()
-    # Terminating the global instance can block if other code still has sockets
-    try:
-        import zmq
-
-        # Just set linger on the global instance, don't terminate it
-        ctx = zmq.Context.instance()
-        ctx.linger = 0
-        # Do NOT call ctx.term() or ctx.destroy() on the global instance!
-        # That would block waiting for all sockets to close
-    except Exception:
-        pass
-
-    # Kill any leftover child processes (including grandchildren)
-    try:
-        import psutil
-
-        current_process = psutil.Process()
-        # Get ALL descendants recursively
-        children = current_process.children(recursive=True)
-
-        if children:
-            print(f"\n⚠️  Cleaning up {len(children)} leftover child processes...")
-
-            # First try to terminate gracefully
-            for child in children:
-                try:
-                    print(f"  Terminating {child.pid} ({child.name()})")
-                    child.terminate()
-                except (psutil.NoSuchProcess, psutil.AccessDenied):
-                    pass
-
-            # Wait a bit for processes to terminate
-            gone, alive = psutil.wait_procs(children, timeout=2)
-
-            # Force kill any remaining processes
-            for child in alive:
-                try:
-                    print(f"  Force killing process {child.pid} ({child.name()})")
-                    child.kill()
-                except (psutil.NoSuchProcess, psutil.AccessDenied):
-                    pass
-
-            # Final wait to ensure cleanup
-            psutil.wait_procs(alive, timeout=1)
-    except ImportError:
-        # psutil not installed, try basic process cleanup
-        try:
-            # Send SIGTERM to all child processes
-            os.killpg(os.getpgid(os.getpid()), signal.SIGTERM)
-        except Exception:
-            pass
-    except Exception as e:
-        print(f"Warning: Error during process cleanup: {e}")
-
-    # List and clean up remaining threads
-    try:
-        import threading
-
-        threads = [t for t in threading.enumerate() if t is not threading.main_thread()]
-        if threads:
-            print(f"\n⚠️  {len(threads)} non-main threads still running:")
-            for t in threads:
-                print(f"  - {t.name} (daemon={t.daemon})")
-
-                # Force cleanup of pytest-timeout threads that block exit
-                if "pytest_timeout" in t.name and not t.daemon:
-                    print(f"  🔧 Converting pytest-timeout thread to daemon: {t.name}")
-                    try:
-                        t.daemon = True
-                        print("     ✓ Converted to daemon thread")
-                    except Exception as e:
-                        print(f"     ✗ Failed: {e}")
-
-        # Check if only daemon threads remain
-        non_daemon = [
-            t for t in threading.enumerate() if t is not threading.main_thread() and not t.daemon
-        ]
-        if non_daemon:
-            print(f"\n⚠️  {len(non_daemon)} non-daemon threads still blocking exit")
-            # Force exit in CI to prevent hanging
-            if os.environ.get("CI") == "true":
-                print("🔨 Forcing exit in CI environment...")
-                os._exit(0)
-    except Exception as e:
-        print(f"Thread cleanup error: {e}")
-
-
-@pytest.fixture
-def auto_cleanup_searcher():
-    """Fixture that automatically cleans up LeannSearcher instances."""
-    searchers = []
-
-    def register(searcher):
-        """Register a searcher for cleanup."""
-        searchers.append(searcher)
-        return searcher
-
-    yield register
-
-    # Cleanup all registered searchers
-    for searcher in searchers:
-        try:
-            searcher.cleanup()
-        except Exception:
-            pass
-
-    # Force garbage collection
-    import gc
-
-    gc.collect()
-    time.sleep(0.1)
-
-
-@pytest.fixture(scope="session", autouse=True)
-def _reap_children():
-    """Reap all child processes at session end as a safety net."""
-    yield
-
-    # Final aggressive cleanup
-    try:
-        import psutil
-
-        me = psutil.Process()
-        kids = me.children(recursive=True)
-        for p in kids:
-            try:
-                p.terminate()
-            except Exception:
-                pass
-
-        _, alive = psutil.wait_procs(kids, timeout=2)
-        for p in alive:
-            try:
-                p.kill()
-            except Exception:
-                pass
-    except Exception:
-        pass
-
-
-@pytest.fixture(autouse=True)
-def cleanup_after_each_test():
-    """Cleanup after each test to prevent resource leaks."""
-    yield
-
-    # Force garbage collection to trigger any __del__ methods
-    import gc
-
-    gc.collect()
-
-    # Give a moment for async cleanup
-    time.sleep(0.1)
-
-
-def pytest_configure(config):
-    """Configure pytest with better timeout handling."""
-    # Set default timeout method to thread if not specified
-    if not config.getoption("--timeout-method", None):
-        config.option.timeout_method = "thread"
-
-    # Add more logging
-    print(f"🔧 Pytest configured at {time.strftime('%Y-%m-%d %H:%M:%S')}")
-    print(f"   Python version: {os.sys.version}")
-    print(f"   Platform: {os.sys.platform}")
-
-
-def pytest_sessionstart(session):
-    """Called after the Session object has been created."""
-    print(f"🏁 Pytest session starting at {time.strftime('%Y-%m-%d %H:%M:%S')}")
-    print(f"   Session ID: {id(session)}")
-
-    # Show initial process state
-    try:
-        import psutil
-
-        current = psutil.Process()
-        print(f"   Current PID: {current.pid}")
-        print(f"   Parent PID: {current.ppid()}")
-        children = current.children(recursive=True)
-        if children:
-            print(f"   ⚠️ Already have {len(children)} child processes at start!")
-    except Exception:
-        pass
-
-
-def pytest_sessionfinish(session, exitstatus):
-    """Called after whole test run finished."""
-    print(f"🏁 Pytest session finishing at {time.strftime('%Y-%m-%d %H:%M:%S')}")
-    print(f"   Exit status: {exitstatus}")
-
-    # Aggressive cleanup before pytest exits
-    print("🧹 Starting aggressive cleanup...")
-
-    # First, clean up child processes
-    try:
-        import psutil
-
-        current = psutil.Process()
-        children = current.children(recursive=True)
-
-        if children:
-            print(f"   Found {len(children)} child processes to clean up:")
-            for child in children:
-                try:
-                    print(f"     - PID {child.pid}: {child.name()} (status: {child.status()})")
-                    child.terminate()
-                except Exception as e:
-                    print(f"     - Failed to terminate {child.pid}: {e}")
-
-            # Wait briefly then kill
-            time.sleep(0.5)
-            _, alive = psutil.wait_procs(children, timeout=1)
-
-            for child in alive:
-                try:
-                    print(f"     - Force killing {child.pid}")
-                    child.kill()
-                except Exception:
-                    pass
-        else:
-            print("   No child processes found")
-
-    except Exception as e:
-        print(f"   Process cleanup error: {e}")
-
-    # Second, clean up problematic threads
-    try:
-        import threading
-
-        threads = [t for t in threading.enumerate() if t is not threading.main_thread()]
-        if threads:
-            print(f"   Found {len(threads)} non-main threads:")
-            for t in threads:
-                print(f"     - {t.name} (daemon={t.daemon})")
-                # Convert pytest-timeout threads to daemon so they don't block exit
-                if "pytest_timeout" in t.name and not t.daemon:
-                    try:
-                        t.daemon = True
-                        print("       ✓ Converted to daemon")
-                    except Exception:
-                        pass
-
-        # Force exit if non-daemon threads remain in CI
-        non_daemon = [
-            t for t in threading.enumerate() if t is not threading.main_thread() and not t.daemon
-        ]
-        if non_daemon and os.environ.get("CI") == "true":
-            print(f"   ⚠️ {len(non_daemon)} non-daemon threads remain, forcing exit...")
-            os._exit(exitstatus or 0)
-
-    except Exception as e:
-        print(f"   Thread cleanup error: {e}")
-
-    print(f"✅ Pytest exiting at {time.strftime('%Y-%m-%d %H:%M:%S')}")

tests/test_astchunk_integration.py

@@ -0,0 +1,397 @@
+"""
+Test suite for astchunk integration with LEANN.
+Tests AST-aware chunking functionality, language detection, and fallback mechanisms.
+"""
+
+import os
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+# Add apps directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent.parent / "apps"))
+
+from typing import Optional
+
+from chunking import (
+    create_ast_chunks,
+    create_text_chunks,
+    create_traditional_chunks,
+    detect_code_files,
+    get_language_from_extension,
+)
+
+
+class MockDocument:
+    """Mock LlamaIndex Document for testing."""
+
+    def __init__(self, content: str, file_path: str = "", metadata: Optional[dict] = None):
+        self.content = content
+        self.metadata = metadata or {}
+        if file_path:
+            self.metadata["file_path"] = file_path
+
+    def get_content(self) -> str:
+        return self.content
+
+
+class TestCodeFileDetection:
+    """Test code file detection and language mapping."""
+
+    def test_detect_code_files_python(self):
+        """Test detection of Python files."""
+        docs = [
+            MockDocument("print('hello')", "/path/to/file.py"),
+            MockDocument("This is text", "/path/to/file.txt"),
+        ]
+
+        code_docs, text_docs = detect_code_files(docs)
+
+        assert len(code_docs) == 1
+        assert len(text_docs) == 1
+        assert code_docs[0].metadata["language"] == "python"
+        assert code_docs[0].metadata["is_code"] is True
+        assert text_docs[0].metadata["is_code"] is False
+
+    def test_detect_code_files_multiple_languages(self):
+        """Test detection of multiple programming languages."""
+        docs = [
+            MockDocument("def func():", "/path/to/script.py"),
+            MockDocument("public class Test {}", "/path/to/Test.java"),
+            MockDocument("interface ITest {}", "/path/to/test.ts"),
+            MockDocument("using System;", "/path/to/Program.cs"),
+            MockDocument("Regular text content", "/path/to/document.txt"),
+        ]
+
+        code_docs, text_docs = detect_code_files(docs)
+
+        assert len(code_docs) == 4
+        assert len(text_docs) == 1
+
+        languages = [doc.metadata["language"] for doc in code_docs]
+        assert "python" in languages
+        assert "java" in languages
+        assert "typescript" in languages
+        assert "csharp" in languages
+
+    def test_detect_code_files_no_file_path(self):
+        """Test handling of documents without file paths."""
+        docs = [
+            MockDocument("some content"),
+            MockDocument("other content", metadata={"some_key": "value"}),
+        ]
+
+        code_docs, text_docs = detect_code_files(docs)
+
+        assert len(code_docs) == 0
+        assert len(text_docs) == 2
+        for doc in text_docs:
+            assert doc.metadata["is_code"] is False
+
+    def test_get_language_from_extension(self):
+        """Test language detection from file extensions."""
+        assert get_language_from_extension("test.py") == "python"
+        assert get_language_from_extension("Test.java") == "java"
+        assert get_language_from_extension("component.tsx") == "typescript"
+        assert get_language_from_extension("Program.cs") == "csharp"
+        assert get_language_from_extension("document.txt") is None
+        assert get_language_from_extension("") is None
+
+
+class TestChunkingFunctions:
+    """Test various chunking functionality."""
+
+    def test_create_traditional_chunks(self):
+        """Test traditional text chunking."""
+        docs = [
+            MockDocument(
+                "This is a test document. It has multiple sentences. We want to test chunking."
+            )
+        ]
+
+        chunks = create_traditional_chunks(docs, chunk_size=50, chunk_overlap=10)
+
+        assert len(chunks) > 0
+        assert all(isinstance(chunk, str) for chunk in chunks)
+        assert all(len(chunk.strip()) > 0 for chunk in chunks)
+
+    def test_create_traditional_chunks_empty_docs(self):
+        """Test traditional chunking with empty documents."""
+        chunks = create_traditional_chunks([], chunk_size=50, chunk_overlap=10)
+        assert chunks == []
+
+    @pytest.mark.skipif(
+        os.environ.get("CI") == "true",
+        reason="Skip astchunk tests in CI - dependency may not be available",
+    )
+    def test_create_ast_chunks_with_astchunk_available(self):
+        """Test AST chunking when astchunk is available."""
+        python_code = '''
+def hello_world():
+    """Print hello world message."""
+    print("Hello, World!")
+
+def add_numbers(a, b):
+    """Add two numbers and return the result."""
+    return a + b
+
+class Calculator:
+    """A simple calculator class."""
+
+    def __init__(self):
+        self.history = []
+
+    def add(self, a, b):
+        result = a + b
+        self.history.append(f"{a} + {b} = {result}")
+        return result
+'''
+
+        docs = [MockDocument(python_code, "/test/calculator.py", {"language": "python"})]
+
+        try:
+            chunks = create_ast_chunks(docs, max_chunk_size=200, chunk_overlap=50)
+
+            # Should have multiple chunks due to different functions/classes
+            assert len(chunks) > 0
+            assert all(isinstance(chunk, str) for chunk in chunks)
+            assert all(len(chunk.strip()) > 0 for chunk in chunks)
+
+            # Check that code structure is somewhat preserved
+            combined_content = " ".join(chunks)
+            assert "def hello_world" in combined_content
+            assert "class Calculator" in combined_content
+
+        except ImportError:
+            # astchunk not available, should fall back to traditional chunking
+            chunks = create_ast_chunks(docs, max_chunk_size=200, chunk_overlap=50)
+            assert len(chunks) > 0  # Should still get chunks from fallback
+
+    def test_create_ast_chunks_fallback_to_traditional(self):
+        """Test AST chunking falls back to traditional when astchunk is not available."""
+        docs = [MockDocument("def test(): pass", "/test/script.py", {"language": "python"})]
+
+        # Mock astchunk import to fail
+        with patch("chunking.create_ast_chunks"):
+            # First call (actual test) should import astchunk and potentially fail
+            # Let's call the actual function to test the import error handling
+            chunks = create_ast_chunks(docs)
+
+            # Should return some chunks (either from astchunk or fallback)
+            assert isinstance(chunks, list)
+
+    def test_create_text_chunks_traditional_mode(self):
+        """Test text chunking in traditional mode."""
+        docs = [
+            MockDocument("def test(): pass", "/test/script.py"),
+            MockDocument("This is regular text.", "/test/doc.txt"),
+        ]
+
+        chunks = create_text_chunks(docs, use_ast_chunking=False, chunk_size=50, chunk_overlap=10)
+
+        assert len(chunks) > 0
+        assert all(isinstance(chunk, str) for chunk in chunks)
+
+    def test_create_text_chunks_ast_mode(self):
+        """Test text chunking in AST mode."""
+        docs = [
+            MockDocument("def test(): pass", "/test/script.py"),
+            MockDocument("This is regular text.", "/test/doc.txt"),
+        ]
+
+        chunks = create_text_chunks(
+            docs,
+            use_ast_chunking=True,
+            ast_chunk_size=100,
+            ast_chunk_overlap=20,
+            chunk_size=50,
+            chunk_overlap=10,
+        )
+
+        assert len(chunks) > 0
+        assert all(isinstance(chunk, str) for chunk in chunks)
+
+    def test_create_text_chunks_custom_extensions(self):
+        """Test text chunking with custom code file extensions."""
+        docs = [
+            MockDocument("function test() {}", "/test/script.js"),  # Not in default extensions
+            MockDocument("Regular text", "/test/doc.txt"),
+        ]
+
+        # First without custom extensions - should treat .js as text
+        chunks_without = create_text_chunks(docs, use_ast_chunking=True, code_file_extensions=None)
+
+        # Then with custom extensions - should treat .js as code
+        chunks_with = create_text_chunks(
+            docs, use_ast_chunking=True, code_file_extensions=[".js", ".jsx"]
+        )
+
+        # Both should return chunks
+        assert len(chunks_without) > 0
+        assert len(chunks_with) > 0
+
+
+class TestIntegrationWithDocumentRAG:
+    """Integration tests with the document RAG system."""
+
+    @pytest.fixture
+    def temp_code_dir(self):
+        """Create a temporary directory with sample code files."""
+        with tempfile.TemporaryDirectory() as temp_dir:
+            temp_path = Path(temp_dir)
+
+            # Create sample Python file
+            python_file = temp_path / "example.py"
+            python_file.write_text('''
+def fibonacci(n):
+    """Calculate fibonacci number."""
+    if n <= 1:
+        return n
+    return fibonacci(n-1) + fibonacci(n-2)
+
+class MathUtils:
+    @staticmethod
+    def factorial(n):
+        if n <= 1:
+            return 1
+        return n * MathUtils.factorial(n-1)
+''')
+
+            # Create sample text file
+            text_file = temp_path / "readme.txt"
+            text_file.write_text("This is a sample text file for testing purposes.")
+
+            yield temp_path
+
+    @pytest.mark.skipif(
+        os.environ.get("CI") == "true",
+        reason="Skip integration tests in CI to avoid dependency issues",
+    )
+    def test_document_rag_with_ast_chunking(self, temp_code_dir):
+        """Test document RAG with AST chunking enabled."""
+        with tempfile.TemporaryDirectory() as index_dir:
+            cmd = [
+                sys.executable,
+                "apps/document_rag.py",
+                "--llm",
+                "simulated",
+                "--embedding-model",
+                "facebook/contriever",
+                "--embedding-mode",
+                "sentence-transformers",
+                "--index-dir",
+                index_dir,
+                "--data-dir",
+                str(temp_code_dir),
+                "--enable-code-chunking",
+                "--query",
+                "How does the fibonacci function work?",
+            ]
+
+            env = os.environ.copy()
+            env["HF_HUB_DISABLE_SYMLINKS"] = "1"
+            env["TOKENIZERS_PARALLELISM"] = "false"
+
+            try:
+                result = subprocess.run(
+                    cmd,
+                    capture_output=True,
+                    text=True,
+                    timeout=300,  # 5 minutes
+                    env=env,
+                )
+
+                # Should succeed even if astchunk is not available (fallback)
+                assert result.returncode == 0, f"Command failed: {result.stderr}"
+
+                output = result.stdout + result.stderr
+                assert "Index saved to" in output or "Using existing index" in output
+
+            except subprocess.TimeoutExpired:
+                pytest.skip("Test timed out - likely due to model download in CI")
+
+    @pytest.mark.skipif(
+        os.environ.get("CI") == "true",
+        reason="Skip integration tests in CI to avoid dependency issues",
+    )
+    def test_code_rag_application(self, temp_code_dir):
+        """Test the specialized code RAG application."""
+        with tempfile.TemporaryDirectory() as index_dir:
+            cmd = [
+                sys.executable,
+                "apps/code_rag.py",
+                "--llm",
+                "simulated",
+                "--embedding-model",
+                "facebook/contriever",
+                "--index-dir",
+                index_dir,
+                "--repo-dir",
+                str(temp_code_dir),
+                "--query",
+                "What classes are defined in this code?",
+            ]
+
+            env = os.environ.copy()
+            env["HF_HUB_DISABLE_SYMLINKS"] = "1"
+            env["TOKENIZERS_PARALLELISM"] = "false"
+
+            try:
+                result = subprocess.run(cmd, capture_output=True, text=True, timeout=300, env=env)
+
+                # Should succeed
+                assert result.returncode == 0, f"Command failed: {result.stderr}"
+
+                output = result.stdout + result.stderr
+                assert "Using AST-aware chunking" in output or "traditional chunking" in output
+
+            except subprocess.TimeoutExpired:
+                pytest.skip("Test timed out - likely due to model download in CI")
+
+
+class TestErrorHandling:
+    """Test error handling and edge cases."""
+
+    def test_text_chunking_empty_documents(self):
+        """Test text chunking with empty document list."""
+        chunks = create_text_chunks([])
+        assert chunks == []
+
+    def test_text_chunking_invalid_parameters(self):
+        """Test text chunking with invalid parameters."""
+        docs = [MockDocument("test content")]
+
+        # Should handle negative chunk sizes gracefully
+        chunks = create_text_chunks(
+            docs, chunk_size=0, chunk_overlap=0, ast_chunk_size=0, ast_chunk_overlap=0
+        )
+
+        # Should still return some result
+        assert isinstance(chunks, list)
+
+    def test_create_ast_chunks_no_language(self):
+        """Test AST chunking with documents missing language metadata."""
+        docs = [MockDocument("def test(): pass", "/test/script.py")]  # No language set
+
+        chunks = create_ast_chunks(docs)
+
+        # Should fall back to traditional chunking
+        assert isinstance(chunks, list)
+        assert len(chunks) >= 0  # May be empty if fallback also fails
+
+    def test_create_ast_chunks_empty_content(self):
+        """Test AST chunking with empty content."""
+        docs = [MockDocument("", "/test/script.py", {"language": "python"})]
+
+        chunks = create_ast_chunks(docs)
+
+        # Should handle empty content gracefully
+        assert isinstance(chunks, list)
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

tests/test_basic.py

@@ -7,7 +7,6 @@ import tempfile
 from pathlib import Path
 
 import pytest
-from test_timeout import ci_timeout
 
 
 def test_imports():
@@ -20,7 +19,6 @@ def test_imports():
     os.environ.get("CI") == "true", reason="Skip model tests in CI to avoid MPS memory issues"
 )
 @pytest.mark.parametrize("backend_name", ["hnsw", "diskann"])
-@ci_timeout(120)  # 2 minute timeout for backend tests
 def test_backend_basic(backend_name):
     """Test basic functionality for each backend."""
     from leann.api import LeannBuilder, LeannSearcher, SearchResult
@@ -66,11 +64,13 @@ def test_backend_basic(backend_name):
         assert isinstance(results[0], SearchResult)
         assert "topic 2" in results[0].text or "document" in results[0].text
 
+        # Ensure cleanup to avoid hanging background servers
+        searcher.cleanup()
+
 
 @pytest.mark.skipif(
     os.environ.get("CI") == "true", reason="Skip model tests in CI to avoid MPS memory issues"
 )
-@ci_timeout(180)  # 3 minute timeout for large index test
 def test_large_index():
     """Test with larger dataset."""
     from leann.api import LeannBuilder, LeannSearcher
@@ -93,3 +93,5 @@ def test_large_index():
         searcher = LeannSearcher(index_path)
         results = searcher.search(["word10 word20"], top_k=10)
         assert len(results[0]) == 10
+        # Cleanup
+        searcher.cleanup()

tests/test_document_rag.py

@@ -9,7 +9,6 @@ import tempfile
 from pathlib import Path
 
 import pytest
-from test_timeout import ci_timeout
 
 
 @pytest.fixture
@@ -58,11 +57,55 @@ def test_document_rag_simulated(test_data_dir):
         assert "This is a simulated answer" in output
 
 
+@pytest.mark.skipif(
+    os.environ.get("CI") == "true",
+    reason="Skip AST chunking tests in CI to avoid dependency issues",
+)
+def test_document_rag_with_ast_chunking(test_data_dir):
+    """Test document_rag with AST-aware chunking enabled."""
+    with tempfile.TemporaryDirectory() as temp_dir:
+        # Use a subdirectory that doesn't exist yet to force index creation
+        index_dir = Path(temp_dir) / "test_ast_index"
+        cmd = [
+            sys.executable,
+            "apps/document_rag.py",
+            "--llm",
+            "simulated",
+            "--embedding-model",
+            "facebook/contriever",
+            "--embedding-mode",
+            "sentence-transformers",
+            "--index-dir",
+            str(index_dir),
+            "--data-dir",
+            str(test_data_dir),
+            "--enable-code-chunking",  # Enable AST chunking
+            "--query",
+            "What is Pride and Prejudice about?",
+        ]
+
+        env = os.environ.copy()
+        env["HF_HUB_DISABLE_SYMLINKS"] = "1"
+        env["TOKENIZERS_PARALLELISM"] = "false"
+
+        result = subprocess.run(cmd, capture_output=True, text=True, timeout=600, env=env)
+
+        # Check return code
+        assert result.returncode == 0, f"Command failed: {result.stderr}"
+
+        # Verify output
+        output = result.stdout + result.stderr
+        assert "Index saved to" in output or "Using existing index" in output
+        assert "This is a simulated answer" in output
+
+        # Should mention AST chunking if code files are present
+        # (might not be relevant for the test data, but command should succeed)
+
+
 @pytest.mark.skipif(not os.environ.get("OPENAI_API_KEY"), reason="OpenAI API key not available")
 @pytest.mark.skipif(
-    os.environ.get("CI") == "true", reason="Skip OpenAI embedding tests in CI to avoid hanging"
+    os.environ.get("CI") == "true", reason="Skip OpenAI tests in CI to avoid API costs"
 )
-@ci_timeout(60)  # 60 second timeout to avoid hanging on OpenAI API calls
 def test_document_rag_openai(test_data_dir):
     """Test document_rag with OpenAI embeddings."""
     with tempfile.TemporaryDirectory() as temp_dir:

tests/test_metadata_filtering.py

@@ -0,0 +1,365 @@
+"""
+Comprehensive tests for metadata filtering functionality.
+
+This module tests the MetadataFilterEngine class and its integration
+with the LEANN search system.
+"""
+
+import os
+
+# Import the modules we're testing
+import sys
+from unittest.mock import Mock, patch
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../packages/leann-core/src"))
+
+from leann.api import PassageManager, SearchResult
+from leann.metadata_filter import MetadataFilterEngine
+
+
+class TestMetadataFilterEngine:
+    """Test suite for the MetadataFilterEngine class."""
+
+    def setup_method(self):
+        """Setup test fixtures."""
+        self.engine = MetadataFilterEngine()
+
+        # Sample search results for testing
+        self.sample_results = [
+            {
+                "id": "doc1",
+                "score": 0.95,
+                "text": "This is chapter 1 content",
+                "metadata": {
+                    "chapter": 1,
+                    "character": "Alice",
+                    "tags": ["adventure", "fantasy"],
+                    "word_count": 150,
+                    "is_published": True,
+                    "genre": "fiction",
+                },
+            },
+            {
+                "id": "doc2",
+                "score": 0.87,
+                "text": "This is chapter 3 content",
+                "metadata": {
+                    "chapter": 3,
+                    "character": "Bob",
+                    "tags": ["mystery", "thriller"],
+                    "word_count": 250,
+                    "is_published": True,
+                    "genre": "fiction",
+                },
+            },
+            {
+                "id": "doc3",
+                "score": 0.82,
+                "text": "This is chapter 5 content",
+                "metadata": {
+                    "chapter": 5,
+                    "character": "Alice",
+                    "tags": ["romance", "drama"],
+                    "word_count": 300,
+                    "is_published": False,
+                    "genre": "non-fiction",
+                },
+            },
+            {
+                "id": "doc4",
+                "score": 0.78,
+                "text": "This is chapter 10 content",
+                "metadata": {
+                    "chapter": 10,
+                    "character": "Charlie",
+                    "tags": ["action", "adventure"],
+                    "word_count": 400,
+                    "is_published": True,
+                    "genre": "fiction",
+                },
+            },
+        ]
+
+    def test_engine_initialization(self):
+        """Test that the filter engine initializes correctly."""
+        assert self.engine is not None
+        assert len(self.engine.operators) > 0
+        assert "==" in self.engine.operators
+        assert "contains" in self.engine.operators
+        assert "in" in self.engine.operators
+
+    def test_direct_instantiation(self):
+        """Test direct instantiation of the engine."""
+        engine = MetadataFilterEngine()
+        assert isinstance(engine, MetadataFilterEngine)
+
+    def test_no_filters_returns_all_results(self):
+        """Test that passing None or empty filters returns all results."""
+        # Test with None
+        result = self.engine.apply_filters(self.sample_results, None)
+        assert len(result) == len(self.sample_results)
+
+        # Test with empty dict
+        result = self.engine.apply_filters(self.sample_results, {})
+        assert len(result) == len(self.sample_results)
+
+    # Test comparison operators
+    def test_equals_filter(self):
+        """Test equals (==) filter."""
+        filters = {"chapter": {"==": 1}}
+        result = self.engine.apply_filters(self.sample_results, filters)
+        assert len(result) == 1
+        assert result[0]["id"] == "doc1"
+
+    def test_not_equals_filter(self):
+        """Test not equals (!=) filter."""
+        filters = {"genre": {"!=": "fiction"}}
+        result = self.engine.apply_filters(self.sample_results, filters)
+        assert len(result) == 1
+        assert result[0]["metadata"]["genre"] == "non-fiction"
+
+    def test_less_than_filter(self):
+        """Test less than (<) filter."""
+        filters = {"chapter": {"<": 5}}
+        result = self.engine.apply_filters(self.sample_results, filters)
+        assert len(result) == 2
+        chapters = [r["metadata"]["chapter"] for r in result]
+        assert all(ch < 5 for ch in chapters)
+
+    def test_less_than_or_equal_filter(self):
+        """Test less than or equal (<=) filter."""
+        filters = {"chapter": {"<=": 5}}
+        result = self.engine.apply_filters(self.sample_results, filters)
+        assert len(result) == 3
+        chapters = [r["metadata"]["chapter"] for r in result]
+        assert all(ch <= 5 for ch in chapters)
+
+    def test_greater_than_filter(self):
+        """Test greater than (>) filter."""
+        filters = {"word_count": {">": 200}}
+        result = self.engine.apply_filters(self.sample_results, filters)
+        assert len(result) == 3  # Documents with word_count 250, 300, 400
+        word_counts = [r["metadata"]["word_count"] for r in result]
+        assert all(wc > 200 for wc in word_counts)
+
+    def test_greater_than_or_equal_filter(self):
+        """Test greater than or equal (>=) filter."""
+        filters = {"word_count": {">=": 250}}
+        result = self.engine.apply_filters(self.sample_results, filters)
+        assert len(result) == 3
+        word_counts = [r["metadata"]["word_count"] for r in result]
+        assert all(wc >= 250 for wc in word_counts)
+
+    # Test membership operators
+    def test_in_filter(self):
+        """Test in filter."""
+        filters = {"character": {"in": ["Alice", "Bob"]}}
+        result = self.engine.apply_filters(self.sample_results, filters)
+        assert len(result) == 3
+        characters = [r["metadata"]["character"] for r in result]
+        assert all(ch in ["Alice", "Bob"] for ch in characters)
+
+    def test_not_in_filter(self):
+        """Test not_in filter."""
+        filters = {"character": {"not_in": ["Alice", "Bob"]}}
+        result = self.engine.apply_filters(self.sample_results, filters)
+        assert len(result) == 1
+        assert result[0]["metadata"]["character"] == "Charlie"
+
+    # Test string operators
+    def test_contains_filter(self):
+        """Test contains filter."""
+        filters = {"genre": {"contains": "fiction"}}
+        result = self.engine.apply_filters(self.sample_results, filters)
+        assert len(result) == 4  # Both "fiction" and "non-fiction"
+
+    def test_starts_with_filter(self):
+        """Test starts_with filter."""
+        filters = {"genre": {"starts_with": "non"}}
+        result = self.engine.apply_filters(self.sample_results, filters)
+        assert len(result) == 1
+        assert result[0]["metadata"]["genre"] == "non-fiction"
+
+    def test_ends_with_filter(self):
+        """Test ends_with filter."""
+        filters = {"text": {"ends_with": "content"}}
+        result = self.engine.apply_filters(self.sample_results, filters)
+        assert len(result) == 4  # All sample texts end with "content"
+
+    # Test boolean operators
+    def test_is_true_filter(self):
+        """Test is_true filter."""
+        filters = {"is_published": {"is_true": True}}
+        result = self.engine.apply_filters(self.sample_results, filters)
+        assert len(result) == 3
+        assert all(r["metadata"]["is_published"] for r in result)
+
+    def test_is_false_filter(self):
+        """Test is_false filter."""
+        filters = {"is_published": {"is_false": False}}
+        result = self.engine.apply_filters(self.sample_results, filters)
+        assert len(result) == 1
+        assert not result[0]["metadata"]["is_published"]
+
+    # Test compound filters (AND logic)
+    def test_compound_filters(self):
+        """Test multiple filters applied together (AND logic)."""
+        filters = {"genre": {"==": "fiction"}, "chapter": {"<=": 5}}
+        result = self.engine.apply_filters(self.sample_results, filters)
+        assert len(result) == 2
+        for r in result:
+            assert r["metadata"]["genre"] == "fiction"
+            assert r["metadata"]["chapter"] <= 5
+
+    def test_multiple_operators_same_field(self):
+        """Test multiple operators on the same field."""
+        filters = {"word_count": {">=": 200, "<=": 350}}
+        result = self.engine.apply_filters(self.sample_results, filters)
+        assert len(result) == 2
+        for r in result:
+            wc = r["metadata"]["word_count"]
+            assert 200 <= wc <= 350
+
+    # Test edge cases
+    def test_missing_field_fails_filter(self):
+        """Test that missing metadata fields fail filters."""
+        filters = {"nonexistent_field": {"==": "value"}}
+        result = self.engine.apply_filters(self.sample_results, filters)
+        assert len(result) == 0
+
+    def test_invalid_operator(self):
+        """Test that invalid operators are handled gracefully."""
+        filters = {"chapter": {"invalid_op": 1}}
+        result = self.engine.apply_filters(self.sample_results, filters)
+        assert len(result) == 0  # Should filter out all results
+
+    def test_type_coercion_numeric(self):
+        """Test numeric type coercion in comparisons."""
+        # Add a result with string chapter number
+        test_results = [
+            *self.sample_results,
+            {
+                "id": "doc5",
+                "score": 0.75,
+                "text": "String chapter test",
+                "metadata": {"chapter": "2", "genre": "test"},
+            },
+        ]
+
+        filters = {"chapter": {"<": 3}}
+        result = self.engine.apply_filters(test_results, filters)
+        # Should include doc1 (chapter=1) and doc5 (chapter="2")
+        assert len(result) == 2
+        ids = [r["id"] for r in result]
+        assert "doc1" in ids
+        assert "doc5" in ids
+
+    def test_list_membership_with_nested_tags(self):
+        """Test membership operations with list metadata."""
+        # Note: This tests the metadata structure, not list field filtering
+        # For list field filtering, we'd need to modify the test data
+        filters = {"character": {"in": ["Alice"]}}
+        result = self.engine.apply_filters(self.sample_results, filters)
+        assert len(result) == 2
+        assert all(r["metadata"]["character"] == "Alice" for r in result)
+
+    def test_empty_results_list(self):
+        """Test filtering on empty results list."""
+        filters = {"chapter": {"==": 1}}
+        result = self.engine.apply_filters([], filters)
+        assert len(result) == 0
+
+
+class TestPassageManagerFiltering:
+    """Test suite for PassageManager filtering integration."""
+
+    def setup_method(self):
+        """Setup test fixtures."""
+        # Mock the passage manager without actual file I/O
+        self.passage_manager = Mock(spec=PassageManager)
+        self.passage_manager.filter_engine = MetadataFilterEngine()
+
+        # Sample SearchResult objects
+        self.search_results = [
+            SearchResult(
+                id="doc1",
+                score=0.95,
+                text="Chapter 1 content",
+                metadata={"chapter": 1, "character": "Alice"},
+            ),
+            SearchResult(
+                id="doc2",
+                score=0.87,
+                text="Chapter 5 content",
+                metadata={"chapter": 5, "character": "Bob"},
+            ),
+            SearchResult(
+                id="doc3",
+                score=0.82,
+                text="Chapter 10 content",
+                metadata={"chapter": 10, "character": "Alice"},
+            ),
+        ]
+
+    def test_search_result_filtering(self):
+        """Test filtering SearchResult objects."""
+        # Create a real PassageManager instance just for the filtering method
+        # We'll mock the file operations
+        with patch("builtins.open"), patch("json.loads"), patch("pickle.load"):
+            pm = PassageManager([{"type": "jsonl", "path": "test.jsonl"}])
+
+            filters = {"chapter": {"<=": 5}}
+            result = pm.filter_search_results(self.search_results, filters)
+
+            assert len(result) == 2
+            chapters = [r.metadata["chapter"] for r in result]
+            assert all(ch <= 5 for ch in chapters)
+
+    def test_filter_search_results_no_filters(self):
+        """Test that None filters return all results."""
+        with patch("builtins.open"), patch("json.loads"), patch("pickle.load"):
+            pm = PassageManager([{"type": "jsonl", "path": "test.jsonl"}])
+
+            result = pm.filter_search_results(self.search_results, None)
+            assert len(result) == len(self.search_results)
+
+    def test_filter_maintains_search_result_type(self):
+        """Test that filtering returns SearchResult objects."""
+        with patch("builtins.open"), patch("json.loads"), patch("pickle.load"):
+            pm = PassageManager([{"type": "jsonl", "path": "test.jsonl"}])
+
+            filters = {"character": {"==": "Alice"}}
+            result = pm.filter_search_results(self.search_results, filters)
+
+            assert len(result) == 2
+            for r in result:
+                assert isinstance(r, SearchResult)
+                assert r.metadata["character"] == "Alice"
+
+
+# Integration tests would go here, but they require actual LEANN backend setup
+# These would test the full pipeline from LeannSearcher.search() with metadata_filters
+
+if __name__ == "__main__":
+    # Run basic smoke tests
+    engine = MetadataFilterEngine()
+
+    sample_data = [
+        {
+            "id": "test1",
+            "score": 0.9,
+            "text": "Test content",
+            "metadata": {"chapter": 1, "published": True},
+        }
+    ]
+
+    # Test basic filtering
+    result = engine.apply_filters(sample_data, {"chapter": {"==": 1}})
+    assert len(result) == 1
+    print("✅ Basic filtering test passed")
+
+    result = engine.apply_filters(sample_data, {"chapter": {"==": 2}})
+    assert len(result) == 0
+    print("✅ No match filtering test passed")
+
+    print("🎉 All smoke tests passed!")

tests/test_readme_examples.py

@@ -8,16 +8,17 @@ import tempfile
 from pathlib import Path
 
 import pytest
-from test_timeout import ci_timeout
 
 
 @pytest.mark.parametrize("backend_name", ["hnsw", "diskann"])
-@ci_timeout(90)  # 90 second timeout for this comprehensive test
 def test_readme_basic_example(backend_name):
     """Test the basic example from README.md with both backends."""
     # Skip on macOS CI due to MPS environment issues with all-MiniLM-L6-v2
     if os.environ.get("CI") == "true" and platform.system() == "Darwin":
         pytest.skip("Skipping on macOS CI due to MPS environment issues with all-MiniLM-L6-v2")
+    # Skip DiskANN on CI (Linux runners) due to C++ extension memory/hardware constraints
+    if os.environ.get("CI") == "true" and backend_name == "diskann":
+        pytest.skip("Skip DiskANN tests in CI due to resource constraints and instability")
 
     # This is the exact code from README (with smaller model for CI)
     from leann import LeannBuilder, LeannChat, LeannSearcher
@@ -61,6 +62,9 @@ def test_readme_basic_example(backend_name):
         # The second text about banana-crocodile should be more relevant
         assert "banana" in results[0].text or "crocodile" in results[0].text
 
+        # Ensure we cleanup background embedding server
+        searcher.cleanup()
+
         # Chat with your data (using simulated LLM to avoid external dependencies)
         chat = LeannChat(INDEX_PATH, llm_config={"type": "simulated"})
         response = chat.ask("How much storage does LEANN save?", top_k=1)
@@ -68,6 +72,8 @@ def test_readme_basic_example(backend_name):
         # Verify chat works
         assert isinstance(response, str)
         assert len(response) > 0
+        # Cleanup chat resources
+        chat.cleanup()
 
 
 def test_readme_imports():
@@ -81,7 +87,6 @@ def test_readme_imports():
     assert callable(LeannChat)
 
 
-@ci_timeout(60)  # 60 second timeout
 def test_backend_options():
     """Test different backend options mentioned in documentation."""
     # Skip on macOS CI due to MPS environment issues with all-MiniLM-L6-v2
@@ -118,7 +123,6 @@ def test_backend_options():
 
 
 @pytest.mark.parametrize("backend_name", ["hnsw", "diskann"])
-@ci_timeout(75)  # 75 second timeout for LLM tests
 def test_llm_config_simulated(backend_name):
     """Test simulated LLM configuration option with both backends."""
     # Skip on macOS CI due to MPS environment issues with all-MiniLM-L6-v2

tests/test_timeout.py

@@ -1,129 +0,0 @@
-"""
-Test timeout utilities for CI environments.
-"""
-
-import functools
-import os
-import signal
-import sys
-from typing import Any, Callable
-
-
-def timeout_test(seconds: int = 30):
-    """
-    Decorator to add timeout to test functions, especially useful in CI environments.
-
-    Args:
-        seconds: Timeout in seconds (default: 30)
-    """
-
-    def decorator(func: Callable) -> Callable:
-        @functools.wraps(func)
-        def wrapper(*args: Any, **kwargs: Any) -> Any:
-            # Only apply timeout in CI environment
-            if os.environ.get("CI") != "true":
-                return func(*args, **kwargs)
-
-            # Set up timeout handler
-            def timeout_handler(signum, frame):
-                print(f"\n❌ Test {func.__name__} timed out after {seconds} seconds in CI!")
-                print("This usually indicates a hanging process or infinite loop.")
-                # Try to cleanup any hanging processes
-                try:
-                    import subprocess
-
-                    subprocess.run(
-                        ["pkill", "-f", "embedding_server"], capture_output=True, timeout=2
-                    )
-                    subprocess.run(
-                        ["pkill", "-f", "hnsw_embedding"], capture_output=True, timeout=2
-                    )
-                except Exception:
-                    pass
-                # Exit with timeout code
-                sys.exit(124)  # Standard timeout exit code
-
-            # Set signal handler and alarm
-            old_handler = signal.signal(signal.SIGALRM, timeout_handler)
-            signal.alarm(seconds)
-
-            try:
-                result = func(*args, **kwargs)
-                signal.alarm(0)  # Cancel alarm
-                return result
-            except Exception:
-                signal.alarm(0)  # Cancel alarm on exception
-                raise
-            finally:
-                # Restore original handler
-                signal.signal(signal.SIGALRM, old_handler)
-
-        return wrapper
-
-    return decorator
-
-
-def ci_timeout(seconds: int = 60):
-    """
-    Timeout decorator specifically for CI environments.
-    Uses threading for more reliable timeout handling.
-
-    Args:
-        seconds: Timeout in seconds (default: 60)
-    """
-
-    def decorator(func: Callable) -> Callable:
-        @functools.wraps(func)
-        def wrapper(*args: Any, **kwargs: Any) -> Any:
-            # Only apply in CI
-            if os.environ.get("CI") != "true":
-                return func(*args, **kwargs)
-
-            import threading
-
-            result = [None]
-            exception = [None]
-            finished = threading.Event()
-
-            def target():
-                try:
-                    result[0] = func(*args, **kwargs)
-                except Exception as e:
-                    exception[0] = e
-                finally:
-                    finished.set()
-
-            # Start function in thread
-            thread = threading.Thread(target=target, daemon=True)
-            thread.start()
-
-            # Wait for completion or timeout
-            if not finished.wait(timeout=seconds):
-                print(f"\n💥 CI TIMEOUT: Test {func.__name__} exceeded {seconds}s limit!")
-                print("This usually indicates hanging embedding servers or infinite loops.")
-
-                # Try to cleanup embedding servers
-                try:
-                    import subprocess
-
-                    subprocess.run(
-                        ["pkill", "-9", "-f", "embedding_server"], capture_output=True, timeout=2
-                    )
-                    subprocess.run(
-                        ["pkill", "-9", "-f", "hnsw_embedding"], capture_output=True, timeout=2
-                    )
-                    print("Attempted to kill hanging embedding servers.")
-                except Exception as e:
-                    print(f"Cleanup failed: {e}")
-
-                # Raise TimeoutError instead of sys.exit for better pytest integration
-                raise TimeoutError(f"Test {func.__name__} timed out after {seconds} seconds")
-
-            if exception[0]:
-                raise exception[0]
-
-            return result[0]
-
-        return wrapper
-
-    return decorator