LEANN/.github/workflows/build-reusable.yml

name: Reusable Build

on:
  workflow_call:
    inputs:
      ref:
        description: 'Git ref to build'
        required: false
        type: string
        default: ''

jobs:
  lint:
    name: Lint and Format Check
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref }}

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install uv
        uses: astral-sh/setup-uv@v4

      - name: Install ruff
        run: |
          uv tool install ruff

      - name: Run ruff check
        run: |
          ruff check .

      - name: Run ruff format check
        run: |
          ruff format --check .

  build:
    needs: lint
    name: Build ${{ matrix.os }} Python ${{ matrix.python }}
    strategy:
      matrix:
        include:
          - os: ubuntu-22.04
            python: '3.9'
          - os: ubuntu-22.04
            python: '3.10'
          - os: ubuntu-22.04
            python: '3.11'
          - os: ubuntu-22.04
            python: '3.12'
          - os: ubuntu-22.04
            python: '3.13'
          # ARM64 Linux builds
          - os: ubuntu-24.04-arm
            python: '3.9'
          - os: ubuntu-24.04-arm
            python: '3.10'
          - os: ubuntu-24.04-arm
            python: '3.11'
          - os: ubuntu-24.04-arm
            python: '3.12'
          - os: ubuntu-24.04-arm
            python: '3.13'
          - os: macos-14
            python: '3.9'
          - os: macos-14
            python: '3.10'
          - os: macos-14
            python: '3.11'
          - os: macos-14
            python: '3.12'
          - os: macos-14
            python: '3.13'
          - os: macos-15
            python: '3.9'
          - os: macos-15
            python: '3.10'
          - os: macos-15
            python: '3.11'
          - os: macos-15
            python: '3.12'
          - os: macos-15
            python: '3.13'
          - os: macos-13
            python: '3.9'
          - os: macos-13
            python: '3.10'
          - os: macos-13
            python: '3.11'
          - os: macos-13
            python: '3.12'
          # Note: macos-13 + Python 3.13 excluded due to PyTorch compatibility
          # (PyTorch 2.5+ supports Python 3.13 but not Intel Mac x86_64)
    runs-on: ${{ matrix.os }}

    steps:
      - uses: actions/checkout@v5
        with:
          ref: ${{ inputs.ref }}
          submodules: recursive

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}

      - name: Install uv
        uses: astral-sh/setup-uv@v6

      - name: Install system dependencies (Ubuntu)
        if: runner.os == 'Linux'
        run: |
          sudo apt-get update
          sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \
            pkg-config libabsl-dev libaio-dev libprotobuf-dev \
            patchelf

          # Debug: Show system information
          echo "🔍 System Information:"
          echo "Architecture: $(uname -m)"
          echo "OS: $(uname -a)"
          echo "CPU info: $(lscpu | head -5)"

          # Install math library based on architecture
          ARCH=$(uname -m)
          echo "🔍 Setting up math library for architecture: $ARCH"

          if [[ "$ARCH" == "x86_64" ]]; then
            # Install Intel MKL for DiskANN on x86_64
            echo "📦 Installing Intel MKL for x86_64..."
            wget -q https://registrationcenter-download.intel.com/akdlm/IRC_NAS/79153e0f-74d7-45af-b8c2-258941adf58a/intel-onemkl-2025.0.0.940.sh
            sudo sh intel-onemkl-2025.0.0.940.sh -a --components intel.oneapi.lin.mkl.devel --action install --eula accept -s
            source /opt/intel/oneapi/setvars.sh
            echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV
            echo "LD_LIBRARY_PATH=/opt/intel/oneapi/compiler/latest/linux/compiler/lib/intel64_lin" >> $GITHUB_ENV
            echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/intel/oneapi/mkl/latest/lib/intel64" >> $GITHUB_ENV
            echo "✅ Intel MKL installed for x86_64"

            # Debug: Check MKL installation
            echo "🔍 MKL Installation Check:"
            ls -la /opt/intel/oneapi/mkl/latest/ || echo "MKL directory not found"
            ls -la /opt/intel/oneapi/mkl/latest/lib/ || echo "MKL lib directory not found"

          elif [[ "$ARCH" == "aarch64" ]]; then
            # Use OpenBLAS for ARM64 (MKL installer not compatible with ARM64)
            echo "📦 Installing OpenBLAS for ARM64..."
            sudo apt-get install -y libopenblas-dev liblapack-dev liblapacke-dev
            echo "✅ OpenBLAS installed for ARM64"

            # Debug: Check OpenBLAS installation
            echo "🔍 OpenBLAS Installation Check:"
            dpkg -l | grep openblas || echo "OpenBLAS package not found"
            ls -la /usr/lib/aarch64-linux-gnu/openblas/ || echo "OpenBLAS directory not found"
          fi

          # Debug: Show final library paths
          echo "🔍 Final LD_LIBRARY_PATH: $LD_LIBRARY_PATH"

      - name: Install system dependencies (macOS)
        if: runner.os == 'macOS'
        run: |
          # Don't install LLVM, use system clang for better compatibility
          brew install libomp boost protobuf zeromq

      - name: Install build dependencies
        run: |
          uv pip install --system scikit-build-core numpy swig Cython pybind11
          if [[ "$RUNNER_OS" == "Linux" ]]; then
            uv pip install --system auditwheel
          else
            uv pip install --system delocate
          fi

      - name: Set macOS environment variables
        if: runner.os == 'macOS'
        run: |
          # Use brew --prefix to automatically detect Homebrew installation path
          HOMEBREW_PREFIX=$(brew --prefix)
          echo "HOMEBREW_PREFIX=${HOMEBREW_PREFIX}" >> $GITHUB_ENV
          echo "OpenMP_ROOT=${HOMEBREW_PREFIX}/opt/libomp" >> $GITHUB_ENV

          # Set CMAKE_PREFIX_PATH to let CMake find all packages automatically
          echo "CMAKE_PREFIX_PATH=${HOMEBREW_PREFIX}" >> $GITHUB_ENV

          # Set compiler flags for OpenMP (required for both backends)
          echo "LDFLAGS=-L${HOMEBREW_PREFIX}/opt/libomp/lib" >> $GITHUB_ENV
          echo "CPPFLAGS=-I${HOMEBREW_PREFIX}/opt/libomp/include" >> $GITHUB_ENV

      - name: Build packages
        run: |
          # Build core (platform independent)
          cd packages/leann-core
          uv build
          cd ../..

          # Build HNSW backend
          cd packages/leann-backend-hnsw
          if [[ "${{ matrix.os }}" == macos-* ]]; then
            # Use system clang for better compatibility
            export CC=clang
            export CXX=clang++
            # Homebrew libraries on each macOS version require matching minimum version
            if [[ "${{ matrix.os }}" == "macos-13" ]]; then
              export MACOSX_DEPLOYMENT_TARGET=13.0
            elif [[ "${{ matrix.os }}" == "macos-14" ]]; then
              export MACOSX_DEPLOYMENT_TARGET=14.0
            elif [[ "${{ matrix.os }}" == "macos-15" ]]; then
              export MACOSX_DEPLOYMENT_TARGET=15.0
            fi
            uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
          else
            uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
          fi
          cd ../..

          # Build DiskANN backend
          cd packages/leann-backend-diskann
          if [[ "${{ matrix.os }}" == macos-* ]]; then
            # Use system clang for better compatibility
            export CC=clang
            export CXX=clang++
            # DiskANN requires macOS 13.3+ for sgesdd_ LAPACK function
            # But Homebrew libraries on each macOS version require matching minimum version
            if [[ "${{ matrix.os }}" == "macos-13" ]]; then
              export MACOSX_DEPLOYMENT_TARGET=13.3
            elif [[ "${{ matrix.os }}" == "macos-14" ]]; then
              export MACOSX_DEPLOYMENT_TARGET=14.0
            elif [[ "${{ matrix.os }}" == "macos-15" ]]; then
              export MACOSX_DEPLOYMENT_TARGET=15.0
            fi
            uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
          else
            uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
          fi
          cd ../..

          # Build meta package (platform independent)
          cd packages/leann
          uv build
          cd ../..

      - name: Repair wheels (Linux)
        if: runner.os == 'Linux'
        run: |
          # Repair HNSW wheel
          cd packages/leann-backend-hnsw
          if [ -d dist ]; then
            auditwheel repair dist/*.whl -w dist_repaired
            rm -rf dist
            mv dist_repaired dist
          fi
          cd ../..

          # Repair DiskANN wheel
          cd packages/leann-backend-diskann
          if [ -d dist ]; then
            auditwheel repair dist/*.whl -w dist_repaired
            rm -rf dist
            mv dist_repaired dist
          fi
          cd ../..

      - name: Repair wheels (macOS)
        if: runner.os == 'macOS'
        run: |
          # Determine deployment target based on runner OS
          # Must match the Homebrew libraries for each macOS version
          if [[ "${{ matrix.os }}" == "macos-13" ]]; then
            HNSW_TARGET="13.0"
            DISKANN_TARGET="13.3"
          elif [[ "${{ matrix.os }}" == "macos-14" ]]; then
            HNSW_TARGET="14.0"
            DISKANN_TARGET="14.0"
          elif [[ "${{ matrix.os }}" == "macos-15" ]]; then
            HNSW_TARGET="15.0"
            DISKANN_TARGET="15.0"
          fi

          # Repair HNSW wheel
          cd packages/leann-backend-hnsw
          if [ -d dist ]; then
            export MACOSX_DEPLOYMENT_TARGET=$HNSW_TARGET
            delocate-wheel -w dist_repaired -v --require-target-macos-version $HNSW_TARGET dist/*.whl
            rm -rf dist
            mv dist_repaired dist
          fi
          cd ../..

          # Repair DiskANN wheel
          cd packages/leann-backend-diskann
          if [ -d dist ]; then
            export MACOSX_DEPLOYMENT_TARGET=$DISKANN_TARGET
            delocate-wheel -w dist_repaired -v --require-target-macos-version $DISKANN_TARGET dist/*.whl
            rm -rf dist
            mv dist_repaired dist
          fi
          cd ../..

      - name: List built packages
        run: |
          echo "📦 Built packages:"
          find packages/*/dist -name "*.whl" -o -name "*.tar.gz" | sort


      - name: Install built packages for testing
        run: |
          # Create a virtual environment with the correct Python version
          uv venv --python ${{ matrix.python }}
          source .venv/bin/activate || source .venv/Scripts/activate

          # Install packages using --find-links to prioritize local builds
          uv pip install --find-links packages/leann-core/dist --find-links packages/leann-backend-hnsw/dist --find-links packages/leann-backend-diskann/dist packages/leann-core/dist/*.whl || uv pip install --find-links packages/leann-core/dist packages/leann-core/dist/*.tar.gz
          uv pip install --find-links packages/leann-core/dist packages/leann-backend-hnsw/dist/*.whl
          uv pip install --find-links packages/leann-core/dist packages/leann-backend-diskann/dist/*.whl
          uv pip install packages/leann/dist/*.whl || uv pip install packages/leann/dist/*.tar.gz

          # Install test dependencies using extras
          uv pip install -e ".[test]"

      - name: Run tests with pytest
        env:
          CI: true
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          HF_HUB_DISABLE_SYMLINKS: 1
          TOKENIZERS_PARALLELISM: false
          PYTORCH_ENABLE_MPS_FALLBACK: 0
          OMP_NUM_THREADS: 1
          MKL_NUM_THREADS: 1
        run: |
          source .venv/bin/activate || source .venv/Scripts/activate
          pytest tests/ -v --tb=short

      - name: Run sanity checks (optional)
        run: |
          # Activate virtual environment
          source .venv/bin/activate || source .venv/Scripts/activate

          # Run distance function tests if available
          if [ -f test/sanity_checks/test_distance_functions.py ]; then
            echo "Running distance function sanity checks..."
            python test/sanity_checks/test_distance_functions.py || echo "⚠️ Distance function test failed, continuing..."
          fi

      - name: Upload artifacts
        uses: actions/upload-artifact@v4
        with:
          name: packages-${{ matrix.os }}-py${{ matrix.python }}
          path: packages/*/dist/


  arch-smoke:
    name: Arch Linux smoke test (install & import)
    needs: build
    runs-on: ubuntu-latest
    container:
      image: archlinux:latest

    steps:
      - name: Prepare system
        run: |
          pacman -Syu --noconfirm
          pacman -S --noconfirm python python-pip gcc git zlib openssl

      - name: Download ALL wheel artifacts from this run
        uses: actions/download-artifact@v5
        with:
          # Don't specify name, download all artifacts
          path: ./wheels

      - name: Install uv
        uses: astral-sh/setup-uv@v6

      - name: Create virtual environment and install wheels
        run: |
          uv venv
          source .venv/bin/activate || source .venv/Scripts/activate
          uv pip install --find-links wheels leann-core
          uv pip install --find-links wheels leann-backend-hnsw
          uv pip install --find-links wheels leann-backend-diskann
          uv pip install --find-links wheels leann

      - name: Import & tiny runtime check
        env:
          OMP_NUM_THREADS: 1
          MKL_NUM_THREADS: 1
        run: |
          source .venv/bin/activate || source .venv/Scripts/activate
          python - <<'PY'
          import leann
          import leann_backend_hnsw as h
          import leann_backend_diskann as d
          from leann import LeannBuilder, LeannSearcher
          b = LeannBuilder(backend_name="hnsw")
          b.add_text("hello arch")
          b.build_index("arch_demo.leann")
          s = LeannSearcher("arch_demo.leann")
          print("search:", s.search("hello", top_k=1))
          PY