936fa525de
- Add liblapacke-dev to ARM64 dependencies alongside libopenblas-dev - Provides lapacke.h header file needed for LAPACK C interface - Fixes 'lapacke.h: No such file or directory' compilation error - Enables complete OpenBLAS + LAPACKE support for ARM64 wheel builds 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
403 lines
14 KiB
YAML
403 lines
14 KiB
YAML
name: Reusable Build
|
|
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
ref:
|
|
description: 'Git ref to build'
|
|
required: false
|
|
type: string
|
|
default: ''
|
|
|
|
jobs:
|
|
lint:
|
|
name: Lint and Format Check
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ inputs.ref }}
|
|
|
|
- name: Setup Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.11'
|
|
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@v4
|
|
|
|
- name: Install ruff
|
|
run: |
|
|
uv tool install ruff
|
|
|
|
- name: Run ruff check
|
|
run: |
|
|
ruff check .
|
|
|
|
- name: Run ruff format check
|
|
run: |
|
|
ruff format --check .
|
|
|
|
build:
|
|
needs: lint
|
|
name: Build ${{ matrix.os }} Python ${{ matrix.python }}
|
|
strategy:
|
|
matrix:
|
|
include:
|
|
- os: ubuntu-22.04
|
|
python: '3.9'
|
|
- os: ubuntu-22.04
|
|
python: '3.10'
|
|
- os: ubuntu-22.04
|
|
python: '3.11'
|
|
- os: ubuntu-22.04
|
|
python: '3.12'
|
|
- os: ubuntu-22.04
|
|
python: '3.13'
|
|
# ARM64 Linux builds
|
|
- os: ubuntu-24.04-arm
|
|
python: '3.9'
|
|
- os: ubuntu-24.04-arm
|
|
python: '3.10'
|
|
- os: ubuntu-24.04-arm
|
|
python: '3.11'
|
|
- os: ubuntu-24.04-arm
|
|
python: '3.12'
|
|
- os: ubuntu-24.04-arm
|
|
python: '3.13'
|
|
- os: macos-14
|
|
python: '3.9'
|
|
- os: macos-14
|
|
python: '3.10'
|
|
- os: macos-14
|
|
python: '3.11'
|
|
- os: macos-14
|
|
python: '3.12'
|
|
- os: macos-14
|
|
python: '3.13'
|
|
- os: macos-15
|
|
python: '3.9'
|
|
- os: macos-15
|
|
python: '3.10'
|
|
- os: macos-15
|
|
python: '3.11'
|
|
- os: macos-15
|
|
python: '3.12'
|
|
- os: macos-15
|
|
python: '3.13'
|
|
- os: macos-13
|
|
python: '3.9'
|
|
- os: macos-13
|
|
python: '3.10'
|
|
- os: macos-13
|
|
python: '3.11'
|
|
- os: macos-13
|
|
python: '3.12'
|
|
# Note: macos-13 + Python 3.13 excluded due to PyTorch compatibility
|
|
# (PyTorch 2.5+ supports Python 3.13 but not Intel Mac x86_64)
|
|
runs-on: ${{ matrix.os }}
|
|
|
|
steps:
|
|
- uses: actions/checkout@v5
|
|
with:
|
|
ref: ${{ inputs.ref }}
|
|
submodules: recursive
|
|
|
|
- name: Setup Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: ${{ matrix.python }}
|
|
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@v6
|
|
|
|
- name: Install system dependencies (Ubuntu)
|
|
if: runner.os == 'Linux'
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \
|
|
pkg-config libabsl-dev libaio-dev libprotobuf-dev \
|
|
patchelf
|
|
|
|
# Debug: Show system information
|
|
echo "🔍 System Information:"
|
|
echo "Architecture: $(uname -m)"
|
|
echo "OS: $(uname -a)"
|
|
echo "CPU info: $(lscpu | head -5)"
|
|
|
|
# Install math library based on architecture
|
|
ARCH=$(uname -m)
|
|
echo "🔍 Setting up math library for architecture: $ARCH"
|
|
|
|
if [[ "$ARCH" == "x86_64" ]]; then
|
|
# Install Intel MKL for DiskANN on x86_64
|
|
echo "📦 Installing Intel MKL for x86_64..."
|
|
wget -q https://registrationcenter-download.intel.com/akdlm/IRC_NAS/79153e0f-74d7-45af-b8c2-258941adf58a/intel-onemkl-2025.0.0.940.sh
|
|
sudo sh intel-onemkl-2025.0.0.940.sh -a --components intel.oneapi.lin.mkl.devel --action install --eula accept -s
|
|
source /opt/intel/oneapi/setvars.sh
|
|
echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV
|
|
echo "LD_LIBRARY_PATH=/opt/intel/oneapi/compiler/latest/linux/compiler/lib/intel64_lin" >> $GITHUB_ENV
|
|
echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/intel/oneapi/mkl/latest/lib/intel64" >> $GITHUB_ENV
|
|
echo "✅ Intel MKL installed for x86_64"
|
|
|
|
# Debug: Check MKL installation
|
|
echo "🔍 MKL Installation Check:"
|
|
ls -la /opt/intel/oneapi/mkl/latest/ || echo "MKL directory not found"
|
|
ls -la /opt/intel/oneapi/mkl/latest/lib/ || echo "MKL lib directory not found"
|
|
|
|
elif [[ "$ARCH" == "aarch64" ]]; then
|
|
# Use OpenBLAS for ARM64 (MKL installer not compatible with ARM64)
|
|
echo "📦 Installing OpenBLAS for ARM64..."
|
|
sudo apt-get install -y libopenblas-dev liblapack-dev liblapacke-dev
|
|
echo "✅ OpenBLAS installed for ARM64"
|
|
|
|
# Debug: Check OpenBLAS installation
|
|
echo "🔍 OpenBLAS Installation Check:"
|
|
dpkg -l | grep openblas || echo "OpenBLAS package not found"
|
|
ls -la /usr/lib/aarch64-linux-gnu/openblas/ || echo "OpenBLAS directory not found"
|
|
fi
|
|
|
|
# Debug: Show final library paths
|
|
echo "🔍 Final LD_LIBRARY_PATH: $LD_LIBRARY_PATH"
|
|
|
|
- name: Install system dependencies (macOS)
|
|
if: runner.os == 'macOS'
|
|
run: |
|
|
# Don't install LLVM, use system clang for better compatibility
|
|
brew install libomp boost protobuf zeromq
|
|
|
|
- name: Install build dependencies
|
|
run: |
|
|
uv pip install --system scikit-build-core numpy swig Cython pybind11
|
|
if [[ "$RUNNER_OS" == "Linux" ]]; then
|
|
uv pip install --system auditwheel
|
|
else
|
|
uv pip install --system delocate
|
|
fi
|
|
|
|
- name: Set macOS environment variables
|
|
if: runner.os == 'macOS'
|
|
run: |
|
|
# Use brew --prefix to automatically detect Homebrew installation path
|
|
HOMEBREW_PREFIX=$(brew --prefix)
|
|
echo "HOMEBREW_PREFIX=${HOMEBREW_PREFIX}" >> $GITHUB_ENV
|
|
echo "OpenMP_ROOT=${HOMEBREW_PREFIX}/opt/libomp" >> $GITHUB_ENV
|
|
|
|
# Set CMAKE_PREFIX_PATH to let CMake find all packages automatically
|
|
echo "CMAKE_PREFIX_PATH=${HOMEBREW_PREFIX}" >> $GITHUB_ENV
|
|
|
|
# Set compiler flags for OpenMP (required for both backends)
|
|
echo "LDFLAGS=-L${HOMEBREW_PREFIX}/opt/libomp/lib" >> $GITHUB_ENV
|
|
echo "CPPFLAGS=-I${HOMEBREW_PREFIX}/opt/libomp/include" >> $GITHUB_ENV
|
|
|
|
- name: Build packages
|
|
run: |
|
|
# Build core (platform independent)
|
|
cd packages/leann-core
|
|
uv build
|
|
cd ../..
|
|
|
|
# Build HNSW backend
|
|
cd packages/leann-backend-hnsw
|
|
if [[ "${{ matrix.os }}" == macos-* ]]; then
|
|
# Use system clang for better compatibility
|
|
export CC=clang
|
|
export CXX=clang++
|
|
# Homebrew libraries on each macOS version require matching minimum version
|
|
if [[ "${{ matrix.os }}" == "macos-13" ]]; then
|
|
export MACOSX_DEPLOYMENT_TARGET=13.0
|
|
elif [[ "${{ matrix.os }}" == "macos-14" ]]; then
|
|
export MACOSX_DEPLOYMENT_TARGET=14.0
|
|
elif [[ "${{ matrix.os }}" == "macos-15" ]]; then
|
|
export MACOSX_DEPLOYMENT_TARGET=15.0
|
|
fi
|
|
uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
|
|
else
|
|
uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
|
|
fi
|
|
cd ../..
|
|
|
|
# Build DiskANN backend
|
|
cd packages/leann-backend-diskann
|
|
if [[ "${{ matrix.os }}" == macos-* ]]; then
|
|
# Use system clang for better compatibility
|
|
export CC=clang
|
|
export CXX=clang++
|
|
# DiskANN requires macOS 13.3+ for sgesdd_ LAPACK function
|
|
# But Homebrew libraries on each macOS version require matching minimum version
|
|
if [[ "${{ matrix.os }}" == "macos-13" ]]; then
|
|
export MACOSX_DEPLOYMENT_TARGET=13.3
|
|
elif [[ "${{ matrix.os }}" == "macos-14" ]]; then
|
|
export MACOSX_DEPLOYMENT_TARGET=14.0
|
|
elif [[ "${{ matrix.os }}" == "macos-15" ]]; then
|
|
export MACOSX_DEPLOYMENT_TARGET=15.0
|
|
fi
|
|
uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
|
|
else
|
|
uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
|
|
fi
|
|
cd ../..
|
|
|
|
# Build meta package (platform independent)
|
|
cd packages/leann
|
|
uv build
|
|
cd ../..
|
|
|
|
- name: Repair wheels (Linux)
|
|
if: runner.os == 'Linux'
|
|
run: |
|
|
# Repair HNSW wheel
|
|
cd packages/leann-backend-hnsw
|
|
if [ -d dist ]; then
|
|
auditwheel repair dist/*.whl -w dist_repaired
|
|
rm -rf dist
|
|
mv dist_repaired dist
|
|
fi
|
|
cd ../..
|
|
|
|
# Repair DiskANN wheel
|
|
cd packages/leann-backend-diskann
|
|
if [ -d dist ]; then
|
|
auditwheel repair dist/*.whl -w dist_repaired
|
|
rm -rf dist
|
|
mv dist_repaired dist
|
|
fi
|
|
cd ../..
|
|
|
|
- name: Repair wheels (macOS)
|
|
if: runner.os == 'macOS'
|
|
run: |
|
|
# Determine deployment target based on runner OS
|
|
# Must match the Homebrew libraries for each macOS version
|
|
if [[ "${{ matrix.os }}" == "macos-13" ]]; then
|
|
HNSW_TARGET="13.0"
|
|
DISKANN_TARGET="13.3"
|
|
elif [[ "${{ matrix.os }}" == "macos-14" ]]; then
|
|
HNSW_TARGET="14.0"
|
|
DISKANN_TARGET="14.0"
|
|
elif [[ "${{ matrix.os }}" == "macos-15" ]]; then
|
|
HNSW_TARGET="15.0"
|
|
DISKANN_TARGET="15.0"
|
|
fi
|
|
|
|
# Repair HNSW wheel
|
|
cd packages/leann-backend-hnsw
|
|
if [ -d dist ]; then
|
|
export MACOSX_DEPLOYMENT_TARGET=$HNSW_TARGET
|
|
delocate-wheel -w dist_repaired -v --require-target-macos-version $HNSW_TARGET dist/*.whl
|
|
rm -rf dist
|
|
mv dist_repaired dist
|
|
fi
|
|
cd ../..
|
|
|
|
# Repair DiskANN wheel
|
|
cd packages/leann-backend-diskann
|
|
if [ -d dist ]; then
|
|
export MACOSX_DEPLOYMENT_TARGET=$DISKANN_TARGET
|
|
delocate-wheel -w dist_repaired -v --require-target-macos-version $DISKANN_TARGET dist/*.whl
|
|
rm -rf dist
|
|
mv dist_repaired dist
|
|
fi
|
|
cd ../..
|
|
|
|
- name: List built packages
|
|
run: |
|
|
echo "📦 Built packages:"
|
|
find packages/*/dist -name "*.whl" -o -name "*.tar.gz" | sort
|
|
|
|
|
|
- name: Install built packages for testing
|
|
run: |
|
|
# Create a virtual environment with the correct Python version
|
|
uv venv --python ${{ matrix.python }}
|
|
source .venv/bin/activate || source .venv/Scripts/activate
|
|
|
|
# Install packages using --find-links to prioritize local builds
|
|
uv pip install --find-links packages/leann-core/dist --find-links packages/leann-backend-hnsw/dist --find-links packages/leann-backend-diskann/dist packages/leann-core/dist/*.whl || uv pip install --find-links packages/leann-core/dist packages/leann-core/dist/*.tar.gz
|
|
uv pip install --find-links packages/leann-core/dist packages/leann-backend-hnsw/dist/*.whl
|
|
uv pip install --find-links packages/leann-core/dist packages/leann-backend-diskann/dist/*.whl
|
|
uv pip install packages/leann/dist/*.whl || uv pip install packages/leann/dist/*.tar.gz
|
|
|
|
# Install test dependencies using extras
|
|
uv pip install -e ".[test]"
|
|
|
|
- name: Run tests with pytest
|
|
env:
|
|
CI: true
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
HF_HUB_DISABLE_SYMLINKS: 1
|
|
TOKENIZERS_PARALLELISM: false
|
|
PYTORCH_ENABLE_MPS_FALLBACK: 0
|
|
OMP_NUM_THREADS: 1
|
|
MKL_NUM_THREADS: 1
|
|
run: |
|
|
source .venv/bin/activate || source .venv/Scripts/activate
|
|
pytest tests/ -v --tb=short
|
|
|
|
- name: Run sanity checks (optional)
|
|
run: |
|
|
# Activate virtual environment
|
|
source .venv/bin/activate || source .venv/Scripts/activate
|
|
|
|
# Run distance function tests if available
|
|
if [ -f test/sanity_checks/test_distance_functions.py ]; then
|
|
echo "Running distance function sanity checks..."
|
|
python test/sanity_checks/test_distance_functions.py || echo "⚠️ Distance function test failed, continuing..."
|
|
fi
|
|
|
|
- name: Upload artifacts
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: packages-${{ matrix.os }}-py${{ matrix.python }}
|
|
path: packages/*/dist/
|
|
|
|
|
|
arch-smoke:
|
|
name: Arch Linux smoke test (install & import)
|
|
needs: build
|
|
runs-on: ubuntu-latest
|
|
container:
|
|
image: archlinux:latest
|
|
|
|
steps:
|
|
- name: Prepare system
|
|
run: |
|
|
pacman -Syu --noconfirm
|
|
pacman -S --noconfirm python python-pip gcc git zlib openssl
|
|
|
|
- name: Download ALL wheel artifacts from this run
|
|
uses: actions/download-artifact@v5
|
|
with:
|
|
# Don't specify name, download all artifacts
|
|
path: ./wheels
|
|
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@v6
|
|
|
|
- name: Create virtual environment and install wheels
|
|
run: |
|
|
uv venv
|
|
source .venv/bin/activate || source .venv/Scripts/activate
|
|
uv pip install --find-links wheels leann-core
|
|
uv pip install --find-links wheels leann-backend-hnsw
|
|
uv pip install --find-links wheels leann-backend-diskann
|
|
uv pip install --find-links wheels leann
|
|
|
|
- name: Import & tiny runtime check
|
|
env:
|
|
OMP_NUM_THREADS: 1
|
|
MKL_NUM_THREADS: 1
|
|
run: |
|
|
source .venv/bin/activate || source .venv/Scripts/activate
|
|
python - <<'PY'
|
|
import leann
|
|
import leann_backend_hnsw as h
|
|
import leann_backend_diskann as d
|
|
from leann import LeannBuilder, LeannSearcher
|
|
b = LeannBuilder(backend_name="hnsw")
|
|
b.add_text("hello arch")
|
|
b.build_index("arch_demo.leann")
|
|
s = LeannSearcher("arch_demo.leann")
|
|
print("search:", s.search("hello", top_k=1))
|
|
PY
|