931051e33b
All type errors in multimodal scripts have been fixed, so we can now include them in the CI type checking.
473 lines
17 KiB
YAML
473 lines
17 KiB
YAML
name: Reusable Build
|
|
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
ref:
|
|
description: 'Git ref to build'
|
|
required: false
|
|
type: string
|
|
default: ''
|
|
|
|
jobs:
|
|
lint:
|
|
name: Lint and Format Check
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ inputs.ref }}
|
|
submodules: recursive
|
|
|
|
- name: Install uv and Python
|
|
uses: astral-sh/setup-uv@v6
|
|
with:
|
|
python-version: '3.11'
|
|
|
|
- name: Run pre-commit with only lint group (no project deps)
|
|
run: |
|
|
uv run --only-group lint pre-commit run --all-files --show-diff-on-failure
|
|
|
|
type-check:
|
|
name: Type Check with ty
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ inputs.ref }}
|
|
submodules: recursive
|
|
|
|
- name: Install uv and Python
|
|
uses: astral-sh/setup-uv@v6
|
|
with:
|
|
python-version: '3.11'
|
|
|
|
- name: Install ty
|
|
run: uv tool install ty
|
|
|
|
- name: Run ty type checker
|
|
run: |
|
|
# Run ty on core packages and apps, excluding tests
|
|
ty check --exclude "tests/**" packages/leann-core/src apps
|
|
|
|
build:
|
|
needs: [lint, type-check]
|
|
name: Build ${{ matrix.os }} Python ${{ matrix.python }}
|
|
strategy:
|
|
matrix:
|
|
include:
|
|
- os: ubuntu-22.04
|
|
python: '3.9'
|
|
- os: ubuntu-22.04
|
|
python: '3.10'
|
|
- os: ubuntu-22.04
|
|
python: '3.11'
|
|
- os: ubuntu-22.04
|
|
python: '3.12'
|
|
- os: ubuntu-22.04
|
|
python: '3.13'
|
|
# ARM64 Linux builds
|
|
- os: ubuntu-24.04-arm
|
|
python: '3.9'
|
|
- os: ubuntu-24.04-arm
|
|
python: '3.10'
|
|
- os: ubuntu-24.04-arm
|
|
python: '3.11'
|
|
- os: ubuntu-24.04-arm
|
|
python: '3.12'
|
|
- os: ubuntu-24.04-arm
|
|
python: '3.13'
|
|
- os: macos-14
|
|
python: '3.9'
|
|
- os: macos-14
|
|
python: '3.10'
|
|
- os: macos-14
|
|
python: '3.11'
|
|
- os: macos-14
|
|
python: '3.12'
|
|
- os: macos-14
|
|
python: '3.13'
|
|
- os: macos-15
|
|
python: '3.9'
|
|
- os: macos-15
|
|
python: '3.10'
|
|
- os: macos-15
|
|
python: '3.11'
|
|
- os: macos-15
|
|
python: '3.12'
|
|
- os: macos-15
|
|
python: '3.13'
|
|
- os: macos-13
|
|
python: '3.9'
|
|
- os: macos-13
|
|
python: '3.10'
|
|
- os: macos-13
|
|
python: '3.11'
|
|
- os: macos-13
|
|
python: '3.12'
|
|
# Note: macos-13 + Python 3.13 excluded due to PyTorch compatibility
|
|
# (PyTorch 2.5+ supports Python 3.13 but not Intel Mac x86_64)
|
|
runs-on: ${{ matrix.os }}
|
|
|
|
steps:
|
|
- uses: actions/checkout@v5
|
|
with:
|
|
ref: ${{ inputs.ref }}
|
|
submodules: recursive
|
|
|
|
- name: Install uv and Python
|
|
uses: astral-sh/setup-uv@v6
|
|
with:
|
|
python-version: ${{ matrix.python }}
|
|
|
|
- name: Install system dependencies (Ubuntu)
|
|
if: runner.os == 'Linux'
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \
|
|
pkg-config libabsl-dev libaio-dev libprotobuf-dev \
|
|
patchelf
|
|
|
|
# Debug: Show system information
|
|
echo "🔍 System Information:"
|
|
echo "Architecture: $(uname -m)"
|
|
echo "OS: $(uname -a)"
|
|
echo "CPU info: $(lscpu | head -5)"
|
|
|
|
# Install math library based on architecture
|
|
ARCH=$(uname -m)
|
|
echo "🔍 Setting up math library for architecture: $ARCH"
|
|
|
|
if [[ "$ARCH" == "x86_64" ]]; then
|
|
# Install Intel MKL for DiskANN on x86_64
|
|
echo "📦 Installing Intel MKL for x86_64..."
|
|
wget -q https://registrationcenter-download.intel.com/akdlm/IRC_NAS/79153e0f-74d7-45af-b8c2-258941adf58a/intel-onemkl-2025.0.0.940.sh
|
|
sudo sh intel-onemkl-2025.0.0.940.sh -a --components intel.oneapi.lin.mkl.devel --action install --eula accept -s
|
|
source /opt/intel/oneapi/setvars.sh
|
|
echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV
|
|
echo "LD_LIBRARY_PATH=/opt/intel/oneapi/compiler/latest/linux/compiler/lib/intel64_lin" >> $GITHUB_ENV
|
|
echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/intel/oneapi/mkl/latest/lib/intel64" >> $GITHUB_ENV
|
|
echo "✅ Intel MKL installed for x86_64"
|
|
|
|
# Debug: Check MKL installation
|
|
echo "🔍 MKL Installation Check:"
|
|
ls -la /opt/intel/oneapi/mkl/latest/ || echo "MKL directory not found"
|
|
ls -la /opt/intel/oneapi/mkl/latest/lib/ || echo "MKL lib directory not found"
|
|
|
|
elif [[ "$ARCH" == "aarch64" ]]; then
|
|
# Use OpenBLAS for ARM64 (MKL installer not compatible with ARM64)
|
|
echo "📦 Installing OpenBLAS for ARM64..."
|
|
sudo apt-get install -y libopenblas-dev liblapack-dev liblapacke-dev
|
|
echo "✅ OpenBLAS installed for ARM64"
|
|
|
|
# Debug: Check OpenBLAS installation
|
|
echo "🔍 OpenBLAS Installation Check:"
|
|
dpkg -l | grep openblas || echo "OpenBLAS package not found"
|
|
ls -la /usr/lib/aarch64-linux-gnu/openblas/ || echo "OpenBLAS directory not found"
|
|
fi
|
|
|
|
# Debug: Show final library paths
|
|
echo "🔍 Final LD_LIBRARY_PATH: $LD_LIBRARY_PATH"
|
|
|
|
- name: Install system dependencies (macOS)
|
|
if: runner.os == 'macOS'
|
|
run: |
|
|
# Don't install LLVM, use system clang for better compatibility
|
|
brew install libomp boost protobuf zeromq
|
|
|
|
- name: Install build dependencies
|
|
run: |
|
|
uv python install ${{ matrix.python }}
|
|
uv venv --python ${{ matrix.python }} .uv-build
|
|
if [[ "$RUNNER_OS" == "Windows" ]]; then
|
|
BUILD_PY=".uv-build\\Scripts\\python.exe"
|
|
else
|
|
BUILD_PY=".uv-build/bin/python"
|
|
fi
|
|
uv pip install --python "$BUILD_PY" scikit-build-core numpy swig Cython pybind11
|
|
if [[ "$RUNNER_OS" == "Linux" ]]; then
|
|
uv pip install --python "$BUILD_PY" auditwheel
|
|
else
|
|
uv pip install --python "$BUILD_PY" delocate
|
|
fi
|
|
|
|
if [[ "$RUNNER_OS" == "Windows" ]]; then
|
|
echo "$(pwd)\\.uv-build\\Scripts" >> $GITHUB_PATH
|
|
else
|
|
echo "$(pwd)/.uv-build/bin" >> $GITHUB_PATH
|
|
fi
|
|
|
|
- name: Set macOS environment variables
|
|
if: runner.os == 'macOS'
|
|
run: |
|
|
# Use brew --prefix to automatically detect Homebrew installation path
|
|
HOMEBREW_PREFIX=$(brew --prefix)
|
|
echo "HOMEBREW_PREFIX=${HOMEBREW_PREFIX}" >> $GITHUB_ENV
|
|
echo "OpenMP_ROOT=${HOMEBREW_PREFIX}/opt/libomp" >> $GITHUB_ENV
|
|
|
|
# Set CMAKE_PREFIX_PATH to let CMake find all packages automatically
|
|
echo "CMAKE_PREFIX_PATH=${HOMEBREW_PREFIX}" >> $GITHUB_ENV
|
|
|
|
# Set compiler flags for OpenMP (required for both backends)
|
|
echo "LDFLAGS=-L${HOMEBREW_PREFIX}/opt/libomp/lib" >> $GITHUB_ENV
|
|
echo "CPPFLAGS=-I${HOMEBREW_PREFIX}/opt/libomp/include" >> $GITHUB_ENV
|
|
|
|
- name: Build packages
|
|
run: |
|
|
# Build core (platform independent)
|
|
cd packages/leann-core
|
|
uv build
|
|
cd ../..
|
|
|
|
# Build HNSW backend
|
|
cd packages/leann-backend-hnsw
|
|
if [[ "${{ matrix.os }}" == macos-* ]]; then
|
|
# Use system clang for better compatibility
|
|
export CC=clang
|
|
export CXX=clang++
|
|
# Homebrew libraries on each macOS version require matching minimum version
|
|
if [[ "${{ matrix.os }}" == "macos-13" ]]; then
|
|
export MACOSX_DEPLOYMENT_TARGET=13.0
|
|
elif [[ "${{ matrix.os }}" == "macos-14" ]]; then
|
|
export MACOSX_DEPLOYMENT_TARGET=14.0
|
|
elif [[ "${{ matrix.os }}" == "macos-15" ]]; then
|
|
export MACOSX_DEPLOYMENT_TARGET=15.0
|
|
fi
|
|
uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
|
|
else
|
|
uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
|
|
fi
|
|
cd ../..
|
|
|
|
# Build DiskANN backend
|
|
cd packages/leann-backend-diskann
|
|
if [[ "${{ matrix.os }}" == macos-* ]]; then
|
|
# Use system clang for better compatibility
|
|
export CC=clang
|
|
export CXX=clang++
|
|
# DiskANN requires macOS 13.3+ for sgesdd_ LAPACK function
|
|
# But Homebrew libraries on each macOS version require matching minimum version
|
|
if [[ "${{ matrix.os }}" == "macos-13" ]]; then
|
|
export MACOSX_DEPLOYMENT_TARGET=13.3
|
|
elif [[ "${{ matrix.os }}" == "macos-14" ]]; then
|
|
export MACOSX_DEPLOYMENT_TARGET=14.0
|
|
elif [[ "${{ matrix.os }}" == "macos-15" ]]; then
|
|
export MACOSX_DEPLOYMENT_TARGET=15.0
|
|
fi
|
|
uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
|
|
else
|
|
uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
|
|
fi
|
|
cd ../..
|
|
|
|
# Build meta package (platform independent)
|
|
cd packages/leann
|
|
uv build
|
|
cd ../..
|
|
|
|
- name: Repair wheels (Linux)
|
|
if: runner.os == 'Linux'
|
|
run: |
|
|
# Repair HNSW wheel
|
|
cd packages/leann-backend-hnsw
|
|
if [ -d dist ]; then
|
|
auditwheel repair dist/*.whl -w dist_repaired
|
|
rm -rf dist
|
|
mv dist_repaired dist
|
|
fi
|
|
cd ../..
|
|
|
|
# Repair DiskANN wheel
|
|
cd packages/leann-backend-diskann
|
|
if [ -d dist ]; then
|
|
auditwheel repair dist/*.whl -w dist_repaired
|
|
rm -rf dist
|
|
mv dist_repaired dist
|
|
fi
|
|
cd ../..
|
|
|
|
- name: Repair wheels (macOS)
|
|
if: runner.os == 'macOS'
|
|
run: |
|
|
# Determine deployment target based on runner OS
|
|
# Must match the Homebrew libraries for each macOS version
|
|
if [[ "${{ matrix.os }}" == "macos-13" ]]; then
|
|
HNSW_TARGET="13.0"
|
|
DISKANN_TARGET="13.3"
|
|
elif [[ "${{ matrix.os }}" == "macos-14" ]]; then
|
|
HNSW_TARGET="14.0"
|
|
DISKANN_TARGET="14.0"
|
|
elif [[ "${{ matrix.os }}" == "macos-15" ]]; then
|
|
HNSW_TARGET="15.0"
|
|
DISKANN_TARGET="15.0"
|
|
fi
|
|
|
|
# Repair HNSW wheel
|
|
cd packages/leann-backend-hnsw
|
|
if [ -d dist ]; then
|
|
export MACOSX_DEPLOYMENT_TARGET=$HNSW_TARGET
|
|
delocate-wheel -w dist_repaired -v --require-target-macos-version $HNSW_TARGET dist/*.whl
|
|
rm -rf dist
|
|
mv dist_repaired dist
|
|
fi
|
|
cd ../..
|
|
|
|
# Repair DiskANN wheel
|
|
cd packages/leann-backend-diskann
|
|
if [ -d dist ]; then
|
|
export MACOSX_DEPLOYMENT_TARGET=$DISKANN_TARGET
|
|
delocate-wheel -w dist_repaired -v --require-target-macos-version $DISKANN_TARGET dist/*.whl
|
|
rm -rf dist
|
|
mv dist_repaired dist
|
|
fi
|
|
cd ../..
|
|
|
|
- name: List built packages
|
|
run: |
|
|
echo "📦 Built packages:"
|
|
find packages/*/dist -name "*.whl" -o -name "*.tar.gz" | sort
|
|
|
|
|
|
- name: Install built packages for testing
|
|
run: |
|
|
# Create uv-managed virtual environment with the requested interpreter
|
|
uv python install ${{ matrix.python }}
|
|
uv venv --python ${{ matrix.python }}
|
|
source .venv/bin/activate || source .venv/Scripts/activate
|
|
|
|
if [[ "$RUNNER_OS" == "Windows" ]]; then
|
|
UV_PY=".venv\\Scripts\\python.exe"
|
|
else
|
|
UV_PY=".venv/bin/python"
|
|
fi
|
|
|
|
# Install test dependency group only (avoids reinstalling project package)
|
|
uv pip install --python "$UV_PY" --group test
|
|
|
|
# Install core wheel built in this job
|
|
CORE_WHL=$(find packages/leann-core/dist -maxdepth 1 -name "*.whl" -print -quit)
|
|
if [[ -n "$CORE_WHL" ]]; then
|
|
uv pip install --python "$UV_PY" "$CORE_WHL"
|
|
else
|
|
uv pip install --python "$UV_PY" packages/leann-core/dist/*.tar.gz
|
|
fi
|
|
|
|
PY_TAG=$($UV_PY -c "import sys; print(f'cp{sys.version_info[0]}{sys.version_info[1]}')")
|
|
|
|
if [[ "$RUNNER_OS" == "macOS" ]]; then
|
|
if [[ "${{ matrix.os }}" == "macos-13" ]]; then
|
|
export MACOSX_DEPLOYMENT_TARGET=13.3
|
|
elif [[ "${{ matrix.os }}" == "macos-14" ]]; then
|
|
export MACOSX_DEPLOYMENT_TARGET=14.0
|
|
elif [[ "${{ matrix.os }}" == "macos-15" ]]; then
|
|
export MACOSX_DEPLOYMENT_TARGET=15.0
|
|
fi
|
|
fi
|
|
|
|
HNSW_WHL=$(find packages/leann-backend-hnsw/dist -maxdepth 1 -name "*-${PY_TAG}-*.whl" -print -quit)
|
|
if [[ -z "$HNSW_WHL" ]]; then
|
|
HNSW_WHL=$(find packages/leann-backend-hnsw/dist -maxdepth 1 -name "*-py3-*.whl" -print -quit)
|
|
fi
|
|
if [[ -n "$HNSW_WHL" ]]; then
|
|
uv pip install --python "$UV_PY" "$HNSW_WHL"
|
|
else
|
|
uv pip install --python "$UV_PY" ./packages/leann-backend-hnsw
|
|
fi
|
|
|
|
DISKANN_WHL=$(find packages/leann-backend-diskann/dist -maxdepth 1 -name "*-${PY_TAG}-*.whl" -print -quit)
|
|
if [[ -z "$DISKANN_WHL" ]]; then
|
|
DISKANN_WHL=$(find packages/leann-backend-diskann/dist -maxdepth 1 -name "*-py3-*.whl" -print -quit)
|
|
fi
|
|
if [[ -n "$DISKANN_WHL" ]]; then
|
|
uv pip install --python "$UV_PY" "$DISKANN_WHL"
|
|
else
|
|
uv pip install --python "$UV_PY" ./packages/leann-backend-diskann
|
|
fi
|
|
|
|
LEANN_WHL=$(find packages/leann/dist -maxdepth 1 -name "*.whl" -print -quit)
|
|
if [[ -n "$LEANN_WHL" ]]; then
|
|
uv pip install --python "$UV_PY" "$LEANN_WHL"
|
|
else
|
|
uv pip install --python "$UV_PY" packages/leann/dist/*.tar.gz
|
|
fi
|
|
|
|
- name: Run tests with pytest
|
|
env:
|
|
CI: true
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
HF_HUB_DISABLE_SYMLINKS: 1
|
|
TOKENIZERS_PARALLELISM: false
|
|
PYTORCH_ENABLE_MPS_FALLBACK: 0
|
|
OMP_NUM_THREADS: 1
|
|
MKL_NUM_THREADS: 1
|
|
run: |
|
|
source .venv/bin/activate || source .venv/Scripts/activate
|
|
pytest tests/ -v --tb=short
|
|
|
|
- name: Run sanity checks (optional)
|
|
run: |
|
|
# Activate virtual environment
|
|
source .venv/bin/activate || source .venv/Scripts/activate
|
|
|
|
# Run distance function tests if available
|
|
if [ -f test/sanity_checks/test_distance_functions.py ]; then
|
|
echo "Running distance function sanity checks..."
|
|
python test/sanity_checks/test_distance_functions.py || echo "⚠️ Distance function test failed, continuing..."
|
|
fi
|
|
|
|
- name: Upload artifacts
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: packages-${{ matrix.os }}-py${{ matrix.python }}
|
|
path: packages/*/dist/
|
|
|
|
|
|
arch-smoke:
|
|
name: Arch Linux smoke test (install & import)
|
|
needs: build
|
|
runs-on: ubuntu-latest
|
|
container:
|
|
image: archlinux:latest
|
|
|
|
steps:
|
|
- name: Prepare system
|
|
run: |
|
|
pacman -Syu --noconfirm
|
|
pacman -S --noconfirm python python-pip gcc git zlib openssl
|
|
|
|
- name: Download ALL wheel artifacts from this run
|
|
uses: actions/download-artifact@v5
|
|
with:
|
|
# Don't specify name, download all artifacts
|
|
path: ./wheels
|
|
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@v6
|
|
|
|
- name: Create virtual environment and install wheels
|
|
run: |
|
|
uv venv
|
|
source .venv/bin/activate || source .venv/Scripts/activate
|
|
uv pip install --find-links wheels leann-core
|
|
uv pip install --find-links wheels leann-backend-hnsw
|
|
uv pip install --find-links wheels leann-backend-diskann
|
|
uv pip install --find-links wheels leann
|
|
|
|
- name: Import & tiny runtime check
|
|
env:
|
|
OMP_NUM_THREADS: 1
|
|
MKL_NUM_THREADS: 1
|
|
run: |
|
|
source .venv/bin/activate || source .venv/Scripts/activate
|
|
python - <<'PY'
|
|
import leann
|
|
import leann_backend_hnsw as h
|
|
import leann_backend_diskann as d
|
|
from leann import LeannBuilder, LeannSearcher
|
|
b = LeannBuilder(backend_name="hnsw")
|
|
b.add_text("hello arch")
|
|
b.build_index("arch_demo.leann")
|
|
s = LeannSearcher("arch_demo.leann")
|
|
print("search:", s.search("hello", top_k=1))
|
|
PY
|