refactor: use reusable workflow to avoid code duplication
This commit is contained in:
Andy Lee
257
.github/workflows/build-and-publish.yml
vendored
257
.github/workflows/build-and-publish.yml
vendored
@@ -1,262 +1,11 @@
|
||||
name: CI - Build Multi-Platform Packages
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main ]
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
publish:
|
||||
description: 'Publish to PyPI (only use for emergency fixes)'
|
||||
required: true
|
||||
default: 'false'
|
||||
type: choice
|
||||
options:
|
||||
- 'false'
|
||||
- 'test'
|
||||
- 'prod'
|
||||
|
||||
jobs:
|
||||
# Build pure Python package: leann-core
|
||||
build-core:
|
||||
name: Build leann-core
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v4
|
||||
|
||||
- name: Install build dependencies
|
||||
run: |
|
||||
uv pip install --system build twine
|
||||
|
||||
- name: Build package
|
||||
run: |
|
||||
cd packages/leann-core
|
||||
uv build
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: leann-core-dist
|
||||
path: packages/leann-core/dist/
|
||||
|
||||
# Build binary package: leann-backend-hnsw (default backend)
|
||||
build-hnsw:
|
||||
name: Build leann-backend-hnsw
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest]
|
||||
python-version: ['3.9', '3.10', '3.11', '3.12']
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v4
|
||||
|
||||
- name: Install system dependencies (Ubuntu)
|
||||
if: runner.os == 'Linux'
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y libomp-dev libboost-all-dev libzmq3-dev \
|
||||
pkg-config libopenblas-dev patchelf
|
||||
|
||||
- name: Install system dependencies (macOS)
|
||||
if: runner.os == 'macOS'
|
||||
run: |
|
||||
brew install libomp boost zeromq
|
||||
|
||||
- name: Install build dependencies
|
||||
run: |
|
||||
uv pip install --system scikit-build-core numpy swig
|
||||
uv pip install --system auditwheel delocate
|
||||
|
||||
- name: Build wheel
|
||||
run: |
|
||||
cd packages/leann-backend-hnsw
|
||||
uv build --wheel --python python
|
||||
|
||||
- name: Repair wheel (Linux)
|
||||
if: runner.os == 'Linux'
|
||||
run: |
|
||||
cd packages/leann-backend-hnsw
|
||||
auditwheel repair dist/*.whl -w dist_repaired
|
||||
rm -rf dist
|
||||
mv dist_repaired dist
|
||||
|
||||
- name: Repair wheel (macOS)
|
||||
if: runner.os == 'macOS'
|
||||
run: |
|
||||
cd packages/leann-backend-hnsw
|
||||
delocate-wheel -w dist_repaired -v dist/*.whl
|
||||
rm -rf dist
|
||||
mv dist_repaired dist
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: hnsw-${{ matrix.os }}-py${{ matrix.python-version }}
|
||||
path: packages/leann-backend-hnsw/dist/
|
||||
|
||||
# Build binary package: leann-backend-diskann (multi-platform)
|
||||
build-diskann:
|
||||
name: Build leann-backend-diskann
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest]
|
||||
python-version: ['3.9', '3.10', '3.11', '3.12']
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v4
|
||||
|
||||
- name: Install system dependencies (Ubuntu)
|
||||
if: runner.os == 'Linux'
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y libomp-dev libboost-all-dev libaio-dev libzmq3-dev \
|
||||
protobuf-compiler libprotobuf-dev libabsl-dev patchelf
|
||||
|
||||
# Install Intel MKL using Intel's installer
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/79153e0f-74d7-45af-b8c2-258941adf58a/intel-onemkl-2025.0.0.940.sh
|
||||
sudo sh intel-onemkl-2025.0.0.940.sh -a --components intel.oneapi.lin.mkl.devel --action install --eula accept -s
|
||||
source /opt/intel/oneapi/setvars.sh
|
||||
echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV
|
||||
echo "LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH" >> $GITHUB_ENV
|
||||
|
||||
- name: Install system dependencies (macOS)
|
||||
if: runner.os == 'macOS'
|
||||
run: |
|
||||
brew install libomp boost zeromq protobuf
|
||||
# MKL is not available on Homebrew, but DiskANN can work without it
|
||||
|
||||
- name: Install build dependencies
|
||||
run: |
|
||||
uv pip install --system scikit-build-core numpy Cython pybind11
|
||||
if [[ "$RUNNER_OS" == "Linux" ]]; then
|
||||
uv pip install --system auditwheel
|
||||
else
|
||||
uv pip install --system delocate
|
||||
fi
|
||||
|
||||
- name: Build wheel
|
||||
run: |
|
||||
cd packages/leann-backend-diskann
|
||||
uv build --wheel --python python
|
||||
|
||||
- name: Repair wheel (Linux)
|
||||
if: runner.os == 'Linux'
|
||||
run: |
|
||||
cd packages/leann-backend-diskann
|
||||
auditwheel repair dist/*.whl -w dist_repaired
|
||||
rm -rf dist
|
||||
mv dist_repaired dist
|
||||
|
||||
- name: Repair wheel (macOS)
|
||||
if: runner.os == 'macOS'
|
||||
run: |
|
||||
cd packages/leann-backend-diskann
|
||||
delocate-wheel -w dist_repaired -v dist/*.whl
|
||||
rm -rf dist
|
||||
mv dist_repaired dist
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: diskann-${{ matrix.os }}-py${{ matrix.python-version }}
|
||||
path: packages/leann-backend-diskann/dist/
|
||||
|
||||
# Build meta-package: leann (build last)
|
||||
build-meta:
|
||||
name: Build leann meta-package
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v4
|
||||
|
||||
- name: Install build dependencies
|
||||
run: |
|
||||
uv pip install --system build
|
||||
|
||||
- name: Build package
|
||||
run: |
|
||||
cd packages/leann
|
||||
uv build
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: leann-meta-dist
|
||||
path: packages/leann/dist/
|
||||
|
||||
# Publish to PyPI (only for emergency fixes or manual triggers)
|
||||
publish:
|
||||
name: Publish to PyPI (Emergency)
|
||||
needs: [build-core, build-hnsw, build-diskann, build-meta]
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event_name == 'workflow_dispatch' && github.event.inputs.publish != 'false'
|
||||
|
||||
steps:
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: dist
|
||||
|
||||
- name: Flatten directory structure
|
||||
run: |
|
||||
mkdir -p all_wheels
|
||||
find dist -name "*.whl" -exec cp {} all_wheels/ \;
|
||||
find dist -name "*.tar.gz" -exec cp {} all_wheels/ \;
|
||||
|
||||
- name: Show what will be published
|
||||
run: |
|
||||
echo "📦 Packages to be published:"
|
||||
ls -la all_wheels/
|
||||
|
||||
- name: Publish to Test PyPI
|
||||
if: github.event.inputs.publish == 'test'
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
with:
|
||||
password: ${{ secrets.TEST_PYPI_API_TOKEN }}
|
||||
repository-url: https://test.pypi.org/legacy/
|
||||
packages-dir: all_wheels/
|
||||
skip-existing: true
|
||||
|
||||
- name: Publish to PyPI
|
||||
if: github.event.inputs.publish == 'prod'
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
with:
|
||||
password: ${{ secrets.PYPI_API_TOKEN }}
|
||||
packages-dir: all_wheels/
|
||||
skip-existing: true
|
||||
build:
|
||||
uses: ./.github/workflows/build-reusable.yml
|
||||
92
.github/workflows/build-reusable.yml
vendored
Normal file
92
.github/workflows/build-reusable.yml
vendored
Normal file
@@ -0,0 +1,92 @@
|
||||
name: Reusable Build
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
ref:
|
||||
description: 'Git ref to build'
|
||||
required: false
|
||||
type: string
|
||||
default: ''
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- os: ubuntu-latest
|
||||
python: '3.11'
|
||||
- os: macos-latest
|
||||
python: '3.11'
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
submodules: recursive
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python }}
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v4
|
||||
|
||||
- name: Install system dependencies (Ubuntu)
|
||||
if: runner.os == 'Linux'
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev
|
||||
|
||||
- name: Install system dependencies (macOS)
|
||||
if: runner.os == 'macOS'
|
||||
run: |
|
||||
brew install llvm libomp boost protobuf zeromq
|
||||
|
||||
- name: Build packages
|
||||
run: |
|
||||
# Build core (platform independent)
|
||||
if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
|
||||
cd packages/leann-core
|
||||
uv build
|
||||
cd ../..
|
||||
fi
|
||||
|
||||
# Build HNSW backend
|
||||
cd packages/leann-backend-hnsw
|
||||
uv pip install --system -r pyproject.toml --extra build
|
||||
if [ "${{ matrix.os }}" == "macos-latest" ]; then
|
||||
CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build
|
||||
else
|
||||
uv build
|
||||
fi
|
||||
cd ../..
|
||||
|
||||
# Build DiskANN backend
|
||||
cd packages/leann-backend-diskann
|
||||
uv pip install --system -r pyproject.toml --extra build
|
||||
if [ "${{ matrix.os }}" == "macos-latest" ]; then
|
||||
CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build
|
||||
else
|
||||
uv build
|
||||
fi
|
||||
cd ../..
|
||||
|
||||
# Build meta package (platform independent)
|
||||
if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
|
||||
cd packages/leann
|
||||
uv build
|
||||
cd ../..
|
||||
fi
|
||||
|
||||
echo "📦 Built packages:"
|
||||
find packages/*/dist -name "*.whl" -o -name "*.tar.gz" | sort
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: packages-${{ matrix.os }}-${{ matrix.python }}
|
||||
path: packages/*/dist/
|
||||
76
.github/workflows/release-manual.yml
vendored
76
.github/workflows/release-manual.yml
vendored
@@ -45,79 +45,9 @@ jobs:
|
||||
build-packages:
|
||||
name: Build packages
|
||||
needs: update-version
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- os: ubuntu-latest
|
||||
python: '3.11'
|
||||
- os: macos-latest
|
||||
python: '3.11'
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ needs.update-version.outputs.commit-sha }}
|
||||
submodules: recursive
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python }}
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v4
|
||||
|
||||
- name: Install system dependencies (Ubuntu)
|
||||
if: runner.os == 'Linux'
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev
|
||||
|
||||
- name: Install system dependencies (macOS)
|
||||
if: runner.os == 'macOS'
|
||||
run: |
|
||||
brew install llvm libomp boost protobuf zeromq
|
||||
|
||||
- name: Build packages
|
||||
run: |
|
||||
# Build core (platform independent)
|
||||
if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
|
||||
cd packages/leann-core
|
||||
uv build
|
||||
cd ../..
|
||||
fi
|
||||
|
||||
# Build HNSW backend
|
||||
cd packages/leann-backend-hnsw
|
||||
uv pip install --system -r pyproject.toml --extra build
|
||||
CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build
|
||||
cd ../..
|
||||
|
||||
# Build DiskANN backend
|
||||
cd packages/leann-backend-diskann
|
||||
uv pip install --system -r pyproject.toml --extra build
|
||||
CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build
|
||||
cd ../..
|
||||
|
||||
# Build meta package (platform independent)
|
||||
if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
|
||||
cd packages/leann
|
||||
uv build
|
||||
cd ../..
|
||||
fi
|
||||
|
||||
echo "📦 Built packages:"
|
||||
find packages/*/dist -name "*.whl" -o -name "*.tar.gz" | sort
|
||||
env:
|
||||
CC: ${{ runner.os == 'macOS' && '$(brew --prefix llvm)/bin/clang' || '' }}
|
||||
CXX: ${{ runner.os == 'macOS' && '$(brew --prefix llvm)/bin/clang++' || '' }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: packages-${{ matrix.os }}-${{ matrix.python }}
|
||||
path: packages/*/dist/
|
||||
uses: ./.github/workflows/build-reusable.yml
|
||||
with:
|
||||
ref: ${{ needs.update-version.outputs.commit-sha }}
|
||||
|
||||
publish:
|
||||
name: Publish and Release
|
||||
|
||||
Reference in New Issue
Block a user