TBNilles/LEANN
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: use manylinux2014 containers for better Linux compatibility

Browse Source 
- Add manylinux2014 Docker containers for Linux builds
- This will generate wheels compatible with older Linux systems (CentOS 7+, Ubuntu 16.04+)
- Separate build logic for container vs regular environments
- Install appropriate system dependencies for yum-based manylinux environment
- Use pip instead of uv in containers for better compatibility
- Fix Python version format for manylinux container paths
This commit is contained in:
Andy Lee
2025-07-25 00:08:42 -07:00
parent e64b599276
commit 6df7893173
1 changed files with 74 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
.github/workflows/build-reusable.yml vendored
View File

@@ -17,21 +17,30 @@ jobs:
include:
- os: ubuntu-latest
python: '3.9'
container: 'quay.io/pypa/manylinux2014_x86_64'
- os: ubuntu-latest
python: '3.10'
container: 'quay.io/pypa/manylinux2014_x86_64'
- os: ubuntu-latest
python: '3.11'
container: 'quay.io/pypa/manylinux2014_x86_64'
- os: ubuntu-latest
python: '3.12'
container: 'quay.io/pypa/manylinux2014_x86_64'
- os: macos-latest
python: '3.9'
container: ''
- os: macos-latest
python: '3.10'
container: ''
- os: macos-latest
python: '3.11'
container: ''
- os: macos-latest
python: '3.12'
container: ''
runs-on: ${{ matrix.os }}
container: ${{ matrix.container }}
steps:
- uses: actions/checkout@v4
@@ -39,16 +48,35 @@ jobs:
ref: ${{ inputs.ref }}
submodules: recursive
- name: Setup Python
- name: Setup Python (macOS and regular Ubuntu)
if: matrix.container == ''
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: Install uv
- name: Setup Python (manylinux container)
if: matrix.container != ''
run: |
# Use the pre-installed Python version in manylinux container
# Convert Python version format (3.9 -> 39, 3.10 -> 310, etc.)
PY_VER=$(echo "${{ matrix.python }}" | sed 's/\.//g')
/opt/python/cp${PY_VER}-*/bin/python -m pip install --upgrade pip
# Create symlinks for convenience
ln -sf /opt/python/cp${PY_VER}-*/bin/python /usr/local/bin/python
ln -sf /opt/python/cp${PY_VER}-*/bin/pip /usr/local/bin/pip
- name: Install uv (macOS and regular Ubuntu)
if: matrix.container == ''
uses: astral-sh/setup-uv@v4
- name: Install system dependencies (Ubuntu)
if: runner.os == 'Linux'
- name: Install uv (manylinux container)
if: matrix.container != ''
run: |
curl -LsSf https://astral.sh/uv/install.sh | sh
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
- name: Install system dependencies (Ubuntu - regular)
if: runner.os == 'Linux' && matrix.container == ''
run: |
sudo apt-get update
sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \
@@ -61,6 +89,17 @@ jobs:
echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH" >> $GITHUB_ENV
- name: Install system dependencies (manylinux container)
if: runner.os == 'Linux' && matrix.container != ''
run: |
# manylinux2014 uses yum instead of apt
yum install -y epel-release
yum install -y boost-devel protobuf-compiler zeromq-devel \
pkg-config openblas-devel libaio-devel protobuf-devel
# Build tools are pre-installed in manylinux
# MKL is more complex in container, skip for now and use OpenBLAS
- name: Install system dependencies (macOS)
if: runner.os == 'macOS'
run: |
@@ -68,44 +107,65 @@ jobs:
- name: Install build dependencies
run: |
uv pip install --system scikit-build-core numpy swig Cython pybind11
if [[ "$RUNNER_OS" == "Linux" ]]; then
uv pip install --system auditwheel
if [[ -n "${{ matrix.container }}" ]]; then
# In manylinux container, use regular pip
pip install scikit-build-core numpy swig Cython pybind11 auditwheel
else
uv pip install --system delocate
# Regular environment, use uv
uv pip install --system scikit-build-core numpy swig Cython pybind11
if [[ "$RUNNER_OS" == "Linux" ]]; then
uv pip install --system auditwheel
else
uv pip install --system delocate
fi
fi
- name: Build packages
run: |
# Choose build command based on environment
if [[ -n "${{ matrix.container }}" ]]; then
BUILD_CMD="pip wheel . --no-deps -w dist"
else
BUILD_CMD="uv build --wheel --python python"
fi
# Build core (platform independent)
if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
cd packages/leann-core
uv build
if [[ -n "${{ matrix.container }}" ]]; then
pip wheel . --no-deps -w dist
else
uv build
fi
cd ../..
fi
# Build HNSW backend
cd packages/leann-backend-hnsw
if [ "${{ matrix.os }}" == "macos-latest" ]; then
CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build --wheel --python python
CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ $BUILD_CMD
else
uv build --wheel --python python
eval $BUILD_CMD
fi
cd ../..
# Build DiskANN backend
cd packages/leann-backend-diskann
if [ "${{ matrix.os }}" == "macos-latest" ]; then
CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build --wheel --python python
CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ $BUILD_CMD
else
uv build --wheel --python python
eval $BUILD_CMD
fi
cd ../..
# Build meta package (platform independent)
if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
cd packages/leann
uv build
if [[ -n "${{ matrix.container }}" ]]; then
pip wheel . --no-deps -w dist
else
uv build
fi
cd ../..
fi