feat: use manylinux2014 containers for better Linux compatibility
- Add manylinux2014 Docker containers for Linux builds - This will generate wheels compatible with older Linux systems (CentOS 7+, Ubuntu 16.04+) - Separate build logic for container vs regular environments - Install appropriate system dependencies for yum-based manylinux environment - Use pip instead of uv in containers for better compatibility - Fix Python version format for manylinux container paths
This commit is contained in:
88
.github/workflows/build-reusable.yml
vendored
88
.github/workflows/build-reusable.yml
vendored
@@ -17,21 +17,30 @@ jobs:
|
|||||||
include:
|
include:
|
||||||
- os: ubuntu-latest
|
- os: ubuntu-latest
|
||||||
python: '3.9'
|
python: '3.9'
|
||||||
|
container: 'quay.io/pypa/manylinux2014_x86_64'
|
||||||
- os: ubuntu-latest
|
- os: ubuntu-latest
|
||||||
python: '3.10'
|
python: '3.10'
|
||||||
|
container: 'quay.io/pypa/manylinux2014_x86_64'
|
||||||
- os: ubuntu-latest
|
- os: ubuntu-latest
|
||||||
python: '3.11'
|
python: '3.11'
|
||||||
|
container: 'quay.io/pypa/manylinux2014_x86_64'
|
||||||
- os: ubuntu-latest
|
- os: ubuntu-latest
|
||||||
python: '3.12'
|
python: '3.12'
|
||||||
|
container: 'quay.io/pypa/manylinux2014_x86_64'
|
||||||
- os: macos-latest
|
- os: macos-latest
|
||||||
python: '3.9'
|
python: '3.9'
|
||||||
|
container: ''
|
||||||
- os: macos-latest
|
- os: macos-latest
|
||||||
python: '3.10'
|
python: '3.10'
|
||||||
|
container: ''
|
||||||
- os: macos-latest
|
- os: macos-latest
|
||||||
python: '3.11'
|
python: '3.11'
|
||||||
|
container: ''
|
||||||
- os: macos-latest
|
- os: macos-latest
|
||||||
python: '3.12'
|
python: '3.12'
|
||||||
|
container: ''
|
||||||
runs-on: ${{ matrix.os }}
|
runs-on: ${{ matrix.os }}
|
||||||
|
container: ${{ matrix.container }}
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
@@ -39,16 +48,35 @@ jobs:
|
|||||||
ref: ${{ inputs.ref }}
|
ref: ${{ inputs.ref }}
|
||||||
submodules: recursive
|
submodules: recursive
|
||||||
|
|
||||||
- name: Setup Python
|
- name: Setup Python (macOS and regular Ubuntu)
|
||||||
|
if: matrix.container == ''
|
||||||
uses: actions/setup-python@v5
|
uses: actions/setup-python@v5
|
||||||
with:
|
with:
|
||||||
python-version: ${{ matrix.python }}
|
python-version: ${{ matrix.python }}
|
||||||
|
|
||||||
- name: Install uv
|
- name: Setup Python (manylinux container)
|
||||||
|
if: matrix.container != ''
|
||||||
|
run: |
|
||||||
|
# Use the pre-installed Python version in manylinux container
|
||||||
|
# Convert Python version format (3.9 -> 39, 3.10 -> 310, etc.)
|
||||||
|
PY_VER=$(echo "${{ matrix.python }}" | sed 's/\.//g')
|
||||||
|
/opt/python/cp${PY_VER}-*/bin/python -m pip install --upgrade pip
|
||||||
|
# Create symlinks for convenience
|
||||||
|
ln -sf /opt/python/cp${PY_VER}-*/bin/python /usr/local/bin/python
|
||||||
|
ln -sf /opt/python/cp${PY_VER}-*/bin/pip /usr/local/bin/pip
|
||||||
|
|
||||||
|
- name: Install uv (macOS and regular Ubuntu)
|
||||||
|
if: matrix.container == ''
|
||||||
uses: astral-sh/setup-uv@v4
|
uses: astral-sh/setup-uv@v4
|
||||||
|
|
||||||
- name: Install system dependencies (Ubuntu)
|
- name: Install uv (manylinux container)
|
||||||
if: runner.os == 'Linux'
|
if: matrix.container != ''
|
||||||
|
run: |
|
||||||
|
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||||
|
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
|
||||||
|
|
||||||
|
- name: Install system dependencies (Ubuntu - regular)
|
||||||
|
if: runner.os == 'Linux' && matrix.container == ''
|
||||||
run: |
|
run: |
|
||||||
sudo apt-get update
|
sudo apt-get update
|
||||||
sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \
|
sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \
|
||||||
@@ -61,6 +89,17 @@ jobs:
|
|||||||
echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV
|
echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV
|
||||||
echo "LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH" >> $GITHUB_ENV
|
echo "LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH" >> $GITHUB_ENV
|
||||||
|
|
||||||
|
- name: Install system dependencies (manylinux container)
|
||||||
|
if: runner.os == 'Linux' && matrix.container != ''
|
||||||
|
run: |
|
||||||
|
# manylinux2014 uses yum instead of apt
|
||||||
|
yum install -y epel-release
|
||||||
|
yum install -y boost-devel protobuf-compiler zeromq-devel \
|
||||||
|
pkg-config openblas-devel libaio-devel protobuf-devel
|
||||||
|
|
||||||
|
# Build tools are pre-installed in manylinux
|
||||||
|
# MKL is more complex in container, skip for now and use OpenBLAS
|
||||||
|
|
||||||
- name: Install system dependencies (macOS)
|
- name: Install system dependencies (macOS)
|
||||||
if: runner.os == 'macOS'
|
if: runner.os == 'macOS'
|
||||||
run: |
|
run: |
|
||||||
@@ -68,44 +107,65 @@ jobs:
|
|||||||
|
|
||||||
- name: Install build dependencies
|
- name: Install build dependencies
|
||||||
run: |
|
run: |
|
||||||
uv pip install --system scikit-build-core numpy swig Cython pybind11
|
if [[ -n "${{ matrix.container }}" ]]; then
|
||||||
if [[ "$RUNNER_OS" == "Linux" ]]; then
|
# In manylinux container, use regular pip
|
||||||
uv pip install --system auditwheel
|
pip install scikit-build-core numpy swig Cython pybind11 auditwheel
|
||||||
else
|
else
|
||||||
uv pip install --system delocate
|
# Regular environment, use uv
|
||||||
|
uv pip install --system scikit-build-core numpy swig Cython pybind11
|
||||||
|
if [[ "$RUNNER_OS" == "Linux" ]]; then
|
||||||
|
uv pip install --system auditwheel
|
||||||
|
else
|
||||||
|
uv pip install --system delocate
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
- name: Build packages
|
- name: Build packages
|
||||||
run: |
|
run: |
|
||||||
|
# Choose build command based on environment
|
||||||
|
if [[ -n "${{ matrix.container }}" ]]; then
|
||||||
|
BUILD_CMD="pip wheel . --no-deps -w dist"
|
||||||
|
else
|
||||||
|
BUILD_CMD="uv build --wheel --python python"
|
||||||
|
fi
|
||||||
|
|
||||||
# Build core (platform independent)
|
# Build core (platform independent)
|
||||||
if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
|
if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
|
||||||
cd packages/leann-core
|
cd packages/leann-core
|
||||||
uv build
|
if [[ -n "${{ matrix.container }}" ]]; then
|
||||||
|
pip wheel . --no-deps -w dist
|
||||||
|
else
|
||||||
|
uv build
|
||||||
|
fi
|
||||||
cd ../..
|
cd ../..
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Build HNSW backend
|
# Build HNSW backend
|
||||||
cd packages/leann-backend-hnsw
|
cd packages/leann-backend-hnsw
|
||||||
if [ "${{ matrix.os }}" == "macos-latest" ]; then
|
if [ "${{ matrix.os }}" == "macos-latest" ]; then
|
||||||
CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build --wheel --python python
|
CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ $BUILD_CMD
|
||||||
else
|
else
|
||||||
uv build --wheel --python python
|
eval $BUILD_CMD
|
||||||
fi
|
fi
|
||||||
cd ../..
|
cd ../..
|
||||||
|
|
||||||
# Build DiskANN backend
|
# Build DiskANN backend
|
||||||
cd packages/leann-backend-diskann
|
cd packages/leann-backend-diskann
|
||||||
if [ "${{ matrix.os }}" == "macos-latest" ]; then
|
if [ "${{ matrix.os }}" == "macos-latest" ]; then
|
||||||
CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build --wheel --python python
|
CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ $BUILD_CMD
|
||||||
else
|
else
|
||||||
uv build --wheel --python python
|
eval $BUILD_CMD
|
||||||
fi
|
fi
|
||||||
cd ../..
|
cd ../..
|
||||||
|
|
||||||
# Build meta package (platform independent)
|
# Build meta package (platform independent)
|
||||||
if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
|
if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
|
||||||
cd packages/leann
|
cd packages/leann
|
||||||
uv build
|
if [[ -n "${{ matrix.container }}" ]]; then
|
||||||
|
pip wheel . --no-deps -w dist
|
||||||
|
else
|
||||||
|
uv build
|
||||||
|
fi
|
||||||
cd ../..
|
cd ../..
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user