From 6df7893173ee0f7a1b1650e23b38a7996df3eaf7 Mon Sep 17 00:00:00 2001 From: Andy Lee Date: Fri, 25 Jul 2025 00:08:42 -0700 Subject: [PATCH] feat: use manylinux2014 containers for better Linux compatibility - Add manylinux2014 Docker containers for Linux builds - This will generate wheels compatible with older Linux systems (CentOS 7+, Ubuntu 16.04+) - Separate build logic for container vs regular environments - Install appropriate system dependencies for yum-based manylinux environment - Use pip instead of uv in containers for better compatibility - Fix Python version format for manylinux container paths --- .github/workflows/build-reusable.yml | 88 +++++++++++++++++++++++----- 1 file changed, 74 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build-reusable.yml b/.github/workflows/build-reusable.yml index dfdbea2..6c804b7 100644 --- a/.github/workflows/build-reusable.yml +++ b/.github/workflows/build-reusable.yml @@ -17,21 +17,30 @@ jobs: include: - os: ubuntu-latest python: '3.9' + container: 'quay.io/pypa/manylinux2014_x86_64' - os: ubuntu-latest python: '3.10' + container: 'quay.io/pypa/manylinux2014_x86_64' - os: ubuntu-latest python: '3.11' + container: 'quay.io/pypa/manylinux2014_x86_64' - os: ubuntu-latest python: '3.12' + container: 'quay.io/pypa/manylinux2014_x86_64' - os: macos-latest python: '3.9' + container: '' - os: macos-latest python: '3.10' + container: '' - os: macos-latest python: '3.11' + container: '' - os: macos-latest python: '3.12' + container: '' runs-on: ${{ matrix.os }} + container: ${{ matrix.container }} steps: - uses: actions/checkout@v4 @@ -39,16 +48,35 @@ jobs: ref: ${{ inputs.ref }} submodules: recursive - - name: Setup Python + - name: Setup Python (macOS and regular Ubuntu) + if: matrix.container == '' uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - - name: Install uv + - name: Setup Python (manylinux container) + if: matrix.container != '' + run: | + # Use the pre-installed Python version in manylinux container + # Convert Python version format (3.9 -> 39, 3.10 -> 310, etc.) + PY_VER=$(echo "${{ matrix.python }}" | sed 's/\.//g') + /opt/python/cp${PY_VER}-*/bin/python -m pip install --upgrade pip + # Create symlinks for convenience + ln -sf /opt/python/cp${PY_VER}-*/bin/python /usr/local/bin/python + ln -sf /opt/python/cp${PY_VER}-*/bin/pip /usr/local/bin/pip + + - name: Install uv (macOS and regular Ubuntu) + if: matrix.container == '' uses: astral-sh/setup-uv@v4 - - name: Install system dependencies (Ubuntu) - if: runner.os == 'Linux' + - name: Install uv (manylinux container) + if: matrix.container != '' + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + + - name: Install system dependencies (Ubuntu - regular) + if: runner.os == 'Linux' && matrix.container == '' run: | sudo apt-get update sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \ @@ -61,6 +89,17 @@ jobs: echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV echo "LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH" >> $GITHUB_ENV + - name: Install system dependencies (manylinux container) + if: runner.os == 'Linux' && matrix.container != '' + run: | + # manylinux2014 uses yum instead of apt + yum install -y epel-release + yum install -y boost-devel protobuf-compiler zeromq-devel \ + pkg-config openblas-devel libaio-devel protobuf-devel + + # Build tools are pre-installed in manylinux + # MKL is more complex in container, skip for now and use OpenBLAS + - name: Install system dependencies (macOS) if: runner.os == 'macOS' run: | @@ -68,44 +107,65 @@ jobs: - name: Install build dependencies run: | - uv pip install --system scikit-build-core numpy swig Cython pybind11 - if [[ "$RUNNER_OS" == "Linux" ]]; then - uv pip install --system auditwheel + if [[ -n "${{ matrix.container }}" ]]; then + # In manylinux container, use regular pip + pip install scikit-build-core numpy swig Cython pybind11 auditwheel else - uv pip install --system delocate + # Regular environment, use uv + uv pip install --system scikit-build-core numpy swig Cython pybind11 + if [[ "$RUNNER_OS" == "Linux" ]]; then + uv pip install --system auditwheel + else + uv pip install --system delocate + fi fi - name: Build packages run: | + # Choose build command based on environment + if [[ -n "${{ matrix.container }}" ]]; then + BUILD_CMD="pip wheel . --no-deps -w dist" + else + BUILD_CMD="uv build --wheel --python python" + fi + # Build core (platform independent) if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then cd packages/leann-core - uv build + if [[ -n "${{ matrix.container }}" ]]; then + pip wheel . --no-deps -w dist + else + uv build + fi cd ../.. fi # Build HNSW backend cd packages/leann-backend-hnsw if [ "${{ matrix.os }}" == "macos-latest" ]; then - CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build --wheel --python python + CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ $BUILD_CMD else - uv build --wheel --python python + eval $BUILD_CMD fi cd ../.. # Build DiskANN backend cd packages/leann-backend-diskann if [ "${{ matrix.os }}" == "macos-latest" ]; then - CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build --wheel --python python + CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ $BUILD_CMD else - uv build --wheel --python python + eval $BUILD_CMD fi cd ../.. # Build meta package (platform independent) if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then cd packages/leann - uv build + if [[ -n "${{ matrix.container }}" ]]; then + pip wheel . --no-deps -w dist + else + uv build + fi cd ../.. fi