revert: simplify build system by removing manylinux/cibuildwheel
- Revert to simple Ubuntu 22.04 builds that should work with Colab - Remove all manylinux container complexity - Colab runs on Ubuntu 22.04, so direct builds should be compatible - Restore build-reusable.yml to v0.1.5 version - Remove cibuildwheel option from release workflow This should fix the overcomplicated build issues while maintaining Colab compatibility through direct Ubuntu 22.04 builds.
This commit is contained in:
Andy Lee
180
.github/workflows/build-reusable.yml
vendored
180
.github/workflows/build-reusable.yml
vendored
@@ -13,107 +13,42 @@ jobs:
|
||||
build:
|
||||
name: Build ${{ matrix.os }} Python ${{ matrix.python }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- os: ubuntu-latest
|
||||
python: '3.9'
|
||||
container: 'quay.io/pypa/manylinux2014_x86_64'
|
||||
- os: ubuntu-latest
|
||||
python: '3.10'
|
||||
container: 'quay.io/pypa/manylinux2014_x86_64'
|
||||
- os: ubuntu-latest
|
||||
python: '3.11'
|
||||
container: 'quay.io/pypa/manylinux2014_x86_64'
|
||||
- os: ubuntu-latest
|
||||
python: '3.12'
|
||||
container: 'quay.io/pypa/manylinux2014_x86_64'
|
||||
- os: ubuntu-latest
|
||||
python: '3.13'
|
||||
container: 'quay.io/pypa/manylinux2014_x86_64'
|
||||
- os: macos-latest
|
||||
python: '3.9'
|
||||
container: ''
|
||||
- os: macos-latest
|
||||
python: '3.10'
|
||||
container: ''
|
||||
- os: macos-latest
|
||||
python: '3.11'
|
||||
container: ''
|
||||
- os: macos-latest
|
||||
python: '3.12'
|
||||
container: ''
|
||||
- os: macos-latest
|
||||
python: '3.13'
|
||||
container: ''
|
||||
runs-on: ${{ matrix.os }}
|
||||
container: ${{ matrix.container }}
|
||||
|
||||
steps:
|
||||
# For manylinux2014 compatibility, we'll handle checkout differently
|
||||
- uses: actions/checkout@v4
|
||||
if: matrix.container == ''
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
submodules: recursive
|
||||
|
||||
# Manual checkout for containers to avoid Node.js compatibility issues
|
||||
- name: Manual checkout in container
|
||||
if: matrix.container != ''
|
||||
run: |
|
||||
# Install git if not available
|
||||
yum install -y git || true
|
||||
|
||||
# Configure git to handle the directory ownership issue
|
||||
git config --global --add safe.directory ${GITHUB_WORKSPACE}
|
||||
git config --global --add safe.directory /__w/LEANN/LEANN
|
||||
git config --global --add safe.directory /github/workspace
|
||||
git config --global --add safe.directory $(pwd)
|
||||
|
||||
# Clone the repository manually in the container
|
||||
git init
|
||||
git remote add origin https://github.com/${GITHUB_REPOSITORY}.git
|
||||
|
||||
# Fetch the appropriate ref
|
||||
if [ -n "${{ inputs.ref }}" ]; then
|
||||
git fetch --depth=1 origin ${{ inputs.ref }}
|
||||
else
|
||||
git fetch --depth=1 origin ${GITHUB_SHA}
|
||||
fi
|
||||
git checkout FETCH_HEAD
|
||||
|
||||
# Initialize submodules
|
||||
git submodule update --init --recursive
|
||||
|
||||
- name: Setup Python (macOS and regular Ubuntu)
|
||||
if: matrix.container == ''
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python }}
|
||||
|
||||
- name: Setup Python (manylinux container)
|
||||
if: matrix.container != ''
|
||||
run: |
|
||||
# Use the pre-installed Python version in manylinux container
|
||||
# Convert Python version format (3.9 -> 39, 3.10 -> 310, etc.)
|
||||
PY_VER=$(echo "${{ matrix.python }}" | sed 's/\.//g')
|
||||
/opt/python/cp${PY_VER}-*/bin/python -m pip install --upgrade pip
|
||||
# Create symlinks for convenience
|
||||
ln -sf /opt/python/cp${PY_VER}-*/bin/python /usr/local/bin/python
|
||||
ln -sf /opt/python/cp${PY_VER}-*/bin/pip /usr/local/bin/pip
|
||||
|
||||
- name: Install uv (macOS and regular Ubuntu)
|
||||
if: matrix.container == ''
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v4
|
||||
|
||||
- name: Install uv (manylinux container)
|
||||
if: matrix.container != ''
|
||||
run: |
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
|
||||
|
||||
- name: Install system dependencies (Ubuntu - regular)
|
||||
if: runner.os == 'Linux' && matrix.container == ''
|
||||
- name: Install system dependencies (Ubuntu)
|
||||
if: runner.os == 'Linux'
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \
|
||||
@@ -126,64 +61,6 @@ jobs:
|
||||
echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV
|
||||
echo "LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH" >> $GITHUB_ENV
|
||||
|
||||
- name: Install system dependencies (manylinux container)
|
||||
if: runner.os == 'Linux' && matrix.container != ''
|
||||
run: |
|
||||
# manylinux2014 uses yum instead of apt
|
||||
# Update yum cache first
|
||||
yum clean all
|
||||
yum makecache
|
||||
|
||||
# Install EPEL repository
|
||||
yum install -y epel-release || true
|
||||
|
||||
# Update cache again after EPEL
|
||||
yum makecache || true
|
||||
|
||||
# Install development packages
|
||||
# Note: Some packages might have different names in CentOS 7
|
||||
yum install -y \
|
||||
gcc-c++ \
|
||||
boost-devel \
|
||||
protobuf-compiler \
|
||||
protobuf-devel \
|
||||
zeromq-devel \
|
||||
pkgconfig \
|
||||
openblas-devel \
|
||||
cmake || {
|
||||
echo "Some packages failed to install, trying alternatives..."
|
||||
# Try alternative package names
|
||||
yum install -y libzmq3-devel || true
|
||||
yum install -y libzmq-devel || true
|
||||
}
|
||||
|
||||
# Install optional packages that might not be available
|
||||
yum install -y libaio-devel || echo "libaio-devel not available, continuing..."
|
||||
|
||||
# Verify zmq installation and create pkg-config file if needed
|
||||
if [ ! -f /usr/lib64/pkgconfig/libzmq.pc ] && [ ! -f /usr/lib/pkgconfig/libzmq.pc ]; then
|
||||
echo "Creating libzmq.pc file..."
|
||||
mkdir -p /usr/lib64/pkgconfig
|
||||
cat > /usr/lib64/pkgconfig/libzmq.pc << 'EOF'
|
||||
prefix=/usr
|
||||
exec_prefix=${prefix}
|
||||
libdir=${exec_prefix}/lib64
|
||||
includedir=${prefix}/include
|
||||
|
||||
Name: libzmq
|
||||
Description: ZeroMQ library
|
||||
Version: 4.1.4
|
||||
Libs: -L${libdir} -lzmq
|
||||
Cflags: -I${includedir}
|
||||
EOF
|
||||
fi
|
||||
|
||||
# Update PKG_CONFIG_PATH
|
||||
echo "PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH" >> $GITHUB_ENV
|
||||
|
||||
# Build tools are pre-installed in manylinux
|
||||
# MKL is more complex in container, skip for now and use OpenBLAS
|
||||
|
||||
- name: Install system dependencies (macOS)
|
||||
if: runner.os == 'macOS'
|
||||
run: |
|
||||
@@ -191,65 +68,44 @@ jobs:
|
||||
|
||||
- name: Install build dependencies
|
||||
run: |
|
||||
if [[ -n "${{ matrix.container }}" ]]; then
|
||||
# In manylinux container, use regular pip
|
||||
pip install scikit-build-core numpy swig Cython pybind11 auditwheel
|
||||
uv pip install --system scikit-build-core numpy swig Cython pybind11
|
||||
if [[ "$RUNNER_OS" == "Linux" ]]; then
|
||||
uv pip install --system auditwheel
|
||||
else
|
||||
# Regular environment, use uv
|
||||
uv pip install --system scikit-build-core numpy swig Cython pybind11
|
||||
if [[ "$RUNNER_OS" == "Linux" ]]; then
|
||||
uv pip install --system auditwheel
|
||||
else
|
||||
uv pip install --system delocate
|
||||
fi
|
||||
uv pip install --system delocate
|
||||
fi
|
||||
|
||||
- name: Build packages
|
||||
run: |
|
||||
# Choose build command based on environment
|
||||
if [[ -n "${{ matrix.container }}" ]]; then
|
||||
BUILD_CMD="pip wheel . --no-deps -w dist"
|
||||
else
|
||||
BUILD_CMD="uv build --wheel --python python"
|
||||
fi
|
||||
|
||||
# Build core (platform independent)
|
||||
if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
|
||||
cd packages/leann-core
|
||||
if [[ -n "${{ matrix.container }}" ]]; then
|
||||
pip wheel . --no-deps -w dist
|
||||
else
|
||||
uv build
|
||||
fi
|
||||
uv build
|
||||
cd ../..
|
||||
fi
|
||||
|
||||
# Build HNSW backend
|
||||
cd packages/leann-backend-hnsw
|
||||
if [ "${{ matrix.os }}" == "macos-latest" ]; then
|
||||
CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ $BUILD_CMD
|
||||
CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build --wheel --python python
|
||||
else
|
||||
eval $BUILD_CMD
|
||||
uv build --wheel --python python
|
||||
fi
|
||||
cd ../..
|
||||
|
||||
# Build DiskANN backend
|
||||
cd packages/leann-backend-diskann
|
||||
if [ "${{ matrix.os }}" == "macos-latest" ]; then
|
||||
CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ $BUILD_CMD
|
||||
CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build --wheel --python python
|
||||
else
|
||||
eval $BUILD_CMD
|
||||
uv build --wheel --python python
|
||||
fi
|
||||
cd ../..
|
||||
|
||||
# Build meta package (platform independent)
|
||||
if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
|
||||
cd packages/leann
|
||||
if [[ -n "${{ matrix.container }}" ]]; then
|
||||
pip wheel . --no-deps -w dist
|
||||
else
|
||||
uv build
|
||||
fi
|
||||
uv build
|
||||
cd ../..
|
||||
fi
|
||||
|
||||
@@ -259,9 +115,6 @@ jobs:
|
||||
# Repair HNSW wheel
|
||||
cd packages/leann-backend-hnsw
|
||||
if [ -d dist ]; then
|
||||
# Show what platform auditwheel will use
|
||||
auditwheel show dist/*.whl || true
|
||||
# Let auditwheel auto-detect the appropriate manylinux tag
|
||||
auditwheel repair dist/*.whl -w dist_repaired
|
||||
rm -rf dist
|
||||
mv dist_repaired dist
|
||||
@@ -271,9 +124,6 @@ jobs:
|
||||
# Repair DiskANN wheel
|
||||
cd packages/leann-backend-diskann
|
||||
if [ -d dist ]; then
|
||||
# Show what platform auditwheel will use
|
||||
auditwheel show dist/*.whl || true
|
||||
# Let auditwheel auto-detect the appropriate manylinux tag
|
||||
auditwheel repair dist/*.whl -w dist_repaired
|
||||
rm -rf dist
|
||||
mv dist_repaired dist
|
||||
@@ -309,5 +159,5 @@ jobs:
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: packages-${{ matrix.os }}-py${{ matrix.python }}${{ matrix.container && '-manylinux' || '' }}
|
||||
name: packages-${{ matrix.os }}-py${{ matrix.python }}
|
||||
path: packages/*/dist/
|
||||
22
.github/workflows/release-manual.yml
vendored
22
.github/workflows/release-manual.yml
vendored
@@ -7,11 +7,6 @@ on:
|
||||
description: 'Version to release (e.g., 0.1.2)'
|
||||
required: true
|
||||
type: string
|
||||
use_cibuildwheel:
|
||||
description: 'Use cibuildwheel for better compatibility (recommended for Colab)'
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
jobs:
|
||||
update-version:
|
||||
@@ -47,26 +42,17 @@ jobs:
|
||||
echo "commit-sha=$COMMIT_SHA" >> $GITHUB_OUTPUT
|
||||
echo "✅ Pushed version update: $COMMIT_SHA"
|
||||
|
||||
build-packages-reusable:
|
||||
name: Build packages (Standard)
|
||||
build-packages:
|
||||
name: Build packages
|
||||
needs: update-version
|
||||
if: ${{ !inputs.use_cibuildwheel }}
|
||||
uses: ./.github/workflows/build-reusable.yml
|
||||
with:
|
||||
ref: ${{ needs.update-version.outputs.commit-sha }}
|
||||
|
||||
build-packages-cibuildwheel:
|
||||
name: Build packages (cibuildwheel)
|
||||
needs: update-version
|
||||
if: ${{ inputs.use_cibuildwheel }}
|
||||
uses: ./.github/workflows/build-cibuildwheel.yml
|
||||
with:
|
||||
ref: ${{ needs.update-version.outputs.commit-sha }}
|
||||
|
||||
publish:
|
||||
name: Publish and Release
|
||||
needs: [update-version, build-packages-reusable, build-packages-cibuildwheel]
|
||||
if: always() && needs.update-version.result == 'success' && (needs.build-packages-reusable.result == 'success' || needs.build-packages-cibuildwheel.result == 'success')
|
||||
needs: [update-version, build-packages]
|
||||
if: always() && needs.update-version.result == 'success' && needs.build-packages.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
Reference in New Issue
Block a user