From 4aef1d814e04858edac0a85d78a1ddc78bf2b0fd Mon Sep 17 00:00:00 2001 From: Andy Lee Date: Fri, 25 Jul 2025 13:46:51 -0700 Subject: [PATCH] revert: simplify build system by removing manylinux/cibuildwheel - Revert to simple Ubuntu 22.04 builds that should work with Colab - Remove all manylinux container complexity - Colab runs on Ubuntu 22.04, so direct builds should be compatible - Restore build-reusable.yml to v0.1.5 version - Remove cibuildwheel option from release workflow This should fix the overcomplicated build issues while maintaining Colab compatibility through direct Ubuntu 22.04 builds. --- .github/workflows/build-reusable.yml | 180 +++------------------------ .github/workflows/release-manual.yml | 22 +--- 2 files changed, 19 insertions(+), 183 deletions(-) diff --git a/.github/workflows/build-reusable.yml b/.github/workflows/build-reusable.yml index 8ab5bb2..c2e7958 100644 --- a/.github/workflows/build-reusable.yml +++ b/.github/workflows/build-reusable.yml @@ -13,107 +13,42 @@ jobs: build: name: Build ${{ matrix.os }} Python ${{ matrix.python }} strategy: - fail-fast: false matrix: include: - os: ubuntu-latest python: '3.9' - container: 'quay.io/pypa/manylinux2014_x86_64' - os: ubuntu-latest python: '3.10' - container: 'quay.io/pypa/manylinux2014_x86_64' - os: ubuntu-latest python: '3.11' - container: 'quay.io/pypa/manylinux2014_x86_64' - os: ubuntu-latest python: '3.12' - container: 'quay.io/pypa/manylinux2014_x86_64' - - os: ubuntu-latest - python: '3.13' - container: 'quay.io/pypa/manylinux2014_x86_64' - os: macos-latest python: '3.9' - container: '' - os: macos-latest python: '3.10' - container: '' - os: macos-latest python: '3.11' - container: '' - os: macos-latest python: '3.12' - container: '' - - os: macos-latest - python: '3.13' - container: '' runs-on: ${{ matrix.os }} - container: ${{ matrix.container }} steps: - # For manylinux2014 compatibility, we'll handle checkout differently - uses: actions/checkout@v4 - if: matrix.container == '' with: ref: ${{ inputs.ref }} submodules: recursive - # Manual checkout for containers to avoid Node.js compatibility issues - - name: Manual checkout in container - if: matrix.container != '' - run: | - # Install git if not available - yum install -y git || true - - # Configure git to handle the directory ownership issue - git config --global --add safe.directory ${GITHUB_WORKSPACE} - git config --global --add safe.directory /__w/LEANN/LEANN - git config --global --add safe.directory /github/workspace - git config --global --add safe.directory $(pwd) - - # Clone the repository manually in the container - git init - git remote add origin https://github.com/${GITHUB_REPOSITORY}.git - - # Fetch the appropriate ref - if [ -n "${{ inputs.ref }}" ]; then - git fetch --depth=1 origin ${{ inputs.ref }} - else - git fetch --depth=1 origin ${GITHUB_SHA} - fi - git checkout FETCH_HEAD - - # Initialize submodules - git submodule update --init --recursive - - - name: Setup Python (macOS and regular Ubuntu) - if: matrix.container == '' + - name: Setup Python uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - - name: Setup Python (manylinux container) - if: matrix.container != '' - run: | - # Use the pre-installed Python version in manylinux container - # Convert Python version format (3.9 -> 39, 3.10 -> 310, etc.) - PY_VER=$(echo "${{ matrix.python }}" | sed 's/\.//g') - /opt/python/cp${PY_VER}-*/bin/python -m pip install --upgrade pip - # Create symlinks for convenience - ln -sf /opt/python/cp${PY_VER}-*/bin/python /usr/local/bin/python - ln -sf /opt/python/cp${PY_VER}-*/bin/pip /usr/local/bin/pip - - - name: Install uv (macOS and regular Ubuntu) - if: matrix.container == '' + - name: Install uv uses: astral-sh/setup-uv@v4 - - name: Install uv (manylinux container) - if: matrix.container != '' - run: | - curl -LsSf https://astral.sh/uv/install.sh | sh - echo "$HOME/.cargo/bin" >> $GITHUB_PATH - - - name: Install system dependencies (Ubuntu - regular) - if: runner.os == 'Linux' && matrix.container == '' + - name: Install system dependencies (Ubuntu) + if: runner.os == 'Linux' run: | sudo apt-get update sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \ @@ -126,64 +61,6 @@ jobs: echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV echo "LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH" >> $GITHUB_ENV - - name: Install system dependencies (manylinux container) - if: runner.os == 'Linux' && matrix.container != '' - run: | - # manylinux2014 uses yum instead of apt - # Update yum cache first - yum clean all - yum makecache - - # Install EPEL repository - yum install -y epel-release || true - - # Update cache again after EPEL - yum makecache || true - - # Install development packages - # Note: Some packages might have different names in CentOS 7 - yum install -y \ - gcc-c++ \ - boost-devel \ - protobuf-compiler \ - protobuf-devel \ - zeromq-devel \ - pkgconfig \ - openblas-devel \ - cmake || { - echo "Some packages failed to install, trying alternatives..." - # Try alternative package names - yum install -y libzmq3-devel || true - yum install -y libzmq-devel || true - } - - # Install optional packages that might not be available - yum install -y libaio-devel || echo "libaio-devel not available, continuing..." - - # Verify zmq installation and create pkg-config file if needed - if [ ! -f /usr/lib64/pkgconfig/libzmq.pc ] && [ ! -f /usr/lib/pkgconfig/libzmq.pc ]; then - echo "Creating libzmq.pc file..." - mkdir -p /usr/lib64/pkgconfig - cat > /usr/lib64/pkgconfig/libzmq.pc << 'EOF' - prefix=/usr - exec_prefix=${prefix} - libdir=${exec_prefix}/lib64 - includedir=${prefix}/include - - Name: libzmq - Description: ZeroMQ library - Version: 4.1.4 - Libs: -L${libdir} -lzmq - Cflags: -I${includedir} - EOF - fi - - # Update PKG_CONFIG_PATH - echo "PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH" >> $GITHUB_ENV - - # Build tools are pre-installed in manylinux - # MKL is more complex in container, skip for now and use OpenBLAS - - name: Install system dependencies (macOS) if: runner.os == 'macOS' run: | @@ -191,65 +68,44 @@ jobs: - name: Install build dependencies run: | - if [[ -n "${{ matrix.container }}" ]]; then - # In manylinux container, use regular pip - pip install scikit-build-core numpy swig Cython pybind11 auditwheel + uv pip install --system scikit-build-core numpy swig Cython pybind11 + if [[ "$RUNNER_OS" == "Linux" ]]; then + uv pip install --system auditwheel else - # Regular environment, use uv - uv pip install --system scikit-build-core numpy swig Cython pybind11 - if [[ "$RUNNER_OS" == "Linux" ]]; then - uv pip install --system auditwheel - else - uv pip install --system delocate - fi + uv pip install --system delocate fi - name: Build packages run: | - # Choose build command based on environment - if [[ -n "${{ matrix.container }}" ]]; then - BUILD_CMD="pip wheel . --no-deps -w dist" - else - BUILD_CMD="uv build --wheel --python python" - fi - # Build core (platform independent) if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then cd packages/leann-core - if [[ -n "${{ matrix.container }}" ]]; then - pip wheel . --no-deps -w dist - else - uv build - fi + uv build cd ../.. fi # Build HNSW backend cd packages/leann-backend-hnsw if [ "${{ matrix.os }}" == "macos-latest" ]; then - CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ $BUILD_CMD + CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build --wheel --python python else - eval $BUILD_CMD + uv build --wheel --python python fi cd ../.. # Build DiskANN backend cd packages/leann-backend-diskann if [ "${{ matrix.os }}" == "macos-latest" ]; then - CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ $BUILD_CMD + CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build --wheel --python python else - eval $BUILD_CMD + uv build --wheel --python python fi cd ../.. # Build meta package (platform independent) if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then cd packages/leann - if [[ -n "${{ matrix.container }}" ]]; then - pip wheel . --no-deps -w dist - else - uv build - fi + uv build cd ../.. fi @@ -259,9 +115,6 @@ jobs: # Repair HNSW wheel cd packages/leann-backend-hnsw if [ -d dist ]; then - # Show what platform auditwheel will use - auditwheel show dist/*.whl || true - # Let auditwheel auto-detect the appropriate manylinux tag auditwheel repair dist/*.whl -w dist_repaired rm -rf dist mv dist_repaired dist @@ -271,9 +124,6 @@ jobs: # Repair DiskANN wheel cd packages/leann-backend-diskann if [ -d dist ]; then - # Show what platform auditwheel will use - auditwheel show dist/*.whl || true - # Let auditwheel auto-detect the appropriate manylinux tag auditwheel repair dist/*.whl -w dist_repaired rm -rf dist mv dist_repaired dist @@ -309,5 +159,5 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: packages-${{ matrix.os }}-py${{ matrix.python }}${{ matrix.container && '-manylinux' || '' }} + name: packages-${{ matrix.os }}-py${{ matrix.python }} path: packages/*/dist/ \ No newline at end of file diff --git a/.github/workflows/release-manual.yml b/.github/workflows/release-manual.yml index 9a61b86..027489d 100644 --- a/.github/workflows/release-manual.yml +++ b/.github/workflows/release-manual.yml @@ -7,11 +7,6 @@ on: description: 'Version to release (e.g., 0.1.2)' required: true type: string - use_cibuildwheel: - description: 'Use cibuildwheel for better compatibility (recommended for Colab)' - required: false - type: boolean - default: false jobs: update-version: @@ -47,26 +42,17 @@ jobs: echo "commit-sha=$COMMIT_SHA" >> $GITHUB_OUTPUT echo "✅ Pushed version update: $COMMIT_SHA" - build-packages-reusable: - name: Build packages (Standard) + build-packages: + name: Build packages needs: update-version - if: ${{ !inputs.use_cibuildwheel }} uses: ./.github/workflows/build-reusable.yml with: ref: ${{ needs.update-version.outputs.commit-sha }} - - build-packages-cibuildwheel: - name: Build packages (cibuildwheel) - needs: update-version - if: ${{ inputs.use_cibuildwheel }} - uses: ./.github/workflows/build-cibuildwheel.yml - with: - ref: ${{ needs.update-version.outputs.commit-sha }} publish: name: Publish and Release - needs: [update-version, build-packages-reusable, build-packages-cibuildwheel] - if: always() && needs.update-version.result == 'success' && (needs.build-packages-reusable.result == 'success' || needs.build-packages-cibuildwheel.result == 'success') + needs: [update-version, build-packages] + if: always() && needs.update-version.result == 'success' && needs.build-packages.result == 'success' runs-on: ubuntu-latest permissions: contents: write