diff --git a/.github/workflows/build-cibuildwheel.yml b/.github/workflows/build-cibuildwheel.yml new file mode 100644 index 0000000..4cb4a07 --- /dev/null +++ b/.github/workflows/build-cibuildwheel.yml @@ -0,0 +1,144 @@ +name: Build with cibuildwheel + +on: + workflow_call: + inputs: + ref: + description: 'Git ref to build' + required: false + type: string + default: '' + +jobs: + build_wheels: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + submodules: recursive + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' # Version for building pure Python packages + + # Build each package separately in our monorepo + - name: Build pure Python packages (leann-core, leann) + if: matrix.os == 'ubuntu-latest' # Only build once, they're platform-independent + run: | + # Install build tools + python -m pip install --upgrade pip build + + # Build pure Python packages + python -m build packages/leann-core --outdir wheelhouse/ + python -m build packages/leann --outdir wheelhouse/ + + - name: Build leann-backend-hnsw wheels + uses: pypa/cibuildwheel@v2.16.2 + with: + package-dir: packages/leann-backend-hnsw + output-dir: wheelhouse + env: + CIBW_BUILD: cp39-* cp310-* cp311-* cp312-* + CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 + CIBW_SKIP: "*-win32 *-manylinux_i686 pp*" + + CIBW_BEFORE_ALL_LINUX: | + yum clean all && yum makecache + yum install -y epel-release || true + yum makecache || true + # Install system dependencies + yum install -y \ + gcc-c++ \ + boost-devel \ + protobuf-compiler \ + protobuf-devel \ + zeromq-devel \ + pkgconfig \ + openblas-devel \ + cmake || echo "Some packages failed, continuing..." + + # Verify zmq installation and create pkg-config file if needed + if [ ! -f /usr/lib64/pkgconfig/libzmq.pc ] && [ ! -f /usr/lib/pkgconfig/libzmq.pc ]; then + echo "Creating libzmq.pc file..." + mkdir -p /usr/lib64/pkgconfig + cat > /usr/lib64/pkgconfig/libzmq.pc << 'EOF' + prefix=/usr + exec_prefix=${prefix} + libdir=${exec_prefix}/lib64 + includedir=${prefix}/include + + Name: libzmq + Description: ZeroMQ library + Version: 4.1.4 + Libs: -L${libdir} -lzmq + Cflags: -I${includedir} + EOF + fi + + CIBW_BEFORE_ALL_MACOS: | + brew install llvm libomp boost protobuf zeromq + + CIBW_ENVIRONMENT_LINUX: | + PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH + + CIBW_ENVIRONMENT_MACOS: | + CC=$(brew --prefix llvm)/bin/clang + CXX=$(brew --prefix llvm)/bin/clang++ + + CIBW_TEST_REQUIRES: leann-core numpy pyzmq msgpack + CIBW_TEST_COMMAND: python -c "import leann_backend_hnsw" + + - name: Build leann-backend-diskann wheels + uses: pypa/cibuildwheel@v2.16.2 + with: + package-dir: packages/leann-backend-diskann + output-dir: wheelhouse + env: + CIBW_BUILD: cp39-* cp310-* cp311-* cp312-* + CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 + CIBW_SKIP: "*-win32 *-manylinux_i686 pp*" + + CIBW_BEFORE_ALL_LINUX: | + yum clean all && yum makecache + yum install -y epel-release || true + yum makecache || true + # Install system dependencies for DiskANN + yum install -y \ + gcc-c++ \ + protobuf-compiler \ + protobuf-devel \ + openblas-devel \ + pkgconfig \ + cmake || echo "Some packages failed, continuing..." + yum install -y libaio-devel || echo "libaio-devel not available, continuing..." + + CIBW_BEFORE_ALL_MACOS: | + brew install llvm libomp protobuf + + CIBW_ENVIRONMENT_LINUX: | + PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH + + CIBW_ENVIRONMENT_MACOS: | + CC=$(brew --prefix llvm)/bin/clang + CXX=$(brew --prefix llvm)/bin/clang++ + + CIBW_TEST_REQUIRES: leann-core numpy + CIBW_TEST_COMMAND: python -c "import leann_backend_diskann" + + - name: List built packages + run: | + echo "📦 Built packages:" + ls -la wheelhouse/ + + - uses: actions/upload-artifact@v4 + with: + name: cibw-wheels-${{ matrix.os }} + path: ./wheelhouse/*.whl \ No newline at end of file diff --git a/.github/workflows/build-reusable.yml b/.github/workflows/build-reusable.yml index 77c79b0..8830172 100644 --- a/.github/workflows/build-reusable.yml +++ b/.github/workflows/build-reusable.yml @@ -13,6 +13,7 @@ jobs: build: name: Build ${{ matrix.os }} Python ${{ matrix.python }} strategy: + fail-fast: false matrix: include: - os: ubuntu-latest @@ -43,15 +44,44 @@ jobs: container: ${{ matrix.container }} steps: - # Use v3 for manylinux2014 compatibility (Node.js 16 instead of 20) - - uses: actions/checkout@v3 + # For manylinux2014 compatibility, we'll handle checkout differently + - uses: actions/checkout@v4 + if: matrix.container == '' with: ref: ${{ inputs.ref }} submodules: recursive + # Manual checkout for containers to avoid Node.js compatibility issues + - name: Manual checkout in container + if: matrix.container != '' + run: | + # Install git if not available + yum install -y git || true + + # Configure git to handle the directory ownership issue + git config --global --add safe.directory ${GITHUB_WORKSPACE} + git config --global --add safe.directory /__w/LEANN/LEANN + git config --global --add safe.directory /github/workspace + git config --global --add safe.directory $(pwd) + + # Clone the repository manually in the container + git init + git remote add origin https://github.com/${GITHUB_REPOSITORY}.git + + # Fetch the appropriate ref + if [ -n "${{ inputs.ref }}" ]; then + git fetch --depth=1 origin ${{ inputs.ref }} + else + git fetch --depth=1 origin ${GITHUB_SHA} + fi + git checkout FETCH_HEAD + + # Initialize submodules + git submodule update --init --recursive + - name: Setup Python (macOS and regular Ubuntu) if: matrix.container == '' - uses: actions/setup-python@v4 # v4 for better compatibility + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} @@ -94,9 +124,56 @@ jobs: if: runner.os == 'Linux' && matrix.container != '' run: | # manylinux2014 uses yum instead of apt - yum install -y epel-release - yum install -y boost-devel protobuf-compiler zeromq-devel \ - pkg-config openblas-devel libaio-devel protobuf-devel + # Update yum cache first + yum clean all + yum makecache + + # Install EPEL repository + yum install -y epel-release || true + + # Update cache again after EPEL + yum makecache || true + + # Install development packages + # Note: Some packages might have different names in CentOS 7 + yum install -y \ + gcc-c++ \ + boost-devel \ + protobuf-compiler \ + protobuf-devel \ + zeromq-devel \ + pkgconfig \ + openblas-devel \ + cmake || { + echo "Some packages failed to install, trying alternatives..." + # Try alternative package names + yum install -y libzmq3-devel || true + yum install -y libzmq-devel || true + } + + # Install optional packages that might not be available + yum install -y libaio-devel || echo "libaio-devel not available, continuing..." + + # Verify zmq installation and create pkg-config file if needed + if [ ! -f /usr/lib64/pkgconfig/libzmq.pc ] && [ ! -f /usr/lib/pkgconfig/libzmq.pc ]; then + echo "Creating libzmq.pc file..." + mkdir -p /usr/lib64/pkgconfig + cat > /usr/lib64/pkgconfig/libzmq.pc << 'EOF' + prefix=/usr + exec_prefix=${prefix} + libdir=${exec_prefix}/lib64 + includedir=${prefix}/include + + Name: libzmq + Description: ZeroMQ library + Version: 4.1.4 + Libs: -L${libdir} -lzmq + Cflags: -I${includedir} + EOF + fi + + # Update PKG_CONFIG_PATH + echo "PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH" >> $GITHUB_ENV # Build tools are pre-installed in manylinux # MKL is more complex in container, skip for now and use OpenBLAS @@ -224,7 +301,7 @@ jobs: find packages/*/dist -name "*.whl" -o -name "*.tar.gz" | sort - name: Upload artifacts - uses: actions/upload-artifact@v3 # v3 for manylinux2014 compatibility + uses: actions/upload-artifact@v4 with: - name: packages-${{ matrix.os }}-py${{ matrix.python }} + name: packages-${{ matrix.os }}-py${{ matrix.python }}${{ matrix.container && '-manylinux' || '' }} path: packages/*/dist/ \ No newline at end of file diff --git a/.github/workflows/ci-cibuildwheel.yml b/.github/workflows/ci-cibuildwheel.yml new file mode 100644 index 0000000..88f97d1 --- /dev/null +++ b/.github/workflows/ci-cibuildwheel.yml @@ -0,0 +1,12 @@ +name: CI - cibuildwheel (Test) + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + workflow_dispatch: # Allow manual triggering + +jobs: + build: + uses: ./.github/workflows/build-cibuildwheel.yml \ No newline at end of file diff --git a/.github/workflows/release-manual.yml b/.github/workflows/release-manual.yml index 6623f44..9a61b86 100644 --- a/.github/workflows/release-manual.yml +++ b/.github/workflows/release-manual.yml @@ -7,6 +7,11 @@ on: description: 'Version to release (e.g., 0.1.2)' required: true type: string + use_cibuildwheel: + description: 'Use cibuildwheel for better compatibility (recommended for Colab)' + required: false + type: boolean + default: false jobs: update-version: @@ -18,7 +23,7 @@ jobs: commit-sha: ${{ steps.push.outputs.commit-sha }} steps: - - uses: actions/checkout@v3 # Compatibility with manylinux2014 + - uses: actions/checkout@v4 - name: Validate version run: | @@ -42,27 +47,37 @@ jobs: echo "commit-sha=$COMMIT_SHA" >> $GITHUB_OUTPUT echo "✅ Pushed version update: $COMMIT_SHA" - build-packages: - name: Build packages + build-packages-reusable: + name: Build packages (Standard) needs: update-version + if: ${{ !inputs.use_cibuildwheel }} uses: ./.github/workflows/build-reusable.yml with: ref: ${{ needs.update-version.outputs.commit-sha }} + + build-packages-cibuildwheel: + name: Build packages (cibuildwheel) + needs: update-version + if: ${{ inputs.use_cibuildwheel }} + uses: ./.github/workflows/build-cibuildwheel.yml + with: + ref: ${{ needs.update-version.outputs.commit-sha }} publish: name: Publish and Release - needs: build-packages + needs: [update-version, build-packages-reusable, build-packages-cibuildwheel] + if: always() && needs.update-version.result == 'success' && (needs.build-packages-reusable.result == 'success' || needs.build-packages-cibuildwheel.result == 'success') runs-on: ubuntu-latest permissions: contents: write steps: - - uses: actions/checkout@v3 # Consistency with build workflow + - uses: actions/checkout@v4 with: ref: ${{ needs.update-version.outputs.commit-sha }} - name: Download all artifacts - uses: actions/download-artifact@v3 # Match upload-artifact version + uses: actions/download-artifact@v4 with: path: dist-artifacts diff --git a/packages/leann-backend-diskann/third_party/DiskANN b/packages/leann-backend-diskann/third_party/DiskANN index af2a264..25339b0 160000 --- a/packages/leann-backend-diskann/third_party/DiskANN +++ b/packages/leann-backend-diskann/third_party/DiskANN @@ -1 +1 @@ -Subproject commit af2a26481e65232b57b82d96e68833cdee9f7635 +Subproject commit 25339b03413b5067c25b6092ea3e0f77ef8515c8