Merge branch 'main' of https://github.com/yichuan-w/LEANN

.github/workflows/build-cibuildwheel.yml

@@ -0,0 +1,144 @@
+name: Build with cibuildwheel
+
+on:
+  workflow_call:
+    inputs:
+      ref:
+        description: 'Git ref to build'
+        required: false
+        type: string
+        default: ''
+
+jobs:
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+          submodules: recursive
+      
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'  # Version for building pure Python packages
+      
+      # Build each package separately in our monorepo
+      - name: Build pure Python packages (leann-core, leann)
+        if: matrix.os == 'ubuntu-latest'  # Only build once, they're platform-independent
+        run: |
+          # Install build tools
+          python -m pip install --upgrade pip build
+          
+          # Build pure Python packages
+          python -m build packages/leann-core --outdir wheelhouse/
+          python -m build packages/leann --outdir wheelhouse/
+      
+      - name: Build leann-backend-hnsw wheels  
+        uses: pypa/cibuildwheel@v2.16.2
+        with:
+          package-dir: packages/leann-backend-hnsw
+          output-dir: wheelhouse
+        env:
+          CIBW_BUILD: cp39-* cp310-* cp311-* cp312-*
+          CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+          CIBW_SKIP: "*-win32 *-manylinux_i686 pp*"
+          
+          CIBW_BEFORE_ALL_LINUX: |
+            yum clean all && yum makecache
+            yum install -y epel-release || true
+            yum makecache || true
+            # Install system dependencies
+            yum install -y \
+              gcc-c++ \
+              boost-devel \
+              protobuf-compiler \
+              protobuf-devel \
+              zeromq-devel \
+              pkgconfig \
+              openblas-devel \
+              cmake || echo "Some packages failed, continuing..."
+            
+            # Verify zmq installation and create pkg-config file if needed
+            if [ ! -f /usr/lib64/pkgconfig/libzmq.pc ] && [ ! -f /usr/lib/pkgconfig/libzmq.pc ]; then
+              echo "Creating libzmq.pc file..."
+              mkdir -p /usr/lib64/pkgconfig
+              cat > /usr/lib64/pkgconfig/libzmq.pc << 'EOF'
+            prefix=/usr
+            exec_prefix=${prefix}
+            libdir=${exec_prefix}/lib64
+            includedir=${prefix}/include
+            
+            Name: libzmq
+            Description: ZeroMQ library
+            Version: 4.1.4
+            Libs: -L${libdir} -lzmq
+            Cflags: -I${includedir}
+            EOF
+            fi
+          
+          CIBW_BEFORE_ALL_MACOS: |
+            brew install llvm libomp boost protobuf zeromq
+          
+          CIBW_ENVIRONMENT_LINUX: |
+            PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH
+          
+          CIBW_ENVIRONMENT_MACOS: |
+            CC=$(brew --prefix llvm)/bin/clang
+            CXX=$(brew --prefix llvm)/bin/clang++
+          
+          CIBW_TEST_REQUIRES: leann-core numpy pyzmq msgpack
+          CIBW_TEST_COMMAND: python -c "import leann_backend_hnsw"
+      
+      - name: Build leann-backend-diskann wheels
+        uses: pypa/cibuildwheel@v2.16.2
+        with:
+          package-dir: packages/leann-backend-diskann
+          output-dir: wheelhouse
+        env:
+          CIBW_BUILD: cp39-* cp310-* cp311-* cp312-*
+          CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+          CIBW_SKIP: "*-win32 *-manylinux_i686 pp*"
+          
+          CIBW_BEFORE_ALL_LINUX: |
+            yum clean all && yum makecache
+            yum install -y epel-release || true
+            yum makecache || true
+            # Install system dependencies for DiskANN
+            yum install -y \
+              gcc-c++ \
+              protobuf-compiler \
+              protobuf-devel \
+              openblas-devel \
+              pkgconfig \
+              cmake || echo "Some packages failed, continuing..."
+            yum install -y libaio-devel || echo "libaio-devel not available, continuing..."
+          
+          CIBW_BEFORE_ALL_MACOS: |
+            brew install llvm libomp protobuf
+          
+          CIBW_ENVIRONMENT_LINUX: |
+            PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH
+          
+          CIBW_ENVIRONMENT_MACOS: |
+            CC=$(brew --prefix llvm)/bin/clang
+            CXX=$(brew --prefix llvm)/bin/clang++
+          
+          CIBW_TEST_REQUIRES: leann-core numpy
+          CIBW_TEST_COMMAND: python -c "import leann_backend_diskann"
+      
+      - name: List built packages
+        run: |
+          echo "📦 Built packages:"
+          ls -la wheelhouse/
+      
+      - uses: actions/upload-artifact@v4
+        with:
+          name: cibw-wheels-${{ matrix.os }}
+          path: ./wheelhouse/*.whl 

.github/workflows/build-reusable.yml

@@ -13,6 +13,7 @@ jobs:
   build:
     name: Build ${{ matrix.os }} Python ${{ matrix.python }}
     strategy:
+      fail-fast: false
       matrix:
         include:
           - os: ubuntu-latest
@@ -43,15 +44,44 @@ jobs:
     container: ${{ matrix.container }}
     
     steps:
-      # Use v3 for manylinux2014 compatibility (Node.js 16 instead of 20)
+      # For manylinux2014 compatibility, we'll handle checkout differently
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        if: matrix.container == ''
         with:
           ref: ${{ inputs.ref }}
           submodules: recursive
       
+      # Manual checkout for containers to avoid Node.js compatibility issues
+      - name: Manual checkout in container
+        if: matrix.container != ''
+        run: |
+          # Install git if not available
+          yum install -y git || true
+          
+          # Configure git to handle the directory ownership issue
+          git config --global --add safe.directory ${GITHUB_WORKSPACE}
+          git config --global --add safe.directory /__w/LEANN/LEANN
+          git config --global --add safe.directory /github/workspace
+          git config --global --add safe.directory $(pwd)
+          
+          # Clone the repository manually in the container
+          git init
+          git remote add origin https://github.com/${GITHUB_REPOSITORY}.git
+          
+          # Fetch the appropriate ref
+          if [ -n "${{ inputs.ref }}" ]; then
+            git fetch --depth=1 origin ${{ inputs.ref }}
+          else
+            git fetch --depth=1 origin ${GITHUB_SHA}
+          fi
+          git checkout FETCH_HEAD
+          
+          # Initialize submodules
+          git submodule update --init --recursive
+      
       - name: Setup Python (macOS and regular Ubuntu)
         if: matrix.container == ''
-        uses: actions/setup-python@v4  # v4 for better compatibility
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
       
@@ -94,9 +124,56 @@ jobs:
         if: runner.os == 'Linux' && matrix.container != ''
         run: |
           # manylinux2014 uses yum instead of apt
-          yum install -y epel-release
+          # Update yum cache first
-          yum install -y boost-devel protobuf-compiler zeromq-devel \
+          yum clean all
-            pkg-config openblas-devel libaio-devel protobuf-devel
+          yum makecache
+          
+          # Install EPEL repository
+          yum install -y epel-release || true
+          
+          # Update cache again after EPEL
+          yum makecache || true
+          
+          # Install development packages
+          # Note: Some packages might have different names in CentOS 7
+          yum install -y \
+            gcc-c++ \
+            boost-devel \
+            protobuf-compiler \
+            protobuf-devel \
+            zeromq-devel \
+            pkgconfig \
+            openblas-devel \
+            cmake || {
+              echo "Some packages failed to install, trying alternatives..."
+              # Try alternative package names
+              yum install -y libzmq3-devel || true
+              yum install -y libzmq-devel || true
+          }
+          
+          # Install optional packages that might not be available
+          yum install -y libaio-devel || echo "libaio-devel not available, continuing..."
+          
+          # Verify zmq installation and create pkg-config file if needed
+          if [ ! -f /usr/lib64/pkgconfig/libzmq.pc ] && [ ! -f /usr/lib/pkgconfig/libzmq.pc ]; then
+            echo "Creating libzmq.pc file..."
+            mkdir -p /usr/lib64/pkgconfig
+            cat > /usr/lib64/pkgconfig/libzmq.pc << 'EOF'
+          prefix=/usr
+          exec_prefix=${prefix}
+          libdir=${exec_prefix}/lib64
+          includedir=${prefix}/include
+          
+          Name: libzmq
+          Description: ZeroMQ library
+          Version: 4.1.4
+          Libs: -L${libdir} -lzmq
+          Cflags: -I${includedir}
+          EOF
+          fi
+          
+          # Update PKG_CONFIG_PATH
+          echo "PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH" >> $GITHUB_ENV
           
           # Build tools are pre-installed in manylinux
           # MKL is more complex in container, skip for now and use OpenBLAS
@@ -224,7 +301,7 @@ jobs:
           find packages/*/dist -name "*.whl" -o -name "*.tar.gz" | sort
       
       - name: Upload artifacts
-        uses: actions/upload-artifact@v3  # v3 for manylinux2014 compatibility
+        uses: actions/upload-artifact@v4
         with:
-          name: packages-${{ matrix.os }}-py${{ matrix.python }}
+          name: packages-${{ matrix.os }}-py${{ matrix.python }}${{ matrix.container && '-manylinux' || '' }}
           path: packages/*/dist/ 

.github/workflows/ci-cibuildwheel.yml

@@ -0,0 +1,12 @@
+name: CI - cibuildwheel (Test)
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  workflow_dispatch:  # Allow manual triggering
+
+jobs:
+  build:
+    uses: ./.github/workflows/build-cibuildwheel.yml 

.github/workflows/release-manual.yml

@@ -7,6 +7,11 @@ on:
         description: 'Version to release (e.g., 0.1.2)'
         required: true
         type: string
+      use_cibuildwheel:
+        description: 'Use cibuildwheel for better compatibility (recommended for Colab)'
+        required: false
+        type: boolean
+        default: false
 
 jobs:
   update-version:
@@ -18,7 +23,7 @@ jobs:
       commit-sha: ${{ steps.push.outputs.commit-sha }}
     
     steps:
-      - uses: actions/checkout@v3  # Compatibility with manylinux2014
+      - uses: actions/checkout@v4
       
       - name: Validate version
         run: |
@@ -42,27 +47,37 @@ jobs:
           echo "commit-sha=$COMMIT_SHA" >> $GITHUB_OUTPUT
           echo "✅ Pushed version update: $COMMIT_SHA"
 
-  build-packages:
+  build-packages-reusable:
-    name: Build packages
+    name: Build packages (Standard)
     needs: update-version
+    if: ${{ !inputs.use_cibuildwheel }}
     uses: ./.github/workflows/build-reusable.yml
     with:
       ref: ${{ needs.update-version.outputs.commit-sha }}
   
+  build-packages-cibuildwheel:
+    name: Build packages (cibuildwheel)
+    needs: update-version
+    if: ${{ inputs.use_cibuildwheel }}
+    uses: ./.github/workflows/build-cibuildwheel.yml
+    with:
+      ref: ${{ needs.update-version.outputs.commit-sha }}
+
   publish:
     name: Publish and Release
-    needs: build-packages
+    needs: [update-version, build-packages-reusable, build-packages-cibuildwheel]
+    if: always() && needs.update-version.result == 'success' && (needs.build-packages-reusable.result == 'success' || needs.build-packages-cibuildwheel.result == 'success')
     runs-on: ubuntu-latest
     permissions:
       contents: write
     
     steps:
-      - uses: actions/checkout@v3  # Consistency with build workflow
+      - uses: actions/checkout@v4
         with:
           ref: ${{ needs.update-version.outputs.commit-sha }}
       
       - name: Download all artifacts
-        uses: actions/download-artifact@v3  # Match upload-artifact version
+        uses: actions/download-artifact@v4
         with:
           path: dist-artifacts