feat: simplify build using cibuildwheel with standard configuration

- Add Python_FIND_VIRTUALENV hints to pyproject.toml for CMake
- Create standardized cibuildwheel workflow using manylinux_2_28
- Simplify system dependency installation
- Add global cibuildwheel configuration in root pyproject.toml
- Create streamlined test workflow for manylinux compatibility

.github/workflows/build-cibuildwheel.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        os: [ubuntu-latest, macos-latest, windows-latest]
         
     steps:
       - uses: actions/checkout@v4
@@ -27,118 +27,77 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.11'  # Version for building pure Python packages
+          python-version: '3.11'
       
-      # Build each package separately in our monorepo
+      # Build pure Python packages separately
       - name: Build pure Python packages (leann-core, leann)
-        if: matrix.os == 'ubuntu-latest'  # Only build once, they're platform-independent
+        if: matrix.os == 'ubuntu-latest'  # Only build once
         run: |
-          # Install build tools
           python -m pip install --upgrade pip build
-          
-          # Build pure Python packages
           python -m build packages/leann-core --outdir wheelhouse/
           python -m build packages/leann --outdir wheelhouse/
       
       - name: Build leann-backend-hnsw wheels  
-        uses: pypa/cibuildwheel@v2.16.2
+        uses: pypa/cibuildwheel@v2.20.0
         with:
           package-dir: packages/leann-backend-hnsw
           output-dir: wheelhouse
         env:
-          CIBW_BUILD: cp39-* cp310-* cp311-* cp312-*
+          CIBW_BUILD: cp39-* cp310-* cp311-* cp312-* cp313-*
-          CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+          CIBW_SKIP: "*-win32 *-manylinux_i686 pp* *musllinux*"
-          CIBW_SKIP: "*-win32 *-manylinux_i686 pp*"
           
+          # Use manylinux_2_28 for better compatibility
+          CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28
+          CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28
+          
+          # Install dependencies before building
           CIBW_BEFORE_ALL_LINUX: |
-            yum clean all && yum makecache
+            dnf install -y epel-release
-            yum install -y epel-release || true
+            dnf install -y gcc-c++ boost-devel zeromq-devel openblas-devel cmake
-            yum makecache || true
-            # Install system dependencies
-            yum install -y \
-              gcc-c++ \
-              boost-devel \
-              protobuf-compiler \
-              protobuf-devel \
-              zeromq-devel \
-              pkgconfig \
-              openblas-devel \
-              cmake || echo "Some packages failed, continuing..."
-            
-            # Verify zmq installation and create pkg-config file if needed
-            if [ ! -f /usr/lib64/pkgconfig/libzmq.pc ] && [ ! -f /usr/lib/pkgconfig/libzmq.pc ]; then
-              echo "Creating libzmq.pc file..."
-              mkdir -p /usr/lib64/pkgconfig
-              cat > /usr/lib64/pkgconfig/libzmq.pc << 'EOF'
-            prefix=/usr
-            exec_prefix=${prefix}
-            libdir=${exec_prefix}/lib64
-            includedir=${prefix}/include
-            
-            Name: libzmq
-            Description: ZeroMQ library
-            Version: 4.1.4
-            Libs: -L${libdir} -lzmq
-            Cflags: -I${includedir}
-            EOF
-            fi
           
           CIBW_BEFORE_ALL_MACOS: |
-            brew install llvm libomp boost protobuf zeromq
+            brew install boost zeromq openblas cmake
           
-          CIBW_ENVIRONMENT_LINUX: |
+          CIBW_BEFORE_ALL_WINDOWS: |
-            PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH
+            choco install cmake -y
           
-          CIBW_ENVIRONMENT_MACOS: |
+          # Test the wheel
-            CC=$(brew --prefix llvm)/bin/clang
+          CIBW_TEST_COMMAND: |
-            CXX=$(brew --prefix llvm)/bin/clang++
+            python -c "import leann_backend_hnsw; print('HNSW backend imported successfully')"
           
-          CIBW_TEST_REQUIRES: leann-core numpy pyzmq msgpack
+          # Environment variables for build
-          CIBW_TEST_COMMAND: python -c "import leann_backend_hnsw"
+          CIBW_ENVIRONMENT: |
+            CMAKE_BUILD_PARALLEL_LEVEL=8
       
-      - name: Build leann-backend-diskann wheels
+      - name: Build leann-backend-diskann wheels  
-        uses: pypa/cibuildwheel@v2.16.2
+        uses: pypa/cibuildwheel@v2.20.0
         with:
           package-dir: packages/leann-backend-diskann
           output-dir: wheelhouse
         env:
-          CIBW_BUILD: cp39-* cp310-* cp311-* cp312-*
+          CIBW_BUILD: cp39-* cp310-* cp311-* cp312-* cp313-*
-          CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+          CIBW_SKIP: "*-win32 *-manylinux_i686 pp* *musllinux*"
-          CIBW_SKIP: "*-win32 *-manylinux_i686 pp*"
+          
+          CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28
+          CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28
           
           CIBW_BEFORE_ALL_LINUX: |
-            yum clean all && yum makecache
+            dnf install -y epel-release
-            yum install -y epel-release || true
+            dnf install -y gcc-c++ boost-devel zeromq-devel openblas-devel cmake
-            yum makecache || true
-            # Install system dependencies for DiskANN
-            yum install -y \
-              gcc-c++ \
-              protobuf-compiler \
-              protobuf-devel \
-              openblas-devel \
-              pkgconfig \
-              cmake || echo "Some packages failed, continuing..."
-            yum install -y libaio-devel || echo "libaio-devel not available, continuing..."
           
           CIBW_BEFORE_ALL_MACOS: |
-            brew install llvm libomp protobuf
+            brew install boost zeromq openblas cmake
           
-          CIBW_ENVIRONMENT_LINUX: |
+          CIBW_BEFORE_ALL_WINDOWS: |
-            PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH
+            choco install cmake -y
           
-          CIBW_ENVIRONMENT_MACOS: |
+          CIBW_TEST_COMMAND: |
-            CC=$(brew --prefix llvm)/bin/clang
+            python -c "import leann_backend_diskann; print('DiskANN backend imported successfully')"
-            CXX=$(brew --prefix llvm)/bin/clang++
           
-          CIBW_TEST_REQUIRES: leann-core numpy
+          CIBW_ENVIRONMENT: |
-          CIBW_TEST_COMMAND: python -c "import leann_backend_diskann"
+            CMAKE_BUILD_PARALLEL_LEVEL=8
-      
-      - name: List built packages
-        run: |
-          echo "📦 Built packages:"
-          ls -la wheelhouse/
       
       - uses: actions/upload-artifact@v4
         with:
-          name: cibw-wheels-${{ matrix.os }}
+          name: wheels-${{ matrix.os }}
           path: ./wheelhouse/*.whl 

.github/workflows/test-manylinux.yml

@@ -0,0 +1,60 @@
+name: Test Manylinux Build
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [ main ]
+    paths:
+      - '.github/workflows/**'
+      - 'packages/**'
+      - 'pyproject.toml'
+  push:
+    branches:
+      - 'fix/manylinux-*'
+      - 'test/build-*'
+
+jobs:
+  build:
+    uses: ./.github/workflows/build-cibuildwheel.yml
+    
+  test-install:
+    needs: build
+    runs-on: ubuntu-22.04  # Simulating Colab environment
+    strategy:
+      matrix:
+        python-version: ['3.10', '3.11', '3.12']
+    steps:
+      - uses: actions/checkout@v4
+        
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: wheels-*
+          path: dist
+          merge-multiple: true
+          
+      - name: Test installation
+        run: |
+          python -m pip install --upgrade pip
+          # Find and install the appropriate wheels
+          pip install dist/leann_core-*.whl
+          pip install dist/leann_backend_hnsw-*manylinux*.whl
+          pip install dist/leann-*.whl
+          
+      - name: Test import
+        run: |
+          python -c "
+          import leann
+          from leann import LeannBuilder, LeannSearcher
+          print('Successfully imported leann modules')
+          
+          # Quick functionality test
+          builder = LeannBuilder(backend_name='hnsw')
+          builder.add_text('Test document')
+          print('LeannBuilder created and used successfully')
+          " 

packages/leann-backend-diskann/pyproject.toml

@@ -16,4 +16,12 @@ wheel.packages = ["leann_backend_diskann"]
 editable.mode = "redirect"
 cmake.build-type = "Release"
 build.verbose = true
-build.tool-args = ["-j8"]
+build.tool-args = ["-j8"]
+wheel.exclude = ["CMakeLists.txt", "src", "third_party/**", "*.o", "*.so"]
+sdist.include = ["CMakeLists.txt", "src", "third_party", "leann_backend_diskann/*.txt"]
+
+[tool.scikit-build.cmake.define]
+CMAKE_BUILD_PARALLEL_LEVEL = "8"
+# Help CMake find Python in cibuildwheel environments
+Python_FIND_VIRTUALENV = "ONLY"
+Python3_FIND_VIRTUALENV = "ONLY"

packages/leann-backend-hnsw/pyproject.toml

@@ -16,12 +16,15 @@ dependencies = [
 ]
 
 [tool.scikit-build]
-wheel.packages = ["leann_backend_hnsw"]
+wheel.exclude = ["CMakeLists.txt", "src", "third_party"]
-editable.mode = "redirect"
+sdist.include = ["CMakeLists.txt", "src", "third_party", "leann_backend_hnsw/*.txt"]
-cmake.build-type = "Release"
+cmake.args = ["-DCMAKE_BUILD_TYPE=Release"]
-build.verbose = true
+# 确保 CMake 可以找到系统库
-build.tool-args = ["-j8"]
+cmake.verbose = true
 
 # CMake definitions to optimize compilation
 [tool.scikit-build.cmake.define]
-CMAKE_BUILD_PARALLEL_LEVEL = "8"
+CMAKE_BUILD_PARALLEL_LEVEL = "8"
+# Help CMake find Python in cibuildwheel environments
+Python_FIND_VIRTUALENV = "ONLY"
+Python3_FIND_VIRTUALENV = "ONLY"

pyproject.toml

@@ -60,3 +60,26 @@ py-modules = []
 leann-core = { path = "packages/leann-core", editable = true }
 leann-backend-diskann = { path = "packages/leann-backend-diskann", editable = true }
 leann-backend-hnsw = { path = "packages/leann-backend-hnsw", editable = true }
+
+[tool.cibuildwheel]
+# 跳过32位和PyPy构建
+skip = "*-win32 *-manylinux_i686 pp* *musllinux*"
+
+# 使用更新的manylinux镜像以获得更好的兼容性
+manylinux-x86_64-image = "manylinux_2_28"
+manylinux-aarch64-image = "manylinux_2_28"
+
+# Linux系统依赖
+[tool.cibuildwheel.linux]
+before-all = """
+  dnf install -y epel-release
+  dnf install -y gcc-c++ boost-devel zeromq-devel openblas-devel cmake
+"""
+
+# macOS系统依赖
+[tool.cibuildwheel.macos]
+before-all = "brew install boost zeromq openblas cmake"
+
+# Windows系统依赖
+[tool.cibuildwheel.windows]
+before-all = "choco install cmake -y"