diff --git a/.github/workflows/build-cibuildwheel.yml b/.github/workflows/build-cibuildwheel.yml index 4cb4a07..847c00f 100644 --- a/.github/workflows/build-cibuildwheel.yml +++ b/.github/workflows/build-cibuildwheel.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, macos-latest, windows-latest] steps: - uses: actions/checkout@v4 @@ -27,118 +27,77 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: '3.11' # Version for building pure Python packages + python-version: '3.11' - # Build each package separately in our monorepo + # Build pure Python packages separately - name: Build pure Python packages (leann-core, leann) - if: matrix.os == 'ubuntu-latest' # Only build once, they're platform-independent + if: matrix.os == 'ubuntu-latest' # Only build once run: | - # Install build tools python -m pip install --upgrade pip build - - # Build pure Python packages python -m build packages/leann-core --outdir wheelhouse/ python -m build packages/leann --outdir wheelhouse/ - name: Build leann-backend-hnsw wheels - uses: pypa/cibuildwheel@v2.16.2 + uses: pypa/cibuildwheel@v2.20.0 with: package-dir: packages/leann-backend-hnsw output-dir: wheelhouse env: - CIBW_BUILD: cp39-* cp310-* cp311-* cp312-* - CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 - CIBW_SKIP: "*-win32 *-manylinux_i686 pp*" + CIBW_BUILD: cp39-* cp310-* cp311-* cp312-* cp313-* + CIBW_SKIP: "*-win32 *-manylinux_i686 pp* *musllinux*" + # Use manylinux_2_28 for better compatibility + CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 + CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28 + + # Install dependencies before building CIBW_BEFORE_ALL_LINUX: | - yum clean all && yum makecache - yum install -y epel-release || true - yum makecache || true - # Install system dependencies - yum install -y \ - gcc-c++ \ - boost-devel \ - protobuf-compiler \ - protobuf-devel \ - zeromq-devel \ - pkgconfig \ - openblas-devel \ - cmake || echo "Some packages failed, continuing..." - - # Verify zmq installation and create pkg-config file if needed - if [ ! -f /usr/lib64/pkgconfig/libzmq.pc ] && [ ! -f /usr/lib/pkgconfig/libzmq.pc ]; then - echo "Creating libzmq.pc file..." - mkdir -p /usr/lib64/pkgconfig - cat > /usr/lib64/pkgconfig/libzmq.pc << 'EOF' - prefix=/usr - exec_prefix=${prefix} - libdir=${exec_prefix}/lib64 - includedir=${prefix}/include - - Name: libzmq - Description: ZeroMQ library - Version: 4.1.4 - Libs: -L${libdir} -lzmq - Cflags: -I${includedir} - EOF - fi + dnf install -y epel-release + dnf install -y gcc-c++ boost-devel zeromq-devel openblas-devel cmake CIBW_BEFORE_ALL_MACOS: | - brew install llvm libomp boost protobuf zeromq + brew install boost zeromq openblas cmake - CIBW_ENVIRONMENT_LINUX: | - PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH + CIBW_BEFORE_ALL_WINDOWS: | + choco install cmake -y - CIBW_ENVIRONMENT_MACOS: | - CC=$(brew --prefix llvm)/bin/clang - CXX=$(brew --prefix llvm)/bin/clang++ + # Test the wheel + CIBW_TEST_COMMAND: | + python -c "import leann_backend_hnsw; print('HNSW backend imported successfully')" - CIBW_TEST_REQUIRES: leann-core numpy pyzmq msgpack - CIBW_TEST_COMMAND: python -c "import leann_backend_hnsw" + # Environment variables for build + CIBW_ENVIRONMENT: | + CMAKE_BUILD_PARALLEL_LEVEL=8 - - name: Build leann-backend-diskann wheels - uses: pypa/cibuildwheel@v2.16.2 + - name: Build leann-backend-diskann wheels + uses: pypa/cibuildwheel@v2.20.0 with: package-dir: packages/leann-backend-diskann output-dir: wheelhouse env: - CIBW_BUILD: cp39-* cp310-* cp311-* cp312-* - CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 - CIBW_SKIP: "*-win32 *-manylinux_i686 pp*" + CIBW_BUILD: cp39-* cp310-* cp311-* cp312-* cp313-* + CIBW_SKIP: "*-win32 *-manylinux_i686 pp* *musllinux*" + + CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 + CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28 CIBW_BEFORE_ALL_LINUX: | - yum clean all && yum makecache - yum install -y epel-release || true - yum makecache || true - # Install system dependencies for DiskANN - yum install -y \ - gcc-c++ \ - protobuf-compiler \ - protobuf-devel \ - openblas-devel \ - pkgconfig \ - cmake || echo "Some packages failed, continuing..." - yum install -y libaio-devel || echo "libaio-devel not available, continuing..." + dnf install -y epel-release + dnf install -y gcc-c++ boost-devel zeromq-devel openblas-devel cmake CIBW_BEFORE_ALL_MACOS: | - brew install llvm libomp protobuf + brew install boost zeromq openblas cmake - CIBW_ENVIRONMENT_LINUX: | - PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH + CIBW_BEFORE_ALL_WINDOWS: | + choco install cmake -y - CIBW_ENVIRONMENT_MACOS: | - CC=$(brew --prefix llvm)/bin/clang - CXX=$(brew --prefix llvm)/bin/clang++ + CIBW_TEST_COMMAND: | + python -c "import leann_backend_diskann; print('DiskANN backend imported successfully')" - CIBW_TEST_REQUIRES: leann-core numpy - CIBW_TEST_COMMAND: python -c "import leann_backend_diskann" - - - name: List built packages - run: | - echo "📦 Built packages:" - ls -la wheelhouse/ + CIBW_ENVIRONMENT: | + CMAKE_BUILD_PARALLEL_LEVEL=8 - uses: actions/upload-artifact@v4 with: - name: cibw-wheels-${{ matrix.os }} + name: wheels-${{ matrix.os }} path: ./wheelhouse/*.whl \ No newline at end of file diff --git a/.github/workflows/test-manylinux.yml b/.github/workflows/test-manylinux.yml new file mode 100644 index 0000000..1a43d49 --- /dev/null +++ b/.github/workflows/test-manylinux.yml @@ -0,0 +1,60 @@ +name: Test Manylinux Build + +on: + workflow_dispatch: + pull_request: + branches: [ main ] + paths: + - '.github/workflows/**' + - 'packages/**' + - 'pyproject.toml' + push: + branches: + - 'fix/manylinux-*' + - 'test/build-*' + +jobs: + build: + uses: ./.github/workflows/build-cibuildwheel.yml + + test-install: + needs: build + runs-on: ubuntu-22.04 # Simulating Colab environment + strategy: + matrix: + python-version: ['3.10', '3.11', '3.12'] + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + pattern: wheels-* + path: dist + merge-multiple: true + + - name: Test installation + run: | + python -m pip install --upgrade pip + # Find and install the appropriate wheels + pip install dist/leann_core-*.whl + pip install dist/leann_backend_hnsw-*manylinux*.whl + pip install dist/leann-*.whl + + - name: Test import + run: | + python -c " + import leann + from leann import LeannBuilder, LeannSearcher + print('Successfully imported leann modules') + + # Quick functionality test + builder = LeannBuilder(backend_name='hnsw') + builder.add_text('Test document') + print('LeannBuilder created and used successfully') + " \ No newline at end of file diff --git a/packages/leann-backend-diskann/pyproject.toml b/packages/leann-backend-diskann/pyproject.toml index b13ecef..1c63600 100644 --- a/packages/leann-backend-diskann/pyproject.toml +++ b/packages/leann-backend-diskann/pyproject.toml @@ -16,4 +16,12 @@ wheel.packages = ["leann_backend_diskann"] editable.mode = "redirect" cmake.build-type = "Release" build.verbose = true -build.tool-args = ["-j8"] \ No newline at end of file +build.tool-args = ["-j8"] +wheel.exclude = ["CMakeLists.txt", "src", "third_party/**", "*.o", "*.so"] +sdist.include = ["CMakeLists.txt", "src", "third_party", "leann_backend_diskann/*.txt"] + +[tool.scikit-build.cmake.define] +CMAKE_BUILD_PARALLEL_LEVEL = "8" +# Help CMake find Python in cibuildwheel environments +Python_FIND_VIRTUALENV = "ONLY" +Python3_FIND_VIRTUALENV = "ONLY" \ No newline at end of file diff --git a/packages/leann-backend-hnsw/pyproject.toml b/packages/leann-backend-hnsw/pyproject.toml index 71b4cdc..16e97e3 100644 --- a/packages/leann-backend-hnsw/pyproject.toml +++ b/packages/leann-backend-hnsw/pyproject.toml @@ -16,12 +16,15 @@ dependencies = [ ] [tool.scikit-build] -wheel.packages = ["leann_backend_hnsw"] -editable.mode = "redirect" -cmake.build-type = "Release" -build.verbose = true -build.tool-args = ["-j8"] +wheel.exclude = ["CMakeLists.txt", "src", "third_party"] +sdist.include = ["CMakeLists.txt", "src", "third_party", "leann_backend_hnsw/*.txt"] +cmake.args = ["-DCMAKE_BUILD_TYPE=Release"] +# 确保 CMake 可以找到系统库 +cmake.verbose = true # CMake definitions to optimize compilation [tool.scikit-build.cmake.define] -CMAKE_BUILD_PARALLEL_LEVEL = "8" \ No newline at end of file +CMAKE_BUILD_PARALLEL_LEVEL = "8" +# Help CMake find Python in cibuildwheel environments +Python_FIND_VIRTUALENV = "ONLY" +Python3_FIND_VIRTUALENV = "ONLY" \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 84a0783..2fdd41b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,3 +60,26 @@ py-modules = [] leann-core = { path = "packages/leann-core", editable = true } leann-backend-diskann = { path = "packages/leann-backend-diskann", editable = true } leann-backend-hnsw = { path = "packages/leann-backend-hnsw", editable = true } + +[tool.cibuildwheel] +# 跳过32位和PyPy构建 +skip = "*-win32 *-manylinux_i686 pp* *musllinux*" + +# 使用更新的manylinux镜像以获得更好的兼容性 +manylinux-x86_64-image = "manylinux_2_28" +manylinux-aarch64-image = "manylinux_2_28" + +# Linux系统依赖 +[tool.cibuildwheel.linux] +before-all = """ + dnf install -y epel-release + dnf install -y gcc-c++ boost-devel zeromq-devel openblas-devel cmake +""" + +# macOS系统依赖 +[tool.cibuildwheel.macos] +before-all = "brew install boost zeromq openblas cmake" + +# Windows系统依赖 +[tool.cibuildwheel.windows] +before-all = "choco install cmake -y"