Compare commits
22 Commits
fix/drop-p
...
fix/manyli
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
971653fa1a | ||
|
|
02672c040d | ||
|
|
f55108feda | ||
|
|
74d485c908 | ||
|
|
d1fefb6378 | ||
|
|
732384f4f8 | ||
|
|
ae38e10d1b | ||
|
|
ca0fd88934 | ||
|
|
3c8d32f156 | ||
|
|
b8ff00fc6a | ||
|
|
3c836766f8 | ||
|
|
b4a1dfb9c7 | ||
|
|
a4d66e95d8 | ||
|
|
cf58b3e31b | ||
|
|
e9c2ca7936 | ||
|
|
dab154a77b | ||
|
|
13413dfae5 | ||
|
|
0543cc9816 | ||
|
|
fb53ed9a0e | ||
|
|
015f43733a | ||
|
|
2957c8bf5a | ||
|
|
a73194c3f6 |
185
.github/workflows/build-cibuildwheel.yml
vendored
185
.github/workflows/build-cibuildwheel.yml
vendored
@@ -16,7 +16,7 @@ jobs:
|
|||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
os: [ubuntu-latest, macos-latest]
|
os: [ubuntu-latest] # Focus on Linux/manylinux for Colab compatibility
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
@@ -27,118 +27,145 @@ jobs:
|
|||||||
- name: Setup Python
|
- name: Setup Python
|
||||||
uses: actions/setup-python@v5
|
uses: actions/setup-python@v5
|
||||||
with:
|
with:
|
||||||
python-version: '3.11' # Version for building pure Python packages
|
python-version: '3.11'
|
||||||
|
|
||||||
# Build each package separately in our monorepo
|
# Build pure Python packages separately
|
||||||
- name: Build pure Python packages (leann-core, leann)
|
- name: Build pure Python packages (leann-core, leann)
|
||||||
if: matrix.os == 'ubuntu-latest' # Only build once, they're platform-independent
|
if: matrix.os == 'ubuntu-latest' # Only build once
|
||||||
run: |
|
run: |
|
||||||
# Install build tools
|
|
||||||
python -m pip install --upgrade pip build
|
python -m pip install --upgrade pip build
|
||||||
|
|
||||||
# Build pure Python packages
|
|
||||||
python -m build packages/leann-core --outdir wheelhouse/
|
python -m build packages/leann-core --outdir wheelhouse/
|
||||||
python -m build packages/leann --outdir wheelhouse/
|
python -m build packages/leann --outdir wheelhouse/
|
||||||
|
|
||||||
- name: Build leann-backend-hnsw wheels
|
- name: Build leann-backend-hnsw wheels
|
||||||
uses: pypa/cibuildwheel@v2.16.2
|
uses: pypa/cibuildwheel@v2.20.0
|
||||||
with:
|
with:
|
||||||
package-dir: packages/leann-backend-hnsw
|
package-dir: packages/leann-backend-hnsw
|
||||||
output-dir: wheelhouse
|
output-dir: wheelhouse
|
||||||
env:
|
env:
|
||||||
CIBW_BUILD: cp39-* cp310-* cp311-* cp312-*
|
CIBW_BUILD: cp39-* cp310-* cp311-* cp312-* cp313-*
|
||||||
CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
|
CIBW_SKIP: "*-win32 *-manylinux_i686 pp* *musllinux*"
|
||||||
CIBW_SKIP: "*-win32 *-manylinux_i686 pp*"
|
|
||||||
|
|
||||||
|
# Use manylinux_2_35 for Colab compatibility with modern features
|
||||||
|
CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_35
|
||||||
|
CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_35
|
||||||
|
|
||||||
|
# Linux dependencies - using dnf for manylinux_2_35 (based on AlmaLinux 9)
|
||||||
CIBW_BEFORE_ALL_LINUX: |
|
CIBW_BEFORE_ALL_LINUX: |
|
||||||
yum clean all && yum makecache
|
dnf install -y epel-release
|
||||||
yum install -y epel-release || true
|
dnf install -y gcc-c++ boost-devel zeromq-devel openblas-devel cmake python3-devel
|
||||||
yum makecache || true
|
|
||||||
# Install system dependencies
|
# Install numpy before building
|
||||||
yum install -y \
|
CIBW_BEFORE_BUILD: |
|
||||||
gcc-c++ \
|
pip install numpy
|
||||||
boost-devel \
|
pip install --upgrade pip setuptools wheel
|
||||||
protobuf-compiler \
|
|
||||||
protobuf-devel \
|
CIBW_BEFORE_BUILD_LINUX: |
|
||||||
zeromq-devel \
|
pip install numpy
|
||||||
pkgconfig \
|
pip install --upgrade pip setuptools wheel swig
|
||||||
openblas-devel \
|
|
||||||
cmake || echo "Some packages failed, continuing..."
|
|
||||||
|
|
||||||
# Verify zmq installation and create pkg-config file if needed
|
|
||||||
if [ ! -f /usr/lib64/pkgconfig/libzmq.pc ] && [ ! -f /usr/lib/pkgconfig/libzmq.pc ]; then
|
|
||||||
echo "Creating libzmq.pc file..."
|
|
||||||
mkdir -p /usr/lib64/pkgconfig
|
|
||||||
cat > /usr/lib64/pkgconfig/libzmq.pc << 'EOF'
|
|
||||||
prefix=/usr
|
|
||||||
exec_prefix=${prefix}
|
|
||||||
libdir=${exec_prefix}/lib64
|
|
||||||
includedir=${prefix}/include
|
|
||||||
|
|
||||||
Name: libzmq
|
|
||||||
Description: ZeroMQ library
|
|
||||||
Version: 4.1.4
|
|
||||||
Libs: -L${libdir} -lzmq
|
|
||||||
Cflags: -I${includedir}
|
|
||||||
EOF
|
|
||||||
fi
|
|
||||||
|
|
||||||
CIBW_BEFORE_ALL_MACOS: |
|
CIBW_BEFORE_ALL_MACOS: |
|
||||||
brew install llvm libomp boost protobuf zeromq
|
brew install boost zeromq openblas cmake libomp
|
||||||
|
|
||||||
|
# Pre-install test dependencies to avoid compilation
|
||||||
|
CIBW_BEFORE_TEST: |
|
||||||
|
pip install --only-binary :all: "pyzmq>=23.0.0"
|
||||||
|
|
||||||
|
# Test command to verify the wheel works
|
||||||
|
CIBW_TEST_COMMAND: |
|
||||||
|
python -c "import leann_backend_hnsw; print('HNSW backend imported successfully')"
|
||||||
|
|
||||||
|
# Skip problematic configurations
|
||||||
|
CIBW_TEST_SKIP: "*-macosx_arm64" # Skip ARM64 tests on GitHub Actions
|
||||||
|
|
||||||
|
# Test dependencies
|
||||||
|
CIBW_TEST_REQUIRES: "pytest numpy"
|
||||||
|
|
||||||
|
# Environment variables for build
|
||||||
|
CIBW_ENVIRONMENT: |
|
||||||
|
CMAKE_BUILD_PARALLEL_LEVEL=8
|
||||||
|
Python_FIND_VIRTUALENV=ONLY
|
||||||
|
Python3_FIND_VIRTUALENV=ONLY
|
||||||
|
|
||||||
|
# Linux-specific environment variables
|
||||||
CIBW_ENVIRONMENT_LINUX: |
|
CIBW_ENVIRONMENT_LINUX: |
|
||||||
PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH
|
CMAKE_BUILD_PARALLEL_LEVEL=8
|
||||||
|
|
||||||
|
# macOS-specific environment variables
|
||||||
CIBW_ENVIRONMENT_MACOS: |
|
CIBW_ENVIRONMENT_MACOS: |
|
||||||
CC=$(brew --prefix llvm)/bin/clang
|
CMAKE_BUILD_PARALLEL_LEVEL=8
|
||||||
CXX=$(brew --prefix llvm)/bin/clang++
|
MACOSX_DEPLOYMENT_TARGET=11.0
|
||||||
|
CMAKE_OSX_DEPLOYMENT_TARGET=11.0
|
||||||
CIBW_TEST_REQUIRES: leann-core numpy pyzmq msgpack
|
Python_FIND_VIRTUALENV=ONLY
|
||||||
CIBW_TEST_COMMAND: python -c "import leann_backend_hnsw"
|
Python3_FIND_VIRTUALENV=ONLY
|
||||||
|
|
||||||
- name: Build leann-backend-diskann wheels
|
- name: Build leann-backend-diskann wheels
|
||||||
uses: pypa/cibuildwheel@v2.16.2
|
uses: pypa/cibuildwheel@v2.20.0
|
||||||
with:
|
with:
|
||||||
package-dir: packages/leann-backend-diskann
|
package-dir: packages/leann-backend-diskann
|
||||||
output-dir: wheelhouse
|
output-dir: wheelhouse
|
||||||
env:
|
env:
|
||||||
CIBW_BUILD: cp39-* cp310-* cp311-* cp312-*
|
CIBW_BUILD: cp39-* cp310-* cp311-* cp312-* cp313-*
|
||||||
CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
|
CIBW_SKIP: "*-win32 *-manylinux_i686 pp* *musllinux*"
|
||||||
CIBW_SKIP: "*-win32 *-manylinux_i686 pp*"
|
|
||||||
|
CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_35
|
||||||
|
CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_35
|
||||||
|
|
||||||
CIBW_BEFORE_ALL_LINUX: |
|
CIBW_BEFORE_ALL_LINUX: |
|
||||||
yum clean all && yum makecache
|
dnf install -y epel-release
|
||||||
yum install -y epel-release || true
|
dnf install -y gcc-c++ boost-devel zeromq-devel openblas-devel cmake python3-devel
|
||||||
yum makecache || true
|
|
||||||
# Install system dependencies for DiskANN
|
# Install numpy before building
|
||||||
yum install -y \
|
CIBW_BEFORE_BUILD: |
|
||||||
gcc-c++ \
|
pip install numpy
|
||||||
protobuf-compiler \
|
pip install --upgrade pip setuptools wheel
|
||||||
protobuf-devel \
|
|
||||||
openblas-devel \
|
CIBW_BEFORE_BUILD_LINUX: |
|
||||||
pkgconfig \
|
pip install numpy
|
||||||
cmake || echo "Some packages failed, continuing..."
|
pip install --upgrade pip setuptools wheel swig
|
||||||
yum install -y libaio-devel || echo "libaio-devel not available, continuing..."
|
|
||||||
|
|
||||||
CIBW_BEFORE_ALL_MACOS: |
|
CIBW_BEFORE_ALL_MACOS: |
|
||||||
brew install llvm libomp protobuf
|
brew install boost zeromq openblas cmake libomp
|
||||||
|
|
||||||
|
# Pre-install test dependencies to avoid compilation
|
||||||
|
CIBW_BEFORE_TEST: |
|
||||||
|
pip install --only-binary :all: "pyzmq>=23.0.0"
|
||||||
|
|
||||||
|
# Test command to verify the wheel works
|
||||||
|
CIBW_TEST_COMMAND: |
|
||||||
|
python -c "import leann_backend_diskann; print('DiskANN backend imported successfully')"
|
||||||
|
|
||||||
|
# Skip problematic configurations
|
||||||
|
CIBW_TEST_SKIP: "*-macosx_arm64" # Skip ARM64 tests on GitHub Actions
|
||||||
|
|
||||||
|
# Test dependencies - avoid pyzmq due to manylinux2014 compatibility issues
|
||||||
|
CIBW_TEST_REQUIRES: "pytest numpy"
|
||||||
|
|
||||||
|
CIBW_ENVIRONMENT: |
|
||||||
|
CMAKE_BUILD_PARALLEL_LEVEL=8
|
||||||
|
Python_FIND_VIRTUALENV=ONLY
|
||||||
|
Python3_FIND_VIRTUALENV=ONLY
|
||||||
|
|
||||||
|
# Linux-specific environment variables
|
||||||
CIBW_ENVIRONMENT_LINUX: |
|
CIBW_ENVIRONMENT_LINUX: |
|
||||||
PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH
|
CMAKE_BUILD_PARALLEL_LEVEL=8
|
||||||
|
CMAKE_PREFIX_PATH=$VIRTUAL_ENV
|
||||||
|
Python_FIND_VIRTUALENV=ONLY
|
||||||
|
Python3_FIND_VIRTUALENV=ONLY
|
||||||
|
Python_FIND_STRATEGY=LOCATION
|
||||||
|
Python3_FIND_STRATEGY=LOCATION
|
||||||
|
Python_EXECUTABLE=$VIRTUAL_ENV/bin/python
|
||||||
|
Python3_EXECUTABLE=$VIRTUAL_ENV/bin/python
|
||||||
|
|
||||||
|
# macOS-specific environment variables
|
||||||
CIBW_ENVIRONMENT_MACOS: |
|
CIBW_ENVIRONMENT_MACOS: |
|
||||||
CC=$(brew --prefix llvm)/bin/clang
|
CMAKE_BUILD_PARALLEL_LEVEL=8
|
||||||
CXX=$(brew --prefix llvm)/bin/clang++
|
MACOSX_DEPLOYMENT_TARGET=11.0
|
||||||
|
CMAKE_OSX_DEPLOYMENT_TARGET=11.0
|
||||||
CIBW_TEST_REQUIRES: leann-core numpy
|
Python_FIND_VIRTUALENV=ONLY
|
||||||
CIBW_TEST_COMMAND: python -c "import leann_backend_diskann"
|
Python3_FIND_VIRTUALENV=ONLY
|
||||||
|
|
||||||
- name: List built packages
|
|
||||||
run: |
|
|
||||||
echo "📦 Built packages:"
|
|
||||||
ls -la wheelhouse/
|
|
||||||
|
|
||||||
- uses: actions/upload-artifact@v4
|
- uses: actions/upload-artifact@v4
|
||||||
with:
|
with:
|
||||||
name: cibw-wheels-${{ matrix.os }}
|
name: wheels-${{ matrix.os }}
|
||||||
path: ./wheelhouse/*.whl
|
path: ./wheelhouse/*.whl
|
||||||
12
.github/workflows/ci-cibuildwheel.yml
vendored
12
.github/workflows/ci-cibuildwheel.yml
vendored
@@ -1,12 +0,0 @@
|
|||||||
name: CI - cibuildwheel (Test)
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches: [ main ]
|
|
||||||
pull_request:
|
|
||||||
branches: [ main ]
|
|
||||||
workflow_dispatch: # Allow manual triggering
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
build:
|
|
||||||
uses: ./.github/workflows/build-cibuildwheel.yml
|
|
||||||
60
.github/workflows/test-manylinux.yml
vendored
Normal file
60
.github/workflows/test-manylinux.yml
vendored
Normal file
@@ -0,0 +1,60 @@
|
|||||||
|
name: Test Manylinux Build
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
pull_request:
|
||||||
|
branches: [ main ]
|
||||||
|
paths:
|
||||||
|
- '.github/workflows/**'
|
||||||
|
- 'packages/**'
|
||||||
|
- 'pyproject.toml'
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- 'fix/manylinux-*'
|
||||||
|
- 'test/build-*'
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
uses: ./.github/workflows/build-cibuildwheel.yml
|
||||||
|
|
||||||
|
test-install:
|
||||||
|
needs: build
|
||||||
|
runs-on: ubuntu-22.04 # Simulating Colab environment
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
python-version: ['3.10', '3.11', '3.12']
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Set up Python
|
||||||
|
uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: ${{ matrix.python-version }}
|
||||||
|
|
||||||
|
- name: Download artifacts
|
||||||
|
uses: actions/download-artifact@v4
|
||||||
|
with:
|
||||||
|
pattern: wheels-*
|
||||||
|
path: dist
|
||||||
|
merge-multiple: true
|
||||||
|
|
||||||
|
- name: Test installation
|
||||||
|
run: |
|
||||||
|
python -m pip install --upgrade pip
|
||||||
|
# Find and install the appropriate wheels
|
||||||
|
pip install dist/leann_core-*.whl
|
||||||
|
pip install dist/leann_backend_hnsw-*manylinux*.whl
|
||||||
|
pip install dist/leann-*.whl
|
||||||
|
|
||||||
|
- name: Test import
|
||||||
|
run: |
|
||||||
|
python -c "
|
||||||
|
import leann
|
||||||
|
from leann import LeannBuilder, LeannSearcher
|
||||||
|
print('Successfully imported leann modules')
|
||||||
|
|
||||||
|
# Quick functionality test
|
||||||
|
builder = LeannBuilder(backend_name='hnsw')
|
||||||
|
builder.add_text('Test document')
|
||||||
|
print('LeannBuilder created and used successfully')
|
||||||
|
"
|
||||||
50
MANYLINUX_BUILD_STRATEGY.md
Normal file
50
MANYLINUX_BUILD_STRATEGY.md
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
# Manylinux Build Strategy
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
Google Colab requires wheels compatible with `manylinux_2_35_x86_64` or earlier. Our previous builds were producing `manylinux_2_39_x86_64` wheels, which are incompatible.
|
||||||
|
|
||||||
|
## Solution
|
||||||
|
We're using `cibuildwheel` with `manylinux_2_35` images to build wheels that are compatible with Google Colab while maintaining modern toolchain features.
|
||||||
|
|
||||||
|
### Key Changes
|
||||||
|
|
||||||
|
1. **cibuildwheel Configuration**
|
||||||
|
- Using `manylinux2014` images (provides `manylinux_2_17` compatibility)
|
||||||
|
- Using `yum` package manager (CentOS 7 based)
|
||||||
|
- Installing `cmake3` and creating symlink for compatibility
|
||||||
|
|
||||||
|
2. **Build Matrix**
|
||||||
|
- Python versions: 3.9, 3.10, 3.11, 3.12, 3.13
|
||||||
|
- Platforms: Linux (x86_64), macOS
|
||||||
|
- No Windows support (not required)
|
||||||
|
|
||||||
|
3. **Dependencies**
|
||||||
|
- Linux: gcc-c++, boost-devel, zeromq-devel, openblas-devel, cmake3
|
||||||
|
- macOS: boost, zeromq, openblas, cmake (via Homebrew)
|
||||||
|
|
||||||
|
4. **Environment Variables**
|
||||||
|
- `CMAKE_BUILD_PARALLEL_LEVEL=8`: Speed up builds
|
||||||
|
- `Python_FIND_VIRTUALENV=ONLY`: Help CMake find Python in cibuildwheel env
|
||||||
|
- `Python3_FIND_VIRTUALENV=ONLY`: Alternative variable for compatibility
|
||||||
|
|
||||||
|
## Testing Strategy
|
||||||
|
|
||||||
|
1. **CI Pipeline**: `test-manylinux.yml`
|
||||||
|
- Triggers on PR to main, manual dispatch, or push to `fix/manylinux-*` branches
|
||||||
|
- Builds wheels using cibuildwheel
|
||||||
|
- Tests installation on Ubuntu 22.04 (simulating Colab)
|
||||||
|
|
||||||
|
2. **Local Testing**
|
||||||
|
```bash
|
||||||
|
# Download built wheels
|
||||||
|
# Test in fresh environment
|
||||||
|
python -m venv test_env
|
||||||
|
source test_env/bin/activate
|
||||||
|
pip install leann_core-*.whl leann_backend_hnsw-*manylinux*.whl leann-*.whl
|
||||||
|
python -c "from leann import LeannBuilder; print('Success!')"
|
||||||
|
```
|
||||||
|
|
||||||
|
## References
|
||||||
|
- [cibuildwheel documentation](https://cibuildwheel.readthedocs.io/)
|
||||||
|
- [manylinux standards](https://github.com/pypa/manylinux)
|
||||||
|
- [PEP 599 - manylinux2014](https://peps.python.org/pep-0599/)
|
||||||
@@ -3,6 +3,34 @@
|
|||||||
cmake_minimum_required(VERSION 3.20)
|
cmake_minimum_required(VERSION 3.20)
|
||||||
project(leann_backend_diskann_wrapper)
|
project(leann_backend_diskann_wrapper)
|
||||||
|
|
||||||
|
# Find Python - scikit-build-core should provide this
|
||||||
|
find_package(Python REQUIRED COMPONENTS Interpreter Development.Module)
|
||||||
|
|
||||||
|
# Print Python information for debugging
|
||||||
|
message(STATUS "Python_FOUND: ${Python_FOUND}")
|
||||||
|
message(STATUS "Python_VERSION: ${Python_VERSION}")
|
||||||
|
message(STATUS "Python_EXECUTABLE: ${Python_EXECUTABLE}")
|
||||||
|
message(STATUS "Python_INCLUDE_DIRS: ${Python_INCLUDE_DIRS}")
|
||||||
|
|
||||||
|
# Pass Python information to DiskANN through cache variables
|
||||||
|
set(Python_EXECUTABLE ${Python_EXECUTABLE} CACHE FILEPATH "Python executable" FORCE)
|
||||||
|
set(Python_INCLUDE_DIRS ${Python_INCLUDE_DIRS} CACHE PATH "Python include dirs" FORCE)
|
||||||
|
set(Python_LIBRARIES ${Python_LIBRARIES} CACHE FILEPATH "Python libraries" FORCE)
|
||||||
|
set(Python_VERSION ${Python_VERSION} CACHE STRING "Python version" FORCE)
|
||||||
|
set(Python_FOUND ${Python_FOUND} CACHE BOOL "Python found" FORCE)
|
||||||
|
|
||||||
|
# Also set Python3 variables for compatibility
|
||||||
|
set(Python3_EXECUTABLE ${Python_EXECUTABLE} CACHE FILEPATH "Python3 executable" FORCE)
|
||||||
|
set(Python3_INCLUDE_DIRS ${Python_INCLUDE_DIRS} CACHE PATH "Python3 include dirs" FORCE)
|
||||||
|
set(Python3_LIBRARIES ${Python_LIBRARIES} CACHE FILEPATH "Python3 libraries" FORCE)
|
||||||
|
set(Python3_VERSION ${Python_VERSION} CACHE STRING "Python3 version" FORCE)
|
||||||
|
set(Python3_FOUND ${Python_FOUND} CACHE BOOL "Python3 found" FORCE)
|
||||||
|
set(Python3_Development_FOUND TRUE CACHE BOOL "Python3 development found" FORCE)
|
||||||
|
|
||||||
|
# Set Python finding strategy
|
||||||
|
set(Python_FIND_VIRTUALENV ONLY CACHE STRING "" FORCE)
|
||||||
|
set(Python3_FIND_VIRTUALENV ONLY CACHE STRING "" FORCE)
|
||||||
|
|
||||||
# Tell CMake to directly enter the DiskANN submodule and execute its own CMakeLists.txt
|
# Tell CMake to directly enter the DiskANN submodule and execute its own CMakeLists.txt
|
||||||
# DiskANN will handle everything itself, including compiling Python bindings
|
# DiskANN will handle everything itself, including compiling Python bindings
|
||||||
add_subdirectory(src/third_party/DiskANN)
|
add_subdirectory(third_party/DiskANN)
|
||||||
|
|||||||
@@ -16,4 +16,9 @@ wheel.packages = ["leann_backend_diskann"]
|
|||||||
editable.mode = "redirect"
|
editable.mode = "redirect"
|
||||||
cmake.build-type = "Release"
|
cmake.build-type = "Release"
|
||||||
build.verbose = true
|
build.verbose = true
|
||||||
build.tool-args = ["-j8"]
|
build.tool-args = ["-j8"]
|
||||||
|
wheel.exclude = ["CMakeLists.txt", "src", "third_party/**", "*.o", "*.so"]
|
||||||
|
sdist.include = ["CMakeLists.txt", "src", "third_party", "leann_backend_diskann/*.txt"]
|
||||||
|
|
||||||
|
[tool.scikit-build.cmake.define]
|
||||||
|
CMAKE_BUILD_PARALLEL_LEVEL = "8"
|
||||||
Submodule packages/leann-backend-diskann/third_party/DiskANN updated: af2a26481e...81666b7758
@@ -24,6 +24,38 @@ set(MSGPACK_USE_BOOST OFF CACHE BOOL "" FORCE)
|
|||||||
add_compile_definitions(MSGPACK_NO_BOOST)
|
add_compile_definitions(MSGPACK_NO_BOOST)
|
||||||
include_directories(third_party/msgpack-c/include)
|
include_directories(third_party/msgpack-c/include)
|
||||||
|
|
||||||
|
# Find Python for our own use (not for Faiss)
|
||||||
|
if(DEFINED SKBUILD)
|
||||||
|
message(STATUS "Building with scikit-build")
|
||||||
|
# scikit-build-core provides Python information
|
||||||
|
endif()
|
||||||
|
|
||||||
|
# Find Python - scikit-build-core should provide this
|
||||||
|
find_package(Python REQUIRED COMPONENTS Interpreter Development.Module NumPy)
|
||||||
|
|
||||||
|
# Print Python information for debugging
|
||||||
|
message(STATUS "Python_FOUND: ${Python_FOUND}")
|
||||||
|
message(STATUS "Python_VERSION: ${Python_VERSION}")
|
||||||
|
message(STATUS "Python_EXECUTABLE: ${Python_EXECUTABLE}")
|
||||||
|
message(STATUS "Python_INCLUDE_DIRS: ${Python_INCLUDE_DIRS}")
|
||||||
|
message(STATUS "Python_NumPy_INCLUDE_DIRS: ${Python_NumPy_INCLUDE_DIRS}")
|
||||||
|
|
||||||
|
# Pass Python information to faiss through cache variables
|
||||||
|
set(Python_EXECUTABLE ${Python_EXECUTABLE} CACHE FILEPATH "Python executable" FORCE)
|
||||||
|
set(Python_INCLUDE_DIRS ${Python_INCLUDE_DIRS} CACHE PATH "Python include dirs" FORCE)
|
||||||
|
set(Python_NumPy_INCLUDE_DIRS ${Python_NumPy_INCLUDE_DIRS} CACHE PATH "NumPy include dirs" FORCE)
|
||||||
|
set(Python_VERSION ${Python_VERSION} CACHE STRING "Python version" FORCE)
|
||||||
|
set(Python_FOUND ${Python_FOUND} CACHE BOOL "Python found" FORCE)
|
||||||
|
|
||||||
|
# Also set Python3 variables for compatibility
|
||||||
|
set(Python3_EXECUTABLE ${Python_EXECUTABLE} CACHE FILEPATH "Python3 executable" FORCE)
|
||||||
|
set(Python3_INCLUDE_DIRS ${Python_INCLUDE_DIRS} CACHE PATH "Python3 include dirs" FORCE)
|
||||||
|
set(Python3_NumPy_INCLUDE_DIRS ${Python_NumPy_INCLUDE_DIRS} CACHE PATH "NumPy include dirs" FORCE)
|
||||||
|
set(Python3_VERSION ${Python_VERSION} CACHE STRING "Python3 version" FORCE)
|
||||||
|
set(Python3_FOUND ${Python_FOUND} CACHE BOOL "Python3 found" FORCE)
|
||||||
|
set(Python3_Development_FOUND TRUE CACHE BOOL "Python3 development found" FORCE)
|
||||||
|
set(Python3_NumPy_FOUND TRUE CACHE BOOL "Python3 NumPy found" FORCE)
|
||||||
|
|
||||||
# Faiss configuration - streamlined build
|
# Faiss configuration - streamlined build
|
||||||
set(FAISS_ENABLE_PYTHON ON CACHE BOOL "" FORCE)
|
set(FAISS_ENABLE_PYTHON ON CACHE BOOL "" FORCE)
|
||||||
set(FAISS_ENABLE_GPU OFF CACHE BOOL "" FORCE)
|
set(FAISS_ENABLE_GPU OFF CACHE BOOL "" FORCE)
|
||||||
@@ -52,4 +84,8 @@ set(FAISS_BUILD_AVX512 OFF CACHE BOOL "" FORCE)
|
|||||||
# IMPORTANT: Disable building AVX versions to speed up compilation
|
# IMPORTANT: Disable building AVX versions to speed up compilation
|
||||||
set(FAISS_BUILD_AVX_VERSIONS OFF CACHE BOOL "" FORCE)
|
set(FAISS_BUILD_AVX_VERSIONS OFF CACHE BOOL "" FORCE)
|
||||||
|
|
||||||
|
# Force Faiss to use our Python settings
|
||||||
|
set(Python_FIND_VIRTUALENV ONLY CACHE STRING "" FORCE)
|
||||||
|
set(Python3_FIND_VIRTUALENV ONLY CACHE STRING "" FORCE)
|
||||||
|
|
||||||
add_subdirectory(third_party/faiss)
|
add_subdirectory(third_party/faiss)
|
||||||
@@ -16,12 +16,14 @@ dependencies = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
[tool.scikit-build]
|
[tool.scikit-build]
|
||||||
wheel.packages = ["leann_backend_hnsw"]
|
wheel.exclude = ["CMakeLists.txt", "src", "third_party"]
|
||||||
editable.mode = "redirect"
|
sdist.include = ["CMakeLists.txt", "src", "third_party", "leann_backend_hnsw/*.txt"]
|
||||||
cmake.build-type = "Release"
|
cmake.args = ["-DCMAKE_BUILD_TYPE=Release"]
|
||||||
build.verbose = true
|
# Ensure CMake can find system libraries
|
||||||
build.tool-args = ["-j8"]
|
build-dir = "build/{cache_tag}"
|
||||||
|
minimum-version = "build-system.requires"
|
||||||
|
|
||||||
# CMake definitions to optimize compilation
|
# CMake definitions to optimize compilation
|
||||||
[tool.scikit-build.cmake.define]
|
[tool.scikit-build.cmake.define]
|
||||||
CMAKE_BUILD_PARALLEL_LEVEL = "8"
|
CMAKE_BUILD_PARALLEL_LEVEL = "8"
|
||||||
|
SKBUILD_SOABI = "YES"
|
||||||
Submodule packages/leann-backend-hnsw/third_party/faiss updated: ff22e2c86b...553f937220
@@ -15,7 +15,7 @@ dependencies = [
|
|||||||
"numpy>=1.20.0",
|
"numpy>=1.20.0",
|
||||||
"tqdm>=4.60.0",
|
"tqdm>=4.60.0",
|
||||||
"psutil>=5.8.0",
|
"psutil>=5.8.0",
|
||||||
"pyzmq>=23.0.0",
|
"pyzmq>=23.0.0,<27", # Cap at 26.x for manylinux2014 compatibility
|
||||||
"msgpack>=1.0.0",
|
"msgpack>=1.0.0",
|
||||||
"torch>=2.0.0",
|
"torch>=2.0.0",
|
||||||
"sentence-transformers>=2.2.0",
|
"sentence-transformers>=2.2.0",
|
||||||
|
|||||||
@@ -60,3 +60,29 @@ py-modules = []
|
|||||||
leann-core = { path = "packages/leann-core", editable = true }
|
leann-core = { path = "packages/leann-core", editable = true }
|
||||||
leann-backend-diskann = { path = "packages/leann-backend-diskann", editable = true }
|
leann-backend-diskann = { path = "packages/leann-backend-diskann", editable = true }
|
||||||
leann-backend-hnsw = { path = "packages/leann-backend-hnsw", editable = true }
|
leann-backend-hnsw = { path = "packages/leann-backend-hnsw", editable = true }
|
||||||
|
|
||||||
|
[tool.cibuildwheel]
|
||||||
|
# Skip 32-bit and PyPy builds
|
||||||
|
skip = "*-win32 *-manylinux_i686 pp* *musllinux*"
|
||||||
|
|
||||||
|
# Use manylinux_2_35 for Colab compatibility while keeping modern features
|
||||||
|
manylinux-x86_64-image = "manylinux_2_35"
|
||||||
|
manylinux-aarch64-image = "manylinux_2_35"
|
||||||
|
|
||||||
|
# Linux system dependencies
|
||||||
|
[tool.cibuildwheel.linux]
|
||||||
|
before-all = """
|
||||||
|
yum install -y epel-release
|
||||||
|
yum install -y gcc-c++ boost-devel zeromq-devel openblas-devel cmake3 python3-devel
|
||||||
|
ln -sf /usr/bin/cmake3 /usr/bin/cmake
|
||||||
|
"""
|
||||||
|
|
||||||
|
# macOS system dependencies
|
||||||
|
[tool.cibuildwheel.macos]
|
||||||
|
before-all = "brew install boost zeromq openblas cmake libomp"
|
||||||
|
# Set minimum macOS version
|
||||||
|
environment = { MACOSX_DEPLOYMENT_TARGET = "11.0", CMAKE_OSX_DEPLOYMENT_TARGET = "11.0" }
|
||||||
|
|
||||||
|
# Environment variables configuration
|
||||||
|
[tool.cibuildwheel.environment]
|
||||||
|
CMAKE_BUILD_PARALLEL_LEVEL = "8"
|
||||||
|
|||||||
28
uv.lock
generated
28
uv.lock
generated
@@ -1800,7 +1800,7 @@ wheels = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "leann-backend-diskann"
|
name = "leann-backend-diskann"
|
||||||
version = "0.1.0"
|
version = "0.1.8"
|
||||||
source = { editable = "packages/leann-backend-diskann" }
|
source = { editable = "packages/leann-backend-diskann" }
|
||||||
dependencies = [
|
dependencies = [
|
||||||
{ name = "leann-core" },
|
{ name = "leann-core" },
|
||||||
@@ -1810,39 +1810,57 @@ dependencies = [
|
|||||||
|
|
||||||
[package.metadata]
|
[package.metadata]
|
||||||
requires-dist = [
|
requires-dist = [
|
||||||
{ name = "leann-core", specifier = "==0.1.0" },
|
{ name = "leann-core", specifier = "==0.1.8" },
|
||||||
{ name = "numpy" },
|
{ name = "numpy" },
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "leann-backend-hnsw"
|
name = "leann-backend-hnsw"
|
||||||
version = "0.1.0"
|
version = "0.1.8"
|
||||||
source = { editable = "packages/leann-backend-hnsw" }
|
source = { editable = "packages/leann-backend-hnsw" }
|
||||||
dependencies = [
|
dependencies = [
|
||||||
{ name = "leann-core" },
|
{ name = "leann-core" },
|
||||||
|
{ name = "msgpack" },
|
||||||
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
|
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
|
||||||
{ name = "numpy", version = "2.3.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
|
{ name = "numpy", version = "2.3.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
|
||||||
|
{ name = "pyzmq" },
|
||||||
]
|
]
|
||||||
|
|
||||||
[package.metadata]
|
[package.metadata]
|
||||||
requires-dist = [
|
requires-dist = [
|
||||||
{ name = "leann-core", specifier = "==0.1.0" },
|
{ name = "leann-core", specifier = "==0.1.8" },
|
||||||
|
{ name = "msgpack", specifier = ">=1.0.0" },
|
||||||
{ name = "numpy" },
|
{ name = "numpy" },
|
||||||
|
{ name = "pyzmq", specifier = ">=23.0.0" },
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "leann-core"
|
name = "leann-core"
|
||||||
version = "0.1.0"
|
version = "0.1.8"
|
||||||
source = { editable = "packages/leann-core" }
|
source = { editable = "packages/leann-core" }
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
{ name = "llama-index-core" },
|
||||||
|
{ name = "msgpack" },
|
||||||
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
|
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
|
||||||
{ name = "numpy", version = "2.3.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
|
{ name = "numpy", version = "2.3.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
|
||||||
|
{ name = "psutil" },
|
||||||
|
{ name = "python-dotenv" },
|
||||||
|
{ name = "pyzmq" },
|
||||||
|
{ name = "sentence-transformers" },
|
||||||
|
{ name = "torch" },
|
||||||
{ name = "tqdm" },
|
{ name = "tqdm" },
|
||||||
]
|
]
|
||||||
|
|
||||||
[package.metadata]
|
[package.metadata]
|
||||||
requires-dist = [
|
requires-dist = [
|
||||||
|
{ name = "llama-index-core", specifier = ">=0.12.0" },
|
||||||
|
{ name = "msgpack", specifier = ">=1.0.0" },
|
||||||
{ name = "numpy", specifier = ">=1.20.0" },
|
{ name = "numpy", specifier = ">=1.20.0" },
|
||||||
|
{ name = "psutil", specifier = ">=5.8.0" },
|
||||||
|
{ name = "python-dotenv", specifier = ">=1.0.0" },
|
||||||
|
{ name = "pyzmq", specifier = ">=23.0.0" },
|
||||||
|
{ name = "sentence-transformers", specifier = ">=2.2.0" },
|
||||||
|
{ name = "torch", specifier = ">=2.0.0" },
|
||||||
{ name = "tqdm", specifier = ">=4.60.0" },
|
{ name = "tqdm", specifier = ">=4.60.0" },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user