Compare commits
22 Commits
debug/clea
...
fix/manyli
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
971653fa1a | ||
|
|
02672c040d | ||
|
|
f55108feda | ||
|
|
74d485c908 | ||
|
|
d1fefb6378 | ||
|
|
732384f4f8 | ||
|
|
ae38e10d1b | ||
|
|
ca0fd88934 | ||
|
|
3c8d32f156 | ||
|
|
b8ff00fc6a | ||
|
|
3c836766f8 | ||
|
|
b4a1dfb9c7 | ||
|
|
a4d66e95d8 | ||
|
|
cf58b3e31b | ||
|
|
e9c2ca7936 | ||
|
|
dab154a77b | ||
|
|
13413dfae5 | ||
|
|
0543cc9816 | ||
|
|
fb53ed9a0e | ||
|
|
015f43733a | ||
|
|
2957c8bf5a | ||
|
|
a73194c3f6 |
185
.github/workflows/build-cibuildwheel.yml
vendored
185
.github/workflows/build-cibuildwheel.yml
vendored
@@ -16,7 +16,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest]
|
||||
os: [ubuntu-latest] # Focus on Linux/manylinux for Colab compatibility
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -27,118 +27,145 @@ jobs:
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11' # Version for building pure Python packages
|
||||
python-version: '3.11'
|
||||
|
||||
# Build each package separately in our monorepo
|
||||
# Build pure Python packages separately
|
||||
- name: Build pure Python packages (leann-core, leann)
|
||||
if: matrix.os == 'ubuntu-latest' # Only build once, they're platform-independent
|
||||
if: matrix.os == 'ubuntu-latest' # Only build once
|
||||
run: |
|
||||
# Install build tools
|
||||
python -m pip install --upgrade pip build
|
||||
|
||||
# Build pure Python packages
|
||||
python -m build packages/leann-core --outdir wheelhouse/
|
||||
python -m build packages/leann --outdir wheelhouse/
|
||||
|
||||
- name: Build leann-backend-hnsw wheels
|
||||
uses: pypa/cibuildwheel@v2.16.2
|
||||
uses: pypa/cibuildwheel@v2.20.0
|
||||
with:
|
||||
package-dir: packages/leann-backend-hnsw
|
||||
output-dir: wheelhouse
|
||||
env:
|
||||
CIBW_BUILD: cp39-* cp310-* cp311-* cp312-*
|
||||
CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
|
||||
CIBW_SKIP: "*-win32 *-manylinux_i686 pp*"
|
||||
CIBW_BUILD: cp39-* cp310-* cp311-* cp312-* cp313-*
|
||||
CIBW_SKIP: "*-win32 *-manylinux_i686 pp* *musllinux*"
|
||||
|
||||
# Use manylinux_2_35 for Colab compatibility with modern features
|
||||
CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_35
|
||||
CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_35
|
||||
|
||||
# Linux dependencies - using dnf for manylinux_2_35 (based on AlmaLinux 9)
|
||||
CIBW_BEFORE_ALL_LINUX: |
|
||||
yum clean all && yum makecache
|
||||
yum install -y epel-release || true
|
||||
yum makecache || true
|
||||
# Install system dependencies
|
||||
yum install -y \
|
||||
gcc-c++ \
|
||||
boost-devel \
|
||||
protobuf-compiler \
|
||||
protobuf-devel \
|
||||
zeromq-devel \
|
||||
pkgconfig \
|
||||
openblas-devel \
|
||||
cmake || echo "Some packages failed, continuing..."
|
||||
|
||||
# Verify zmq installation and create pkg-config file if needed
|
||||
if [ ! -f /usr/lib64/pkgconfig/libzmq.pc ] && [ ! -f /usr/lib/pkgconfig/libzmq.pc ]; then
|
||||
echo "Creating libzmq.pc file..."
|
||||
mkdir -p /usr/lib64/pkgconfig
|
||||
cat > /usr/lib64/pkgconfig/libzmq.pc << 'EOF'
|
||||
prefix=/usr
|
||||
exec_prefix=${prefix}
|
||||
libdir=${exec_prefix}/lib64
|
||||
includedir=${prefix}/include
|
||||
|
||||
Name: libzmq
|
||||
Description: ZeroMQ library
|
||||
Version: 4.1.4
|
||||
Libs: -L${libdir} -lzmq
|
||||
Cflags: -I${includedir}
|
||||
EOF
|
||||
fi
|
||||
dnf install -y epel-release
|
||||
dnf install -y gcc-c++ boost-devel zeromq-devel openblas-devel cmake python3-devel
|
||||
|
||||
# Install numpy before building
|
||||
CIBW_BEFORE_BUILD: |
|
||||
pip install numpy
|
||||
pip install --upgrade pip setuptools wheel
|
||||
|
||||
CIBW_BEFORE_BUILD_LINUX: |
|
||||
pip install numpy
|
||||
pip install --upgrade pip setuptools wheel swig
|
||||
|
||||
CIBW_BEFORE_ALL_MACOS: |
|
||||
brew install llvm libomp boost protobuf zeromq
|
||||
brew install boost zeromq openblas cmake libomp
|
||||
|
||||
# Pre-install test dependencies to avoid compilation
|
||||
CIBW_BEFORE_TEST: |
|
||||
pip install --only-binary :all: "pyzmq>=23.0.0"
|
||||
|
||||
# Test command to verify the wheel works
|
||||
CIBW_TEST_COMMAND: |
|
||||
python -c "import leann_backend_hnsw; print('HNSW backend imported successfully')"
|
||||
|
||||
# Skip problematic configurations
|
||||
CIBW_TEST_SKIP: "*-macosx_arm64" # Skip ARM64 tests on GitHub Actions
|
||||
|
||||
# Test dependencies
|
||||
CIBW_TEST_REQUIRES: "pytest numpy"
|
||||
|
||||
# Environment variables for build
|
||||
CIBW_ENVIRONMENT: |
|
||||
CMAKE_BUILD_PARALLEL_LEVEL=8
|
||||
Python_FIND_VIRTUALENV=ONLY
|
||||
Python3_FIND_VIRTUALENV=ONLY
|
||||
|
||||
# Linux-specific environment variables
|
||||
CIBW_ENVIRONMENT_LINUX: |
|
||||
PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH
|
||||
CMAKE_BUILD_PARALLEL_LEVEL=8
|
||||
|
||||
# macOS-specific environment variables
|
||||
CIBW_ENVIRONMENT_MACOS: |
|
||||
CC=$(brew --prefix llvm)/bin/clang
|
||||
CXX=$(brew --prefix llvm)/bin/clang++
|
||||
|
||||
CIBW_TEST_REQUIRES: leann-core numpy pyzmq msgpack
|
||||
CIBW_TEST_COMMAND: python -c "import leann_backend_hnsw"
|
||||
CMAKE_BUILD_PARALLEL_LEVEL=8
|
||||
MACOSX_DEPLOYMENT_TARGET=11.0
|
||||
CMAKE_OSX_DEPLOYMENT_TARGET=11.0
|
||||
Python_FIND_VIRTUALENV=ONLY
|
||||
Python3_FIND_VIRTUALENV=ONLY
|
||||
|
||||
- name: Build leann-backend-diskann wheels
|
||||
uses: pypa/cibuildwheel@v2.16.2
|
||||
- name: Build leann-backend-diskann wheels
|
||||
uses: pypa/cibuildwheel@v2.20.0
|
||||
with:
|
||||
package-dir: packages/leann-backend-diskann
|
||||
output-dir: wheelhouse
|
||||
env:
|
||||
CIBW_BUILD: cp39-* cp310-* cp311-* cp312-*
|
||||
CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
|
||||
CIBW_SKIP: "*-win32 *-manylinux_i686 pp*"
|
||||
CIBW_BUILD: cp39-* cp310-* cp311-* cp312-* cp313-*
|
||||
CIBW_SKIP: "*-win32 *-manylinux_i686 pp* *musllinux*"
|
||||
|
||||
CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_35
|
||||
CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_35
|
||||
|
||||
CIBW_BEFORE_ALL_LINUX: |
|
||||
yum clean all && yum makecache
|
||||
yum install -y epel-release || true
|
||||
yum makecache || true
|
||||
# Install system dependencies for DiskANN
|
||||
yum install -y \
|
||||
gcc-c++ \
|
||||
protobuf-compiler \
|
||||
protobuf-devel \
|
||||
openblas-devel \
|
||||
pkgconfig \
|
||||
cmake || echo "Some packages failed, continuing..."
|
||||
yum install -y libaio-devel || echo "libaio-devel not available, continuing..."
|
||||
dnf install -y epel-release
|
||||
dnf install -y gcc-c++ boost-devel zeromq-devel openblas-devel cmake python3-devel
|
||||
|
||||
# Install numpy before building
|
||||
CIBW_BEFORE_BUILD: |
|
||||
pip install numpy
|
||||
pip install --upgrade pip setuptools wheel
|
||||
|
||||
CIBW_BEFORE_BUILD_LINUX: |
|
||||
pip install numpy
|
||||
pip install --upgrade pip setuptools wheel swig
|
||||
|
||||
CIBW_BEFORE_ALL_MACOS: |
|
||||
brew install llvm libomp protobuf
|
||||
brew install boost zeromq openblas cmake libomp
|
||||
|
||||
# Pre-install test dependencies to avoid compilation
|
||||
CIBW_BEFORE_TEST: |
|
||||
pip install --only-binary :all: "pyzmq>=23.0.0"
|
||||
|
||||
# Test command to verify the wheel works
|
||||
CIBW_TEST_COMMAND: |
|
||||
python -c "import leann_backend_diskann; print('DiskANN backend imported successfully')"
|
||||
|
||||
# Skip problematic configurations
|
||||
CIBW_TEST_SKIP: "*-macosx_arm64" # Skip ARM64 tests on GitHub Actions
|
||||
|
||||
# Test dependencies - avoid pyzmq due to manylinux2014 compatibility issues
|
||||
CIBW_TEST_REQUIRES: "pytest numpy"
|
||||
|
||||
CIBW_ENVIRONMENT: |
|
||||
CMAKE_BUILD_PARALLEL_LEVEL=8
|
||||
Python_FIND_VIRTUALENV=ONLY
|
||||
Python3_FIND_VIRTUALENV=ONLY
|
||||
|
||||
# Linux-specific environment variables
|
||||
CIBW_ENVIRONMENT_LINUX: |
|
||||
PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH
|
||||
CMAKE_BUILD_PARALLEL_LEVEL=8
|
||||
CMAKE_PREFIX_PATH=$VIRTUAL_ENV
|
||||
Python_FIND_VIRTUALENV=ONLY
|
||||
Python3_FIND_VIRTUALENV=ONLY
|
||||
Python_FIND_STRATEGY=LOCATION
|
||||
Python3_FIND_STRATEGY=LOCATION
|
||||
Python_EXECUTABLE=$VIRTUAL_ENV/bin/python
|
||||
Python3_EXECUTABLE=$VIRTUAL_ENV/bin/python
|
||||
|
||||
# macOS-specific environment variables
|
||||
CIBW_ENVIRONMENT_MACOS: |
|
||||
CC=$(brew --prefix llvm)/bin/clang
|
||||
CXX=$(brew --prefix llvm)/bin/clang++
|
||||
|
||||
CIBW_TEST_REQUIRES: leann-core numpy
|
||||
CIBW_TEST_COMMAND: python -c "import leann_backend_diskann"
|
||||
|
||||
- name: List built packages
|
||||
run: |
|
||||
echo "📦 Built packages:"
|
||||
ls -la wheelhouse/
|
||||
CMAKE_BUILD_PARALLEL_LEVEL=8
|
||||
MACOSX_DEPLOYMENT_TARGET=11.0
|
||||
CMAKE_OSX_DEPLOYMENT_TARGET=11.0
|
||||
Python_FIND_VIRTUALENV=ONLY
|
||||
Python3_FIND_VIRTUALENV=ONLY
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: cibw-wheels-${{ matrix.os }}
|
||||
name: wheels-${{ matrix.os }}
|
||||
path: ./wheelhouse/*.whl
|
||||
12
.github/workflows/ci-cibuildwheel.yml
vendored
12
.github/workflows/ci-cibuildwheel.yml
vendored
@@ -1,12 +0,0 @@
|
||||
name: CI - cibuildwheel (Test)
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main ]
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
workflow_dispatch: # Allow manual triggering
|
||||
|
||||
jobs:
|
||||
build:
|
||||
uses: ./.github/workflows/build-cibuildwheel.yml
|
||||
60
.github/workflows/test-manylinux.yml
vendored
Normal file
60
.github/workflows/test-manylinux.yml
vendored
Normal file
@@ -0,0 +1,60 @@
|
||||
name: Test Manylinux Build
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
paths:
|
||||
- '.github/workflows/**'
|
||||
- 'packages/**'
|
||||
- 'pyproject.toml'
|
||||
push:
|
||||
branches:
|
||||
- 'fix/manylinux-*'
|
||||
- 'test/build-*'
|
||||
|
||||
jobs:
|
||||
build:
|
||||
uses: ./.github/workflows/build-cibuildwheel.yml
|
||||
|
||||
test-install:
|
||||
needs: build
|
||||
runs-on: ubuntu-22.04 # Simulating Colab environment
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ['3.10', '3.11', '3.12']
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Download artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: wheels-*
|
||||
path: dist
|
||||
merge-multiple: true
|
||||
|
||||
- name: Test installation
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
# Find and install the appropriate wheels
|
||||
pip install dist/leann_core-*.whl
|
||||
pip install dist/leann_backend_hnsw-*manylinux*.whl
|
||||
pip install dist/leann-*.whl
|
||||
|
||||
- name: Test import
|
||||
run: |
|
||||
python -c "
|
||||
import leann
|
||||
from leann import LeannBuilder, LeannSearcher
|
||||
print('Successfully imported leann modules')
|
||||
|
||||
# Quick functionality test
|
||||
builder = LeannBuilder(backend_name='hnsw')
|
||||
builder.add_text('Test document')
|
||||
print('LeannBuilder created and used successfully')
|
||||
"
|
||||
50
MANYLINUX_BUILD_STRATEGY.md
Normal file
50
MANYLINUX_BUILD_STRATEGY.md
Normal file
@@ -0,0 +1,50 @@
|
||||
# Manylinux Build Strategy
|
||||
|
||||
## Problem
|
||||
Google Colab requires wheels compatible with `manylinux_2_35_x86_64` or earlier. Our previous builds were producing `manylinux_2_39_x86_64` wheels, which are incompatible.
|
||||
|
||||
## Solution
|
||||
We're using `cibuildwheel` with `manylinux_2_35` images to build wheels that are compatible with Google Colab while maintaining modern toolchain features.
|
||||
|
||||
### Key Changes
|
||||
|
||||
1. **cibuildwheel Configuration**
|
||||
- Using `manylinux2014` images (provides `manylinux_2_17` compatibility)
|
||||
- Using `yum` package manager (CentOS 7 based)
|
||||
- Installing `cmake3` and creating symlink for compatibility
|
||||
|
||||
2. **Build Matrix**
|
||||
- Python versions: 3.9, 3.10, 3.11, 3.12, 3.13
|
||||
- Platforms: Linux (x86_64), macOS
|
||||
- No Windows support (not required)
|
||||
|
||||
3. **Dependencies**
|
||||
- Linux: gcc-c++, boost-devel, zeromq-devel, openblas-devel, cmake3
|
||||
- macOS: boost, zeromq, openblas, cmake (via Homebrew)
|
||||
|
||||
4. **Environment Variables**
|
||||
- `CMAKE_BUILD_PARALLEL_LEVEL=8`: Speed up builds
|
||||
- `Python_FIND_VIRTUALENV=ONLY`: Help CMake find Python in cibuildwheel env
|
||||
- `Python3_FIND_VIRTUALENV=ONLY`: Alternative variable for compatibility
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
1. **CI Pipeline**: `test-manylinux.yml`
|
||||
- Triggers on PR to main, manual dispatch, or push to `fix/manylinux-*` branches
|
||||
- Builds wheels using cibuildwheel
|
||||
- Tests installation on Ubuntu 22.04 (simulating Colab)
|
||||
|
||||
2. **Local Testing**
|
||||
```bash
|
||||
# Download built wheels
|
||||
# Test in fresh environment
|
||||
python -m venv test_env
|
||||
source test_env/bin/activate
|
||||
pip install leann_core-*.whl leann_backend_hnsw-*manylinux*.whl leann-*.whl
|
||||
python -c "from leann import LeannBuilder; print('Success!')"
|
||||
```
|
||||
|
||||
## References
|
||||
- [cibuildwheel documentation](https://cibuildwheel.readthedocs.io/)
|
||||
- [manylinux standards](https://github.com/pypa/manylinux)
|
||||
- [PEP 599 - manylinux2014](https://peps.python.org/pep-0599/)
|
||||
@@ -3,6 +3,34 @@
|
||||
cmake_minimum_required(VERSION 3.20)
|
||||
project(leann_backend_diskann_wrapper)
|
||||
|
||||
# Find Python - scikit-build-core should provide this
|
||||
find_package(Python REQUIRED COMPONENTS Interpreter Development.Module)
|
||||
|
||||
# Print Python information for debugging
|
||||
message(STATUS "Python_FOUND: ${Python_FOUND}")
|
||||
message(STATUS "Python_VERSION: ${Python_VERSION}")
|
||||
message(STATUS "Python_EXECUTABLE: ${Python_EXECUTABLE}")
|
||||
message(STATUS "Python_INCLUDE_DIRS: ${Python_INCLUDE_DIRS}")
|
||||
|
||||
# Pass Python information to DiskANN through cache variables
|
||||
set(Python_EXECUTABLE ${Python_EXECUTABLE} CACHE FILEPATH "Python executable" FORCE)
|
||||
set(Python_INCLUDE_DIRS ${Python_INCLUDE_DIRS} CACHE PATH "Python include dirs" FORCE)
|
||||
set(Python_LIBRARIES ${Python_LIBRARIES} CACHE FILEPATH "Python libraries" FORCE)
|
||||
set(Python_VERSION ${Python_VERSION} CACHE STRING "Python version" FORCE)
|
||||
set(Python_FOUND ${Python_FOUND} CACHE BOOL "Python found" FORCE)
|
||||
|
||||
# Also set Python3 variables for compatibility
|
||||
set(Python3_EXECUTABLE ${Python_EXECUTABLE} CACHE FILEPATH "Python3 executable" FORCE)
|
||||
set(Python3_INCLUDE_DIRS ${Python_INCLUDE_DIRS} CACHE PATH "Python3 include dirs" FORCE)
|
||||
set(Python3_LIBRARIES ${Python_LIBRARIES} CACHE FILEPATH "Python3 libraries" FORCE)
|
||||
set(Python3_VERSION ${Python_VERSION} CACHE STRING "Python3 version" FORCE)
|
||||
set(Python3_FOUND ${Python_FOUND} CACHE BOOL "Python3 found" FORCE)
|
||||
set(Python3_Development_FOUND TRUE CACHE BOOL "Python3 development found" FORCE)
|
||||
|
||||
# Set Python finding strategy
|
||||
set(Python_FIND_VIRTUALENV ONLY CACHE STRING "" FORCE)
|
||||
set(Python3_FIND_VIRTUALENV ONLY CACHE STRING "" FORCE)
|
||||
|
||||
# Tell CMake to directly enter the DiskANN submodule and execute its own CMakeLists.txt
|
||||
# DiskANN will handle everything itself, including compiling Python bindings
|
||||
add_subdirectory(src/third_party/DiskANN)
|
||||
add_subdirectory(third_party/DiskANN)
|
||||
|
||||
@@ -16,4 +16,9 @@ wheel.packages = ["leann_backend_diskann"]
|
||||
editable.mode = "redirect"
|
||||
cmake.build-type = "Release"
|
||||
build.verbose = true
|
||||
build.tool-args = ["-j8"]
|
||||
build.tool-args = ["-j8"]
|
||||
wheel.exclude = ["CMakeLists.txt", "src", "third_party/**", "*.o", "*.so"]
|
||||
sdist.include = ["CMakeLists.txt", "src", "third_party", "leann_backend_diskann/*.txt"]
|
||||
|
||||
[tool.scikit-build.cmake.define]
|
||||
CMAKE_BUILD_PARALLEL_LEVEL = "8"
|
||||
Submodule packages/leann-backend-diskann/third_party/DiskANN updated: af2a26481e...81666b7758
@@ -24,6 +24,38 @@ set(MSGPACK_USE_BOOST OFF CACHE BOOL "" FORCE)
|
||||
add_compile_definitions(MSGPACK_NO_BOOST)
|
||||
include_directories(third_party/msgpack-c/include)
|
||||
|
||||
# Find Python for our own use (not for Faiss)
|
||||
if(DEFINED SKBUILD)
|
||||
message(STATUS "Building with scikit-build")
|
||||
# scikit-build-core provides Python information
|
||||
endif()
|
||||
|
||||
# Find Python - scikit-build-core should provide this
|
||||
find_package(Python REQUIRED COMPONENTS Interpreter Development.Module NumPy)
|
||||
|
||||
# Print Python information for debugging
|
||||
message(STATUS "Python_FOUND: ${Python_FOUND}")
|
||||
message(STATUS "Python_VERSION: ${Python_VERSION}")
|
||||
message(STATUS "Python_EXECUTABLE: ${Python_EXECUTABLE}")
|
||||
message(STATUS "Python_INCLUDE_DIRS: ${Python_INCLUDE_DIRS}")
|
||||
message(STATUS "Python_NumPy_INCLUDE_DIRS: ${Python_NumPy_INCLUDE_DIRS}")
|
||||
|
||||
# Pass Python information to faiss through cache variables
|
||||
set(Python_EXECUTABLE ${Python_EXECUTABLE} CACHE FILEPATH "Python executable" FORCE)
|
||||
set(Python_INCLUDE_DIRS ${Python_INCLUDE_DIRS} CACHE PATH "Python include dirs" FORCE)
|
||||
set(Python_NumPy_INCLUDE_DIRS ${Python_NumPy_INCLUDE_DIRS} CACHE PATH "NumPy include dirs" FORCE)
|
||||
set(Python_VERSION ${Python_VERSION} CACHE STRING "Python version" FORCE)
|
||||
set(Python_FOUND ${Python_FOUND} CACHE BOOL "Python found" FORCE)
|
||||
|
||||
# Also set Python3 variables for compatibility
|
||||
set(Python3_EXECUTABLE ${Python_EXECUTABLE} CACHE FILEPATH "Python3 executable" FORCE)
|
||||
set(Python3_INCLUDE_DIRS ${Python_INCLUDE_DIRS} CACHE PATH "Python3 include dirs" FORCE)
|
||||
set(Python3_NumPy_INCLUDE_DIRS ${Python_NumPy_INCLUDE_DIRS} CACHE PATH "NumPy include dirs" FORCE)
|
||||
set(Python3_VERSION ${Python_VERSION} CACHE STRING "Python3 version" FORCE)
|
||||
set(Python3_FOUND ${Python_FOUND} CACHE BOOL "Python3 found" FORCE)
|
||||
set(Python3_Development_FOUND TRUE CACHE BOOL "Python3 development found" FORCE)
|
||||
set(Python3_NumPy_FOUND TRUE CACHE BOOL "Python3 NumPy found" FORCE)
|
||||
|
||||
# Faiss configuration - streamlined build
|
||||
set(FAISS_ENABLE_PYTHON ON CACHE BOOL "" FORCE)
|
||||
set(FAISS_ENABLE_GPU OFF CACHE BOOL "" FORCE)
|
||||
@@ -52,4 +84,8 @@ set(FAISS_BUILD_AVX512 OFF CACHE BOOL "" FORCE)
|
||||
# IMPORTANT: Disable building AVX versions to speed up compilation
|
||||
set(FAISS_BUILD_AVX_VERSIONS OFF CACHE BOOL "" FORCE)
|
||||
|
||||
# Force Faiss to use our Python settings
|
||||
set(Python_FIND_VIRTUALENV ONLY CACHE STRING "" FORCE)
|
||||
set(Python3_FIND_VIRTUALENV ONLY CACHE STRING "" FORCE)
|
||||
|
||||
add_subdirectory(third_party/faiss)
|
||||
@@ -16,12 +16,14 @@ dependencies = [
|
||||
]
|
||||
|
||||
[tool.scikit-build]
|
||||
wheel.packages = ["leann_backend_hnsw"]
|
||||
editable.mode = "redirect"
|
||||
cmake.build-type = "Release"
|
||||
build.verbose = true
|
||||
build.tool-args = ["-j8"]
|
||||
wheel.exclude = ["CMakeLists.txt", "src", "third_party"]
|
||||
sdist.include = ["CMakeLists.txt", "src", "third_party", "leann_backend_hnsw/*.txt"]
|
||||
cmake.args = ["-DCMAKE_BUILD_TYPE=Release"]
|
||||
# Ensure CMake can find system libraries
|
||||
build-dir = "build/{cache_tag}"
|
||||
minimum-version = "build-system.requires"
|
||||
|
||||
# CMake definitions to optimize compilation
|
||||
[tool.scikit-build.cmake.define]
|
||||
CMAKE_BUILD_PARALLEL_LEVEL = "8"
|
||||
CMAKE_BUILD_PARALLEL_LEVEL = "8"
|
||||
SKBUILD_SOABI = "YES"
|
||||
Submodule packages/leann-backend-hnsw/third_party/faiss updated: ff22e2c86b...553f937220
@@ -15,7 +15,7 @@ dependencies = [
|
||||
"numpy>=1.20.0",
|
||||
"tqdm>=4.60.0",
|
||||
"psutil>=5.8.0",
|
||||
"pyzmq>=23.0.0",
|
||||
"pyzmq>=23.0.0,<27", # Cap at 26.x for manylinux2014 compatibility
|
||||
"msgpack>=1.0.0",
|
||||
"torch>=2.0.0",
|
||||
"sentence-transformers>=2.2.0",
|
||||
|
||||
@@ -60,3 +60,29 @@ py-modules = []
|
||||
leann-core = { path = "packages/leann-core", editable = true }
|
||||
leann-backend-diskann = { path = "packages/leann-backend-diskann", editable = true }
|
||||
leann-backend-hnsw = { path = "packages/leann-backend-hnsw", editable = true }
|
||||
|
||||
[tool.cibuildwheel]
|
||||
# Skip 32-bit and PyPy builds
|
||||
skip = "*-win32 *-manylinux_i686 pp* *musllinux*"
|
||||
|
||||
# Use manylinux_2_35 for Colab compatibility while keeping modern features
|
||||
manylinux-x86_64-image = "manylinux_2_35"
|
||||
manylinux-aarch64-image = "manylinux_2_35"
|
||||
|
||||
# Linux system dependencies
|
||||
[tool.cibuildwheel.linux]
|
||||
before-all = """
|
||||
yum install -y epel-release
|
||||
yum install -y gcc-c++ boost-devel zeromq-devel openblas-devel cmake3 python3-devel
|
||||
ln -sf /usr/bin/cmake3 /usr/bin/cmake
|
||||
"""
|
||||
|
||||
# macOS system dependencies
|
||||
[tool.cibuildwheel.macos]
|
||||
before-all = "brew install boost zeromq openblas cmake libomp"
|
||||
# Set minimum macOS version
|
||||
environment = { MACOSX_DEPLOYMENT_TARGET = "11.0", CMAKE_OSX_DEPLOYMENT_TARGET = "11.0" }
|
||||
|
||||
# Environment variables configuration
|
||||
[tool.cibuildwheel.environment]
|
||||
CMAKE_BUILD_PARALLEL_LEVEL = "8"
|
||||
|
||||
28
uv.lock
generated
28
uv.lock
generated
@@ -1800,7 +1800,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "leann-backend-diskann"
|
||||
version = "0.1.0"
|
||||
version = "0.1.8"
|
||||
source = { editable = "packages/leann-backend-diskann" }
|
||||
dependencies = [
|
||||
{ name = "leann-core" },
|
||||
@@ -1810,39 +1810,57 @@ dependencies = [
|
||||
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "leann-core", specifier = "==0.1.0" },
|
||||
{ name = "leann-core", specifier = "==0.1.8" },
|
||||
{ name = "numpy" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "leann-backend-hnsw"
|
||||
version = "0.1.0"
|
||||
version = "0.1.8"
|
||||
source = { editable = "packages/leann-backend-hnsw" }
|
||||
dependencies = [
|
||||
{ name = "leann-core" },
|
||||
{ name = "msgpack" },
|
||||
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
|
||||
{ name = "numpy", version = "2.3.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
|
||||
{ name = "pyzmq" },
|
||||
]
|
||||
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "leann-core", specifier = "==0.1.0" },
|
||||
{ name = "leann-core", specifier = "==0.1.8" },
|
||||
{ name = "msgpack", specifier = ">=1.0.0" },
|
||||
{ name = "numpy" },
|
||||
{ name = "pyzmq", specifier = ">=23.0.0" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "leann-core"
|
||||
version = "0.1.0"
|
||||
version = "0.1.8"
|
||||
source = { editable = "packages/leann-core" }
|
||||
dependencies = [
|
||||
{ name = "llama-index-core" },
|
||||
{ name = "msgpack" },
|
||||
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
|
||||
{ name = "numpy", version = "2.3.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
|
||||
{ name = "psutil" },
|
||||
{ name = "python-dotenv" },
|
||||
{ name = "pyzmq" },
|
||||
{ name = "sentence-transformers" },
|
||||
{ name = "torch" },
|
||||
{ name = "tqdm" },
|
||||
]
|
||||
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "llama-index-core", specifier = ">=0.12.0" },
|
||||
{ name = "msgpack", specifier = ">=1.0.0" },
|
||||
{ name = "numpy", specifier = ">=1.20.0" },
|
||||
{ name = "psutil", specifier = ">=5.8.0" },
|
||||
{ name = "python-dotenv", specifier = ">=1.0.0" },
|
||||
{ name = "pyzmq", specifier = ">=23.0.0" },
|
||||
{ name = "sentence-transformers", specifier = ">=2.2.0" },
|
||||
{ name = "torch", specifier = ">=2.0.0" },
|
||||
{ name = "tqdm", specifier = ">=4.60.0" },
|
||||
]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user