diff --git a/.github/workflows/build-reusable.yml b/.github/workflows/build-reusable.yml index a6a220d..700b51e 100644 --- a/.github/workflows/build-reusable.yml +++ b/.github/workflows/build-reusable.yml @@ -59,16 +59,26 @@ jobs: python: '3.12' - os: ubuntu-22.04 python: '3.13' - - os: macos-latest + - os: macos-14 python: '3.9' - - os: macos-latest + - os: macos-14 python: '3.10' - - os: macos-latest + - os: macos-14 python: '3.11' - - os: macos-latest + - os: macos-14 python: '3.12' - - os: macos-latest + - os: macos-14 python: '3.13' + - os: macos-13 + python: '3.9' + - os: macos-13 + python: '3.10' + - os: macos-13 + python: '3.11' + - os: macos-13 + python: '3.12' + # Note: macos-13 + Python 3.13 excluded due to PyTorch compatibility + # (PyTorch 2.5+ supports Python 3.13 but not Intel Mac x86_64) runs-on: ${{ matrix.os }} steps: @@ -114,6 +124,21 @@ jobs: uv pip install --system delocate fi + - name: Set macOS environment variables + if: runner.os == 'macOS' + run: | + # Use brew --prefix to automatically detect Homebrew installation path + HOMEBREW_PREFIX=$(brew --prefix) + echo "HOMEBREW_PREFIX=${HOMEBREW_PREFIX}" >> $GITHUB_ENV + echo "OpenMP_ROOT=${HOMEBREW_PREFIX}/opt/libomp" >> $GITHUB_ENV + + # Set CMAKE_PREFIX_PATH to let CMake find all packages automatically + echo "CMAKE_PREFIX_PATH=${HOMEBREW_PREFIX}" >> $GITHUB_ENV + + # Set compiler flags for OpenMP (required for both backends) + echo "LDFLAGS=-L${HOMEBREW_PREFIX}/opt/libomp/lib" >> $GITHUB_ENV + echo "CPPFLAGS=-I${HOMEBREW_PREFIX}/opt/libomp/include" >> $GITHUB_ENV + - name: Build packages run: | # Build core (platform independent) on all platforms for consistency @@ -123,28 +148,28 @@ jobs: # Build HNSW backend cd packages/leann-backend-hnsw - if [ "${{ matrix.os }}" == "macos-latest" ]; then - # Use system clang instead of homebrew LLVM for better compatibility + if [[ "${{ matrix.os }}" == macos-* ]]; then + # Use system clang for better compatibility export CC=clang export CXX=clang++ export MACOSX_DEPLOYMENT_TARGET=11.0 - uv build --wheel --python python + uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist else - uv build --wheel --python python + uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist fi cd ../.. # Build DiskANN backend cd packages/leann-backend-diskann - if [ "${{ matrix.os }}" == "macos-latest" ]; then - # Use system clang instead of homebrew LLVM for better compatibility + if [[ "${{ matrix.os }}" == macos-* ]]; then + # Use system clang for better compatibility export CC=clang export CXX=clang++ # sgesdd_ is only available on macOS 13.3+ export MACOSX_DEPLOYMENT_TARGET=13.3 - uv build --wheel --python python + uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist else - uv build --wheel --python python + uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist fi cd ../.. @@ -205,19 +230,18 @@ jobs: echo "📦 Built packages:" find packages/*/dist -name "*.whl" -o -name "*.tar.gz" | sort + - name: Install built packages for testing run: | # Create a virtual environment with the correct Python version - uv venv --python python${{ matrix.python }} + uv venv --python ${{ matrix.python }} source .venv/bin/activate || source .venv/Scripts/activate - # Install the built wheels directly to ensure we use locally built packages - # Use only locally built wheels on all platforms for full consistency - FIND_LINKS="--find-links packages/leann-core/dist --find-links packages/leann/dist" - FIND_LINKS="$FIND_LINKS --find-links packages/leann-backend-hnsw/dist --find-links packages/leann-backend-diskann/dist" - - uv pip install leann-core leann leann-backend-hnsw leann-backend-diskann \ - $FIND_LINKS --force-reinstall + # Install packages using --find-links to prioritize local builds + uv pip install --find-links packages/leann-core/dist --find-links packages/leann-backend-hnsw/dist --find-links packages/leann-backend-diskann/dist packages/leann-core/dist/*.whl || uv pip install --find-links packages/leann-core/dist packages/leann-core/dist/*.tar.gz + uv pip install --find-links packages/leann-core/dist packages/leann-backend-hnsw/dist/*.whl + uv pip install --find-links packages/leann-core/dist packages/leann-backend-diskann/dist/*.whl + uv pip install packages/leann/dist/*.whl || uv pip install packages/leann/dist/*.tar.gz # Install test dependencies using extras uv pip install -e ".[test]" diff --git a/README.md b/README.md index 68b2810..12802f3 100755 --- a/README.md +++ b/README.md @@ -3,10 +3,11 @@

- Python 3.9+ + Python Versions + CI Status + Platform MIT License - Platform - MCP Integration + MCP Integration

diff --git a/packages/leann-backend-diskann/CMakeLists.txt b/packages/leann-backend-diskann/CMakeLists.txt deleted file mode 100644 index 2638282..0000000 --- a/packages/leann-backend-diskann/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# packages/leann-backend-diskann/CMakeLists.txt (simplified version) - -cmake_minimum_required(VERSION 3.20) -project(leann_backend_diskann_wrapper) - -# Tell CMake to directly enter the DiskANN submodule and execute its own CMakeLists.txt -# DiskANN will handle everything itself, including compiling Python bindings -add_subdirectory(src/third_party/DiskANN) diff --git a/packages/leann-backend-diskann/pyproject.toml b/packages/leann-backend-diskann/pyproject.toml index 6b33379..055a1e7 100644 --- a/packages/leann-backend-diskann/pyproject.toml +++ b/packages/leann-backend-diskann/pyproject.toml @@ -17,3 +17,5 @@ editable.mode = "redirect" cmake.build-type = "Release" build.verbose = true build.tool-args = ["-j8"] +# Let CMake find packages via Homebrew prefix +cmake.define = {CMAKE_PREFIX_PATH = {env = "CMAKE_PREFIX_PATH"}, OpenMP_ROOT = {env = "OpenMP_ROOT"}} diff --git a/packages/leann-backend-diskann/third_party/DiskANN b/packages/leann-backend-diskann/third_party/DiskANN index b2dc4ea..04048bb 160000 --- a/packages/leann-backend-diskann/third_party/DiskANN +++ b/packages/leann-backend-diskann/third_party/DiskANN @@ -1 +1 @@ -Subproject commit b2dc4ea2c7e52e8a6481d3ba10003e192192a7b7 +Subproject commit 04048bb302a5d032371326a6526ff5410f8ffdab diff --git a/packages/leann-backend-hnsw/CMakeLists.txt b/packages/leann-backend-hnsw/CMakeLists.txt index 1f41393..651792c 100644 --- a/packages/leann-backend-hnsw/CMakeLists.txt +++ b/packages/leann-backend-hnsw/CMakeLists.txt @@ -5,11 +5,20 @@ set(CMAKE_CXX_COMPILER_WORKS 1) # Set OpenMP path for macOS if(APPLE) - set(OpenMP_C_FLAGS "-Xpreprocessor -fopenmp -I/opt/homebrew/opt/libomp/include") - set(OpenMP_CXX_FLAGS "-Xpreprocessor -fopenmp -I/opt/homebrew/opt/libomp/include") + # Detect Homebrew installation path (Apple Silicon vs Intel) + if(EXISTS "/opt/homebrew/opt/libomp") + set(HOMEBREW_PREFIX "/opt/homebrew") + elseif(EXISTS "/usr/local/opt/libomp") + set(HOMEBREW_PREFIX "/usr/local") + else() + message(FATAL_ERROR "Could not find libomp installation. Please install with: brew install libomp") + endif() + + set(OpenMP_C_FLAGS "-Xpreprocessor -fopenmp -I${HOMEBREW_PREFIX}/opt/libomp/include") + set(OpenMP_CXX_FLAGS "-Xpreprocessor -fopenmp -I${HOMEBREW_PREFIX}/opt/libomp/include") set(OpenMP_C_LIB_NAMES "omp") set(OpenMP_CXX_LIB_NAMES "omp") - set(OpenMP_omp_LIBRARY "/opt/homebrew/opt/libomp/lib/libomp.dylib") + set(OpenMP_omp_LIBRARY "${HOMEBREW_PREFIX}/opt/libomp/lib/libomp.dylib") # Force use of system libc++ to avoid version mismatch set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") diff --git a/packages/leann-backend-hnsw/pyproject.toml b/packages/leann-backend-hnsw/pyproject.toml index d178c11..c3657e6 100644 --- a/packages/leann-backend-hnsw/pyproject.toml +++ b/packages/leann-backend-hnsw/pyproject.toml @@ -22,6 +22,8 @@ cmake.build-type = "Release" build.verbose = true build.tool-args = ["-j8"] -# CMake definitions to optimize compilation +# CMake definitions to optimize compilation and find Homebrew packages [tool.scikit-build.cmake.define] CMAKE_BUILD_PARALLEL_LEVEL = "8" +CMAKE_PREFIX_PATH = {env = "CMAKE_PREFIX_PATH"} +OpenMP_ROOT = {env = "OpenMP_ROOT"} diff --git a/packages/leann-backend-hnsw/third_party/faiss b/packages/leann-backend-hnsw/third_party/faiss index ff22e2c..4a2c0d6 160000 --- a/packages/leann-backend-hnsw/third_party/faiss +++ b/packages/leann-backend-hnsw/third_party/faiss @@ -1 +1 @@ -Subproject commit ff22e2c86be1784c760265abe146b1ab0db90ebe +Subproject commit 4a2c0d67d37a6f27c9a1cd695a3d703dcce73bad diff --git a/packages/leann-core/pyproject.toml b/packages/leann-core/pyproject.toml index db7259b..98e7d12 100644 --- a/packages/leann-core/pyproject.toml +++ b/packages/leann-core/pyproject.toml @@ -33,8 +33,8 @@ dependencies = [ "pdfplumber>=0.10.0", "nbconvert>=7.0.0", # For .ipynb file support "gitignore-parser>=0.1.12", # For proper .gitignore handling - "mlx>=0.26.3; sys_platform == 'darwin'", - "mlx-lm>=0.26.0; sys_platform == 'darwin'", + "mlx>=0.26.3; sys_platform == 'darwin' and platform_machine == 'arm64'", + "mlx-lm>=0.26.0; sys_platform == 'darwin' and platform_machine == 'arm64'", ] [project.optional-dependencies] diff --git a/packages/leann-core/src/leann/cli.py b/packages/leann-core/src/leann/cli.py index c8f4c4d..31dca55 100644 --- a/packages/leann-core/src/leann/cli.py +++ b/packages/leann-core/src/leann/cli.py @@ -1,6 +1,7 @@ import argparse import asyncio from pathlib import Path +from typing import Union from llama_index.core import SimpleDirectoryReader from llama_index.core.node_parser import SentenceSplitter @@ -310,7 +311,7 @@ Examples: print(f' leann search {example_name} "your query"') print(f" leann ask {example_name} --interactive") - def load_documents(self, docs_dir: str, custom_file_types: str | None = None): + def load_documents(self, docs_dir: str, custom_file_types: Union[str, None] = None): print(f"Loading documents from {docs_dir}...") if custom_file_types: print(f"Using custom file types: {custom_file_types}") diff --git a/pyproject.toml b/pyproject.toml index 1163d32..73dd7a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,8 +40,8 @@ dependencies = [ # Other dependencies "ipykernel==6.29.5", "msgpack>=1.1.1", - "mlx>=0.26.3; sys_platform == 'darwin'", - "mlx-lm>=0.26.0; sys_platform == 'darwin'", + "mlx>=0.26.3; sys_platform == 'darwin' and platform_machine == 'arm64'", + "mlx-lm>=0.26.0; sys_platform == 'darwin' and platform_machine == 'arm64'", "psutil>=5.8.0", "pybind11>=3.0.0", "pathspec>=0.12.1",