diff --git a/.github/workflows/build-reusable.yml b/.github/workflows/build-reusable.yml index f0b7b24..d0bfb16 100644 --- a/.github/workflows/build-reusable.yml +++ b/.github/workflows/build-reusable.yml @@ -54,6 +54,17 @@ jobs: python: '3.12' - os: ubuntu-22.04 python: '3.13' + # ARM64 Linux builds + - os: ubuntu-24.04-arm + python: '3.9' + - os: ubuntu-24.04-arm + python: '3.10' + - os: ubuntu-24.04-arm + python: '3.11' + - os: ubuntu-24.04-arm + python: '3.12' + - os: ubuntu-24.04-arm + python: '3.13' - os: macos-14 python: '3.9' - os: macos-14 @@ -108,13 +119,46 @@ jobs: pkg-config libabsl-dev libaio-dev libprotobuf-dev \ patchelf - # Install Intel MKL for DiskANN - wget -q https://registrationcenter-download.intel.com/akdlm/IRC_NAS/79153e0f-74d7-45af-b8c2-258941adf58a/intel-onemkl-2025.0.0.940.sh - sudo sh intel-onemkl-2025.0.0.940.sh -a --components intel.oneapi.lin.mkl.devel --action install --eula accept -s - source /opt/intel/oneapi/setvars.sh - echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=/opt/intel/oneapi/compiler/latest/linux/compiler/lib/intel64_lin" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/intel/oneapi/mkl/latest/lib/intel64" >> $GITHUB_ENV + # Debug: Show system information + echo "🔍 System Information:" + echo "Architecture: $(uname -m)" + echo "OS: $(uname -a)" + echo "CPU info: $(lscpu | head -5)" + + # Install math library based on architecture + ARCH=$(uname -m) + echo "🔍 Setting up math library for architecture: $ARCH" + + if [[ "$ARCH" == "x86_64" ]]; then + # Install Intel MKL for DiskANN on x86_64 + echo "📦 Installing Intel MKL for x86_64..." + wget -q https://registrationcenter-download.intel.com/akdlm/IRC_NAS/79153e0f-74d7-45af-b8c2-258941adf58a/intel-onemkl-2025.0.0.940.sh + sudo sh intel-onemkl-2025.0.0.940.sh -a --components intel.oneapi.lin.mkl.devel --action install --eula accept -s + source /opt/intel/oneapi/setvars.sh + echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=/opt/intel/oneapi/compiler/latest/linux/compiler/lib/intel64_lin" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/intel/oneapi/mkl/latest/lib/intel64" >> $GITHUB_ENV + echo "✅ Intel MKL installed for x86_64" + + # Debug: Check MKL installation + echo "🔍 MKL Installation Check:" + ls -la /opt/intel/oneapi/mkl/latest/ || echo "MKL directory not found" + ls -la /opt/intel/oneapi/mkl/latest/lib/ || echo "MKL lib directory not found" + + elif [[ "$ARCH" == "aarch64" ]]; then + # Use OpenBLAS for ARM64 (MKL installer not compatible with ARM64) + echo "📦 Installing OpenBLAS for ARM64..." + sudo apt-get install -y libopenblas-dev liblapack-dev liblapacke-dev + echo "✅ OpenBLAS installed for ARM64" + + # Debug: Check OpenBLAS installation + echo "🔍 OpenBLAS Installation Check:" + dpkg -l | grep openblas || echo "OpenBLAS package not found" + ls -la /usr/lib/aarch64-linux-gnu/openblas/ || echo "OpenBLAS directory not found" + fi + + # Debug: Show final library paths + echo "🔍 Final LD_LIBRARY_PATH: $LD_LIBRARY_PATH" - name: Install system dependencies (macOS) if: runner.os == 'macOS' diff --git a/packages/leann-backend-diskann/third_party/DiskANN b/packages/leann-backend-diskann/third_party/DiskANN index c593831..19f9603 160000 --- a/packages/leann-backend-diskann/third_party/DiskANN +++ b/packages/leann-backend-diskann/third_party/DiskANN @@ -1 +1 @@ -Subproject commit c593831474afb26bf167b077c2f0956ddbc54603 +Subproject commit 19f9603c728f51ff4a37df78805a3bb183e9870d diff --git a/packages/leann-backend-hnsw/CMakeLists.txt b/packages/leann-backend-hnsw/CMakeLists.txt index 12e19ef..87d4592 100644 --- a/packages/leann-backend-hnsw/CMakeLists.txt +++ b/packages/leann-backend-hnsw/CMakeLists.txt @@ -49,9 +49,28 @@ set(BUILD_TESTING OFF CACHE BOOL "" FORCE) set(FAISS_ENABLE_C_API OFF CACHE BOOL "" FORCE) set(FAISS_OPT_LEVEL "generic" CACHE STRING "" FORCE) -# Disable additional SIMD versions to speed up compilation +# Disable x86-specific SIMD optimizations (important for ARM64 compatibility) set(FAISS_ENABLE_AVX2 OFF CACHE BOOL "" FORCE) set(FAISS_ENABLE_AVX512 OFF CACHE BOOL "" FORCE) +set(FAISS_ENABLE_SSE4_1 OFF CACHE BOOL "" FORCE) + +# ARM64-specific configuration +if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64") + message(STATUS "Configuring Faiss for ARM64 architecture") + + if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + # Use SVE optimization level for ARM64 Linux (as seen in Faiss conda build) + set(FAISS_OPT_LEVEL "sve" CACHE STRING "" FORCE) + message(STATUS "Setting FAISS_OPT_LEVEL to 'sve' for ARM64 Linux") + else() + # Use generic optimization for other ARM64 platforms (like macOS) + set(FAISS_OPT_LEVEL "generic" CACHE STRING "" FORCE) + message(STATUS "Setting FAISS_OPT_LEVEL to 'generic' for ARM64 ${CMAKE_SYSTEM_NAME}") + endif() + + # ARM64 compatibility: Faiss submodule has been modified to fix x86 header inclusion + message(STATUS "Using ARM64-compatible Faiss submodule") +endif() # Additional optimization options from INSTALL.md set(CMAKE_BUILD_TYPE "Release" CACHE STRING "" FORCE) diff --git a/packages/leann-backend-hnsw/third_party/faiss b/packages/leann-backend-hnsw/third_party/faiss index a036185..ed96ff7 160000 --- a/packages/leann-backend-hnsw/third_party/faiss +++ b/packages/leann-backend-hnsw/third_party/faiss @@ -1 +1 @@ -Subproject commit a0361858fc9cbc95239ef304d9c85d48bb2701c8 +Subproject commit ed96ff7dbaea0562b994f8ce7823af41884b1010