From 8c04a8345ec6705a7c24a70db44b45f9557109ec Mon Sep 17 00:00:00 2001 From: Andy Lee Date: Sat, 16 Aug 2025 13:38:30 -0700 Subject: [PATCH] chore: all linux deps --- .github/workflows/build-reusable.yml | 51 ++++++++++++++++++++++++- README.md | 57 ++++++++++++++++++++++++---- 2 files changed, 99 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-reusable.yml b/.github/workflows/build-reusable.yml index b2323a3..6d46a6a 100644 --- a/.github/workflows/build-reusable.yml +++ b/.github/workflows/build-reusable.yml @@ -105,14 +105,16 @@ jobs: run: | sudo apt-get update sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \ - pkg-config libopenblas-dev patchelf libabsl-dev libaio-dev libprotobuf-dev + pkg-config libabsl-dev libaio-dev libprotobuf-dev \ + patchelf # Install Intel MKL for DiskANN wget -q https://registrationcenter-download.intel.com/akdlm/IRC_NAS/79153e0f-74d7-45af-b8c2-258941adf58a/intel-onemkl-2025.0.0.940.sh sudo sh intel-onemkl-2025.0.0.940.sh -a --components intel.oneapi.lin.mkl.devel --action install --eula accept -s source /opt/intel/oneapi/setvars.sh echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=/opt/intel/oneapi/compiler/latest/linux/compiler/lib/intel64_lin" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/intel/oneapi/mkl/latest/lib/intel64" >> $GITHUB_ENV - name: Install system dependencies (macOS) if: runner.os == 'macOS' @@ -304,3 +306,48 @@ jobs: with: name: packages-${{ matrix.os }}-py${{ matrix.python }} path: packages/*/dist/ + + + arch-smoke: + name: Arch Linux smoke test (install & import) + needs: build + runs-on: ubuntu-latest + container: + image: archlinux:latest + + steps: + - name: Prepare system + run: | + pacman -Syu --noconfirm + pacman -S --noconfirm python python-pip gcc git zlib openssl + + - name: Download ALL wheel artifacts from this run + uses: actions/download-artifact@v4 + with: + # Don't specify name, download all artifacts + path: ./wheels + + - name: Install wheels (pip automatically picks matching tags from wheels directory) + run: | + python -m pip install --upgrade pip + pip install --find-links wheels leann-core + pip install --find-links wheels leann-backend-hnsw + pip install --find-links wheels leann-backend-diskann + pip install --find-links wheels leann + + - name: Import & tiny runtime check + env: + OMP_NUM_THREADS: 1 + MKL_NUM_THREADS: 1 + run: | + python - <<'PY' + import leann + import leann_backend_hnsw as h + import leann_backend_diskann as d + from leann import LeannBuilder, LeannSearcher + b = LeannBuilder(backend_name="hnsw") + b.add_text("hello arch") + b.build_index("arch_demo.leann") + s = LeannSearcher("arch_demo.leann") + print("search:", s.search("hello", top_k=1)) + PY diff --git a/README.md b/README.md index 279a37a..ced89f2 100755 --- a/README.md +++ b/README.md @@ -87,17 +87,60 @@ git submodule update --init --recursive ``` **macOS:** + +Note: DiskANN requires MacOS 13.3 or later. + ```bash -brew install llvm libomp boost protobuf zeromq pkgconf -CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv sync +brew install libomp boost protobuf zeromq pkgconf +uv sync --extra diskann ``` -**Linux:** -```bash -# Ubuntu/Debian (For Arch Linux: sudo pacman -S blas lapack openblas libaio boost protobuf abseil-cpp zeromq) -sudo apt-get update && sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libabsl-dev libmkl-full-dev libaio-dev libzmq3-dev +**Linux (Ubuntu/Debian):** -uv sync +Note: On Ubuntu 20.04, you may need to build a newer Abseil and pin Protobuf (e.g., v3.20.x) for building DiskANN. See [Issue #30](https://github.com/yichuan-w/LEANN/issues/30) for a step-by-step note. + +You can manually downloading [Intel MKL](https://www.intel.com/content/www/us/en/developer/tools/oneapi/onemkl-download.html?operatingsystem=linux&linux-install=online) instead of `libmkl-full-dev` for DiskANN. You can also use `libopenblas-dev` for building HNSW only, by removing `--extra diskann` in the command below. + +```bash +sudo apt-get update && sudo apt-get install -y \ + libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \ + pkg-config libabsl-dev libaio-dev libprotobuf-dev \ + libmkl-full-dev + +uv sync --extra diskann +``` + +**Linux (Arch Linux):** + +```bash +sudo pacman -Syu && sudo pacman -S --needed base-devel cmake pkgconf git gcc \ + boost boost-libs protobuf abseil-cpp libaio zeromq + +# For MKL in DiskANN +sudo pacman -S --needed base-devel git +git clone https://aur.archlinux.org/paru-bin.git +cd paru-bin && makepkg -si +paru -S intel-oneapi-mkl intel-oneapi-compiler +source /opt/intel/oneapi/setvars.sh + +uv sync --extra diskann +``` + +**Linux (RHEL / CentOS Stream / Oracle / Rocky / AlmaLinux):** + +See [Issue #50](https://github.com/yichuan-w/LEANN/issues/50) for more details. + +```bash +sudo dnf groupinstall -y "Development Tools" +sudo dnf install -y libomp-devel boost-devel protobuf-compiler protobuf-devel \ + abseil-cpp-devel libaio-devel zeromq-devel pkgconf-pkg-config + +# For MKL in DiskANN +sudo dnf install -y intel-oneapi-mkl intel-oneapi-mkl-devel \ + intel-oneapi-openmp || sudo dnf install -y intel-oneapi-compiler +source /opt/intel/oneapi/setvars.sh + +uv sync --extra diskann ```