chore: build and package

2025-07-24 00:47:46 -07:00
parent efd6373b32
commit 7add391b2c
8 changed files with 421 additions and 4 deletions
--- a/.github/workflows/build-and-publish.yml
+++ b/.github/workflows/build-and-publish.yml
@@ -0,0 +1,256 @@
+name: Build and Publish to PyPI
+
+on:
+  release:
+    types: [published]
+  push:
+    tags:
+      - 'v*'
+  workflow_dispatch:
+    inputs:
+      publish:
+        description: 'Publish to PyPI'
+        required: true
+        default: 'false'
+        type: choice
+        options:
+          - 'false'
+          - 'test'
+          - 'prod'
+
+jobs:
+  # Build pure Python package: leann-core
+  build-core:
+    name: Build leann-core
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+          
+      - name: Install build dependencies
+        run: |
+          uv pip install --system build twine
+          
+      - name: Build package
+        run: |
+          cd packages/leann-core
+          uv build
+          
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: leann-core-dist
+          path: packages/leann-core/dist/
+
+  # Build binary package: leann-backend-hnsw (default backend)
+  build-hnsw:
+    name: Build leann-backend-hnsw
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python-version: ['3.9', '3.10', '3.11', '3.12']
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+          
+      - name: Install system dependencies (Ubuntu)
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libomp-dev libboost-all-dev libzmq3-dev \
+            pkg-config patchelf
+          
+      - name: Install system dependencies (macOS)
+        if: runner.os == 'macOS'
+        run: |
+          brew install libomp boost zeromq
+          
+      - name: Install build dependencies
+        run: |
+          uv pip install --system scikit-build-core numpy swig
+          uv pip install --system auditwheel delocate
+          
+      - name: Build wheel
+        run: |
+          cd packages/leann-backend-hnsw
+          uv pip wheel . --no-deps -w dist
+          
+      - name: Repair wheel (Linux)
+        if: runner.os == 'Linux'
+        run: |
+          cd packages/leann-backend-hnsw
+          auditwheel repair dist/*.whl -w dist_repaired
+          rm -rf dist
+          mv dist_repaired dist
+          
+      - name: Repair wheel (macOS)
+        if: runner.os == 'macOS'
+        run: |
+          cd packages/leann-backend-hnsw
+          delocate-wheel -w dist_repaired -v dist/*.whl
+          rm -rf dist
+          mv dist_repaired dist
+          
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: hnsw-${{ matrix.os }}-py${{ matrix.python-version }}
+          path: packages/leann-backend-hnsw/dist/
+
+  # Build binary package: leann-backend-diskann (multi-platform)
+  build-diskann:
+    name: Build leann-backend-diskann
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python-version: ['3.9', '3.10', '3.11', '3.12']
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+          
+      - name: Install system dependencies (Ubuntu)
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libomp-dev libboost-all-dev libaio-dev libzmq3-dev \
+            pkg-config protobuf-compiler libprotobuf-dev libabsl-dev patchelf
+          
+          # Install Intel MKL using Intel's installer
+          wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/79153e0f-74d7-45af-b8c2-258941adf58a/intel-onemkl-2025.0.0.940.sh
+          sudo sh intel-onemkl-2025.0.0.940.sh -a --components intel.oneapi.lin.mkl.devel --action install --eula accept -s
+          source /opt/intel/oneapi/setvars.sh
+          echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV
+          echo "LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH" >> $GITHUB_ENV
+          
+      - name: Install system dependencies (macOS)
+        if: runner.os == 'macOS'
+        run: |
+          brew install libomp boost zeromq
+          # MKL is not available on Homebrew, but DiskANN can work without it
+          
+      - name: Install build dependencies
+        run: |
+          uv pip install --system scikit-build-core numpy Cython pybind11
+          if [[ "$RUNNER_OS" == "Linux" ]]; then
+            uv pip install --system auditwheel
+          else
+            uv pip install --system delocate
+          fi
+          
+      - name: Build wheel
+        run: |
+          cd packages/leann-backend-diskann
+          uv pip wheel . --no-deps -w dist
+          
+      - name: Repair wheel (Linux)
+        if: runner.os == 'Linux'
+        run: |
+          cd packages/leann-backend-diskann
+          auditwheel repair dist/*.whl -w dist_repaired
+          rm -rf dist
+          mv dist_repaired dist
+          
+      - name: Repair wheel (macOS)
+        if: runner.os == 'macOS'
+        run: |
+          cd packages/leann-backend-diskann
+          delocate-wheel -w dist_repaired -v dist/*.whl
+          rm -rf dist
+          mv dist_repaired dist
+          
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: diskann-${{ matrix.os }}-py${{ matrix.python-version }}
+          path: packages/leann-backend-diskann/dist/
+
+  # Build meta-package: leann (build last)
+  build-meta:
+    name: Build leann meta-package
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+          
+      - name: Install build dependencies
+        run: |
+          uv pip install --system build
+          
+      - name: Build package
+        run: |
+          cd packages/leann
+          uv build
+          
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: leann-meta-dist
+          path: packages/leann/dist/
+
+  # Publish to PyPI
+  publish:
+    name: Publish to PyPI
+    needs: [build-core, build-hnsw, build-diskann, build-meta]
+    runs-on: ubuntu-latest
+    if: github.event_name == 'release' || github.event.inputs.publish != 'false'
+    
+    steps:
+      - name: Download all artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: dist
+          
+      - name: Flatten directory structure
+        run: |
+          mkdir -p all_wheels
+          find dist -name "*.whl" -exec cp {} all_wheels/ \;
+          find dist -name "*.tar.gz" -exec cp {} all_wheels/ \;
+          
+      - name: Publish to Test PyPI
+        if: github.event.inputs.publish == 'test' || github.event_name == 'workflow_dispatch'
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.TEST_PYPI_API_TOKEN }}
+          repository-url: https://test.pypi.org/legacy/
+          packages-dir: all_wheels/
+          
+      - name: Publish to PyPI
+        if: github.event_name == 'release' || github.event.inputs.publish == 'prod'
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}
+          packages-dir: all_wheels/ 
--- a/docs/RELEASE.md
+++ b/docs/RELEASE.md
@@ -0,0 +1,54 @@
+# Release Guide
+
+## One-line Release 🚀
+
+```bash
+./scripts/release.sh 0.1.1
+```
+
+That's it! This script will:
+1. Update all package versions
+2. Commit and push changes
+3. Create GitHub release
+4. CI automatically builds and publishes to PyPI
+
+## Manual Testing Before Release
+
+For testing specific packages locally (especially DiskANN on macOS):
+
+```bash
+# Build specific package locally
+./scripts/build_and_test.sh diskann  # or hnsw, core, meta, all
+
+# Test installation in a clean environment
+python -m venv test_env
+source test_env/bin/activate
+pip install packages/*/dist/*.whl
+
+# Upload to Test PyPI (optional)
+./scripts/upload_to_pypi.sh test
+
+# Upload to Production PyPI (use with caution)
+./scripts/upload_to_pypi.sh prod
+```
+
+### Why Manual Build for DiskANN?
+
+DiskANN's complex dependencies (protobuf, abseil, etc.) sometimes require local testing before release. The build script will:
+- Compile the C++ extension
+- Use `delocate` (macOS) or `auditwheel` (Linux) to bundle system libraries
+- Create a self-contained wheel with no external dependencies
+
+## First-time setup
+
+1. Install GitHub CLI:
+   ```bash
+   brew install gh
+   gh auth login
+   ```
+
+2. Set PyPI token in GitHub:
+   ```bash
+   gh secret set PYPI_API_TOKEN
+   # Paste your PyPI token when prompted
+   ``` 
--- a/packages/leann-backend-diskann/third_party/DiskANN
+++ b/packages/leann-backend-diskann/third_party/DiskANN
--- a/packages/leann-backend-hnsw/pyproject.toml
+++ b/packages/leann-backend-hnsw/pyproject.toml
@@ -8,7 +8,12 @@ build-backend = "scikit_build_core.build"
 name = "leann-backend-hnsw"
 version = "0.1.0"
 description = "Custom-built HNSW (Faiss) backend for the Leann toolkit."
-dependencies = ["leann-core==0.1.0", "numpy"]
+dependencies = [
+    "leann-core==0.1.0", 
+    "numpy",
+    "pyzmq>=23.0.0",
+    "msgpack>=1.0.0",
+]

 [tool.scikit-build]
 wheel.packages = ["leann_backend_hnsw"]
--- a/packages/leann-core/pyproject.toml
+++ b/packages/leann-core/pyproject.toml
@@ -5,14 +5,22 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "leann-core"
 version = "0.1.0"
-description = "Core API and plugin system for Leann."
+description = "Core API and plugin system for LEANN"
 readme = "README.md"
 requires-python = ">=3.9"
 license = { text = "MIT" }

+# All required dependencies included
 dependencies = [
    "numpy>=1.20.0",
-    "tqdm>=4.60.0"
+    "tqdm>=4.60.0",
+    "psutil>=5.8.0",
+    "pyzmq>=23.0.0",
+    "msgpack>=1.0.0",
+    "torch>=2.0.0",
+    "sentence-transformers>=2.2.0",
+    "llama-index-core>=0.12.0",
+    "python-dotenv>=1.0.0",
 ]

 [project.scripts]
--- a/packages/leann/README.md
+++ b/packages/leann/README.md
@@ -0,0 +1,40 @@
+# LEANN - The smallest vector index in the world
+
+LEANN is a revolutionary vector database that democratizes personal AI. Transform your laptop into a powerful RAG system that can index and search through millions of documents while using **97% less storage** than traditional solutions **without accuracy loss**.
+
+## Installation
+
+```bash
+# Default installation (HNSW backend, recommended)
+uv pip install leann
+
+# With DiskANN backend (for large-scale deployments)
+uv pip install leann[diskann]
+```
+
+## Quick Start
+
+```python
+from leann import LeannBuilder, LeannSearcher, LeannChat
+
+# Build an index
+builder = LeannBuilder(backend_name="hnsw")
+builder.add_text("LEANN saves 97% storage compared to traditional vector databases.")
+builder.build_index("my_index.leann")
+
+# Search
+searcher = LeannSearcher("my_index.leann")
+results = searcher.search("storage savings", top_k=3)
+
+# Chat with your data
+chat = LeannChat("my_index.leann", llm_config={"type": "ollama", "model": "llama3.2:1b"})
+response = chat.ask("How much storage does LEANN save?")
+```
+
+## Documentation
+
+For full documentation, visit [https://leann.readthedocs.io](https://leann.readthedocs.io)
+
+## License
+
+MIT License 
--- a/packages/leann/init.py
+++ b/packages/leann/init.py
@@ -0,0 +1,12 @@
+"""
+LEANN - Low-storage Embedding Approximation for Neural Networks
+
+A revolutionary vector database that democratizes personal AI.
+"""
+
+__version__ = "0.1.0"
+
+# Re-export main API from leann-core
+from leann_core import LeannBuilder, LeannSearcher, LeannChat
+
+__all__ = ["LeannBuilder", "LeannSearcher", "LeannChat"]
--- a/packages/leann/pyproject.toml
+++ b/packages/leann/pyproject.toml
@@ -0,0 +1,42 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "leann"
+version = "0.1.0"
+description = "LEANN - The smallest vector index in the world. RAG Everything with LEANN!"
+readme = "README.md"
+requires-python = ">=3.9"
+license = { text = "MIT" }
+authors = [
+    { name = "LEANN Team" }
+]
+keywords = ["vector-database", "rag", "embeddings", "search", "ai"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+]
+
+# Default installation: core + hnsw
+dependencies = [
+    "leann-core>=0.1.0",
+    "leann-backend-hnsw>=0.1.0",
+]
+
+[project.optional-dependencies]
+diskann = [
+    "leann-backend-diskann>=0.1.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/yourusername/leann"
+Documentation = "https://leann.readthedocs.io"
+Repository = "https://github.com/yourusername/leann"
+Issues = "https://github.com/yourusername/leann/issues"