Compare commits
16 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2dd59c4ba1 | ||
|
|
166986d5e6 | ||
|
|
a6aec68f32 | ||
|
|
ed27a127d5 | ||
|
|
d8b4ea7564 | ||
|
|
f0a2ef96b4 | ||
|
|
7d73c2c803 | ||
|
|
e8d2ecab03 | ||
|
|
32a374d094 | ||
|
|
d45c013806 | ||
|
|
9000a7083d | ||
|
|
8307555d54 | ||
|
|
20f2aece08 | ||
|
|
43eb4f9a1d | ||
|
|
5461b71d8c | ||
|
|
374db0ebb8 |
257
.github/workflows/build-and-publish.yml
vendored
257
.github/workflows/build-and-publish.yml
vendored
@@ -1,262 +1,11 @@
|
|||||||
name: CI - Build Multi-Platform Packages
|
name: CI
|
||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches: [ main ]
|
branches: [ main ]
|
||||||
pull_request:
|
pull_request:
|
||||||
branches: [ main ]
|
branches: [ main ]
|
||||||
workflow_dispatch:
|
|
||||||
inputs:
|
|
||||||
publish:
|
|
||||||
description: 'Publish to PyPI (only use for emergency fixes)'
|
|
||||||
required: true
|
|
||||||
default: 'false'
|
|
||||||
type: choice
|
|
||||||
options:
|
|
||||||
- 'false'
|
|
||||||
- 'test'
|
|
||||||
- 'prod'
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
# Build pure Python package: leann-core
|
build:
|
||||||
build-core:
|
uses: ./.github/workflows/build-reusable.yml
|
||||||
name: Build leann-core
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Set up Python
|
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version: '3.11'
|
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
uses: astral-sh/setup-uv@v4
|
|
||||||
|
|
||||||
- name: Install build dependencies
|
|
||||||
run: |
|
|
||||||
uv pip install --system build twine
|
|
||||||
|
|
||||||
- name: Build package
|
|
||||||
run: |
|
|
||||||
cd packages/leann-core
|
|
||||||
uv build
|
|
||||||
|
|
||||||
- name: Upload artifacts
|
|
||||||
uses: actions/upload-artifact@v4
|
|
||||||
with:
|
|
||||||
name: leann-core-dist
|
|
||||||
path: packages/leann-core/dist/
|
|
||||||
|
|
||||||
# Build binary package: leann-backend-hnsw (default backend)
|
|
||||||
build-hnsw:
|
|
||||||
name: Build leann-backend-hnsw
|
|
||||||
strategy:
|
|
||||||
matrix:
|
|
||||||
os: [ubuntu-latest, macos-latest]
|
|
||||||
python-version: ['3.9', '3.10', '3.11', '3.12']
|
|
||||||
runs-on: ${{ matrix.os }}
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
submodules: recursive
|
|
||||||
|
|
||||||
- name: Set up Python
|
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version: ${{ matrix.python-version }}
|
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
uses: astral-sh/setup-uv@v4
|
|
||||||
|
|
||||||
- name: Install system dependencies (Ubuntu)
|
|
||||||
if: runner.os == 'Linux'
|
|
||||||
run: |
|
|
||||||
sudo apt-get update
|
|
||||||
sudo apt-get install -y libomp-dev libboost-all-dev libzmq3-dev \
|
|
||||||
pkg-config libopenblas-dev patchelf
|
|
||||||
|
|
||||||
- name: Install system dependencies (macOS)
|
|
||||||
if: runner.os == 'macOS'
|
|
||||||
run: |
|
|
||||||
brew install libomp boost zeromq
|
|
||||||
|
|
||||||
- name: Install build dependencies
|
|
||||||
run: |
|
|
||||||
uv pip install --system scikit-build-core numpy swig
|
|
||||||
uv pip install --system auditwheel delocate
|
|
||||||
|
|
||||||
- name: Build wheel
|
|
||||||
run: |
|
|
||||||
cd packages/leann-backend-hnsw
|
|
||||||
uv build --wheel --python python
|
|
||||||
|
|
||||||
- name: Repair wheel (Linux)
|
|
||||||
if: runner.os == 'Linux'
|
|
||||||
run: |
|
|
||||||
cd packages/leann-backend-hnsw
|
|
||||||
auditwheel repair dist/*.whl -w dist_repaired
|
|
||||||
rm -rf dist
|
|
||||||
mv dist_repaired dist
|
|
||||||
|
|
||||||
- name: Repair wheel (macOS)
|
|
||||||
if: runner.os == 'macOS'
|
|
||||||
run: |
|
|
||||||
cd packages/leann-backend-hnsw
|
|
||||||
delocate-wheel -w dist_repaired -v dist/*.whl
|
|
||||||
rm -rf dist
|
|
||||||
mv dist_repaired dist
|
|
||||||
|
|
||||||
- name: Upload artifacts
|
|
||||||
uses: actions/upload-artifact@v4
|
|
||||||
with:
|
|
||||||
name: hnsw-${{ matrix.os }}-py${{ matrix.python-version }}
|
|
||||||
path: packages/leann-backend-hnsw/dist/
|
|
||||||
|
|
||||||
# Build binary package: leann-backend-diskann (multi-platform)
|
|
||||||
build-diskann:
|
|
||||||
name: Build leann-backend-diskann
|
|
||||||
strategy:
|
|
||||||
matrix:
|
|
||||||
os: [ubuntu-latest, macos-latest]
|
|
||||||
python-version: ['3.9', '3.10', '3.11', '3.12']
|
|
||||||
runs-on: ${{ matrix.os }}
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
submodules: recursive
|
|
||||||
|
|
||||||
- name: Set up Python
|
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version: ${{ matrix.python-version }}
|
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
uses: astral-sh/setup-uv@v4
|
|
||||||
|
|
||||||
- name: Install system dependencies (Ubuntu)
|
|
||||||
if: runner.os == 'Linux'
|
|
||||||
run: |
|
|
||||||
sudo apt-get update
|
|
||||||
sudo apt-get install -y libomp-dev libboost-all-dev libaio-dev libzmq3-dev \
|
|
||||||
protobuf-compiler libprotobuf-dev libabsl-dev patchelf
|
|
||||||
|
|
||||||
# Install Intel MKL using Intel's installer
|
|
||||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/79153e0f-74d7-45af-b8c2-258941adf58a/intel-onemkl-2025.0.0.940.sh
|
|
||||||
sudo sh intel-onemkl-2025.0.0.940.sh -a --components intel.oneapi.lin.mkl.devel --action install --eula accept -s
|
|
||||||
source /opt/intel/oneapi/setvars.sh
|
|
||||||
echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV
|
|
||||||
echo "LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH" >> $GITHUB_ENV
|
|
||||||
|
|
||||||
- name: Install system dependencies (macOS)
|
|
||||||
if: runner.os == 'macOS'
|
|
||||||
run: |
|
|
||||||
brew install libomp boost zeromq protobuf
|
|
||||||
# MKL is not available on Homebrew, but DiskANN can work without it
|
|
||||||
|
|
||||||
- name: Install build dependencies
|
|
||||||
run: |
|
|
||||||
uv pip install --system scikit-build-core numpy Cython pybind11
|
|
||||||
if [[ "$RUNNER_OS" == "Linux" ]]; then
|
|
||||||
uv pip install --system auditwheel
|
|
||||||
else
|
|
||||||
uv pip install --system delocate
|
|
||||||
fi
|
|
||||||
|
|
||||||
- name: Build wheel
|
|
||||||
run: |
|
|
||||||
cd packages/leann-backend-diskann
|
|
||||||
uv build --wheel --python python
|
|
||||||
|
|
||||||
- name: Repair wheel (Linux)
|
|
||||||
if: runner.os == 'Linux'
|
|
||||||
run: |
|
|
||||||
cd packages/leann-backend-diskann
|
|
||||||
auditwheel repair dist/*.whl -w dist_repaired
|
|
||||||
rm -rf dist
|
|
||||||
mv dist_repaired dist
|
|
||||||
|
|
||||||
- name: Repair wheel (macOS)
|
|
||||||
if: runner.os == 'macOS'
|
|
||||||
run: |
|
|
||||||
cd packages/leann-backend-diskann
|
|
||||||
delocate-wheel -w dist_repaired -v dist/*.whl
|
|
||||||
rm -rf dist
|
|
||||||
mv dist_repaired dist
|
|
||||||
|
|
||||||
- name: Upload artifacts
|
|
||||||
uses: actions/upload-artifact@v4
|
|
||||||
with:
|
|
||||||
name: diskann-${{ matrix.os }}-py${{ matrix.python-version }}
|
|
||||||
path: packages/leann-backend-diskann/dist/
|
|
||||||
|
|
||||||
# Build meta-package: leann (build last)
|
|
||||||
build-meta:
|
|
||||||
name: Build leann meta-package
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Set up Python
|
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version: '3.11'
|
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
uses: astral-sh/setup-uv@v4
|
|
||||||
|
|
||||||
- name: Install build dependencies
|
|
||||||
run: |
|
|
||||||
uv pip install --system build
|
|
||||||
|
|
||||||
- name: Build package
|
|
||||||
run: |
|
|
||||||
cd packages/leann
|
|
||||||
uv build
|
|
||||||
|
|
||||||
- name: Upload artifacts
|
|
||||||
uses: actions/upload-artifact@v4
|
|
||||||
with:
|
|
||||||
name: leann-meta-dist
|
|
||||||
path: packages/leann/dist/
|
|
||||||
|
|
||||||
# Publish to PyPI (only for emergency fixes or manual triggers)
|
|
||||||
publish:
|
|
||||||
name: Publish to PyPI (Emergency)
|
|
||||||
needs: [build-core, build-hnsw, build-diskann, build-meta]
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
if: github.event_name == 'workflow_dispatch' && github.event.inputs.publish != 'false'
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: Download all artifacts
|
|
||||||
uses: actions/download-artifact@v4
|
|
||||||
with:
|
|
||||||
path: dist
|
|
||||||
|
|
||||||
- name: Flatten directory structure
|
|
||||||
run: |
|
|
||||||
mkdir -p all_wheels
|
|
||||||
find dist -name "*.whl" -exec cp {} all_wheels/ \;
|
|
||||||
find dist -name "*.tar.gz" -exec cp {} all_wheels/ \;
|
|
||||||
|
|
||||||
- name: Show what will be published
|
|
||||||
run: |
|
|
||||||
echo "📦 Packages to be published:"
|
|
||||||
ls -la all_wheels/
|
|
||||||
|
|
||||||
- name: Publish to Test PyPI
|
|
||||||
if: github.event.inputs.publish == 'test'
|
|
||||||
uses: pypa/gh-action-pypi-publish@release/v1
|
|
||||||
with:
|
|
||||||
password: ${{ secrets.TEST_PYPI_API_TOKEN }}
|
|
||||||
repository-url: https://test.pypi.org/legacy/
|
|
||||||
packages-dir: all_wheels/
|
|
||||||
skip-existing: true
|
|
||||||
|
|
||||||
- name: Publish to PyPI
|
|
||||||
if: github.event.inputs.publish == 'prod'
|
|
||||||
uses: pypa/gh-action-pypi-publish@release/v1
|
|
||||||
with:
|
|
||||||
password: ${{ secrets.PYPI_API_TOKEN }}
|
|
||||||
packages-dir: all_wheels/
|
|
||||||
skip-existing: true
|
|
||||||
169
.github/workflows/build-reusable.yml
vendored
Normal file
169
.github/workflows/build-reusable.yml
vendored
Normal file
@@ -0,0 +1,169 @@
|
|||||||
|
name: Reusable Build
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_call:
|
||||||
|
inputs:
|
||||||
|
ref:
|
||||||
|
description: 'Git ref to build'
|
||||||
|
required: false
|
||||||
|
type: string
|
||||||
|
default: ''
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
name: Build ${{ matrix.os }} Python ${{ matrix.python }}
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
include:
|
||||||
|
- os: ubuntu-latest
|
||||||
|
python: '3.9'
|
||||||
|
- os: ubuntu-latest
|
||||||
|
python: '3.10'
|
||||||
|
- os: ubuntu-latest
|
||||||
|
python: '3.11'
|
||||||
|
- os: ubuntu-latest
|
||||||
|
python: '3.12'
|
||||||
|
- os: macos-latest
|
||||||
|
python: '3.9'
|
||||||
|
- os: macos-latest
|
||||||
|
python: '3.10'
|
||||||
|
- os: macos-latest
|
||||||
|
python: '3.11'
|
||||||
|
- os: macos-latest
|
||||||
|
python: '3.12'
|
||||||
|
runs-on: ${{ matrix.os }}
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
ref: ${{ inputs.ref }}
|
||||||
|
submodules: recursive
|
||||||
|
|
||||||
|
- name: Setup Python
|
||||||
|
uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: ${{ matrix.python }}
|
||||||
|
|
||||||
|
- name: Install uv
|
||||||
|
uses: astral-sh/setup-uv@v4
|
||||||
|
|
||||||
|
- name: Install system dependencies (Ubuntu)
|
||||||
|
if: runner.os == 'Linux'
|
||||||
|
run: |
|
||||||
|
sudo apt-get update
|
||||||
|
sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \
|
||||||
|
pkg-config libopenblas-dev patchelf libabsl-dev libaio-dev libprotobuf-dev
|
||||||
|
|
||||||
|
# Install Intel MKL for DiskANN
|
||||||
|
wget -q https://registrationcenter-download.intel.com/akdlm/IRC_NAS/79153e0f-74d7-45af-b8c2-258941adf58a/intel-onemkl-2025.0.0.940.sh
|
||||||
|
sudo sh intel-onemkl-2025.0.0.940.sh -a --components intel.oneapi.lin.mkl.devel --action install --eula accept -s
|
||||||
|
source /opt/intel/oneapi/setvars.sh
|
||||||
|
echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV
|
||||||
|
echo "LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH" >> $GITHUB_ENV
|
||||||
|
|
||||||
|
- name: Install system dependencies (macOS)
|
||||||
|
if: runner.os == 'macOS'
|
||||||
|
run: |
|
||||||
|
brew install llvm libomp boost protobuf zeromq
|
||||||
|
|
||||||
|
- name: Install build dependencies
|
||||||
|
run: |
|
||||||
|
uv pip install --system scikit-build-core numpy swig Cython pybind11
|
||||||
|
if [[ "$RUNNER_OS" == "Linux" ]]; then
|
||||||
|
uv pip install --system auditwheel
|
||||||
|
else
|
||||||
|
uv pip install --system delocate
|
||||||
|
fi
|
||||||
|
|
||||||
|
- name: Build packages
|
||||||
|
run: |
|
||||||
|
# Build core (platform independent)
|
||||||
|
if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
|
||||||
|
cd packages/leann-core
|
||||||
|
uv build
|
||||||
|
cd ../..
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Build HNSW backend
|
||||||
|
cd packages/leann-backend-hnsw
|
||||||
|
if [ "${{ matrix.os }}" == "macos-latest" ]; then
|
||||||
|
CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build --wheel --python python
|
||||||
|
else
|
||||||
|
uv build --wheel --python python
|
||||||
|
fi
|
||||||
|
cd ../..
|
||||||
|
|
||||||
|
# Build DiskANN backend
|
||||||
|
cd packages/leann-backend-diskann
|
||||||
|
if [ "${{ matrix.os }}" == "macos-latest" ]; then
|
||||||
|
CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build --wheel --python python
|
||||||
|
else
|
||||||
|
uv build --wheel --python python
|
||||||
|
fi
|
||||||
|
cd ../..
|
||||||
|
|
||||||
|
# Build meta package (platform independent)
|
||||||
|
if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
|
||||||
|
cd packages/leann
|
||||||
|
uv build
|
||||||
|
cd ../..
|
||||||
|
fi
|
||||||
|
|
||||||
|
- name: Repair wheels (Linux)
|
||||||
|
if: runner.os == 'Linux'
|
||||||
|
run: |
|
||||||
|
# Repair HNSW wheel
|
||||||
|
cd packages/leann-backend-hnsw
|
||||||
|
if [ -d dist ]; then
|
||||||
|
# Show what platform auditwheel will use
|
||||||
|
auditwheel show dist/*.whl || true
|
||||||
|
# Let auditwheel auto-detect the appropriate manylinux tag
|
||||||
|
auditwheel repair dist/*.whl -w dist_repaired
|
||||||
|
rm -rf dist
|
||||||
|
mv dist_repaired dist
|
||||||
|
fi
|
||||||
|
cd ../..
|
||||||
|
|
||||||
|
# Repair DiskANN wheel
|
||||||
|
cd packages/leann-backend-diskann
|
||||||
|
if [ -d dist ]; then
|
||||||
|
# Show what platform auditwheel will use
|
||||||
|
auditwheel show dist/*.whl || true
|
||||||
|
# Let auditwheel auto-detect the appropriate manylinux tag
|
||||||
|
auditwheel repair dist/*.whl -w dist_repaired
|
||||||
|
rm -rf dist
|
||||||
|
mv dist_repaired dist
|
||||||
|
fi
|
||||||
|
cd ../..
|
||||||
|
|
||||||
|
- name: Repair wheels (macOS)
|
||||||
|
if: runner.os == 'macOS'
|
||||||
|
run: |
|
||||||
|
# Repair HNSW wheel
|
||||||
|
cd packages/leann-backend-hnsw
|
||||||
|
if [ -d dist ]; then
|
||||||
|
delocate-wheel -w dist_repaired -v dist/*.whl
|
||||||
|
rm -rf dist
|
||||||
|
mv dist_repaired dist
|
||||||
|
fi
|
||||||
|
cd ../..
|
||||||
|
|
||||||
|
# Repair DiskANN wheel
|
||||||
|
cd packages/leann-backend-diskann
|
||||||
|
if [ -d dist ]; then
|
||||||
|
delocate-wheel -w dist_repaired -v dist/*.whl
|
||||||
|
rm -rf dist
|
||||||
|
mv dist_repaired dist
|
||||||
|
fi
|
||||||
|
cd ../..
|
||||||
|
|
||||||
|
- name: List built packages
|
||||||
|
run: |
|
||||||
|
echo "📦 Built packages:"
|
||||||
|
find packages/*/dist -name "*.whl" -o -name "*.tar.gz" | sort
|
||||||
|
|
||||||
|
- name: Upload artifacts
|
||||||
|
uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: packages-${{ matrix.os }}-py${{ matrix.python }}
|
||||||
|
path: packages/*/dist/
|
||||||
213
.github/workflows/release-manual.yml
vendored
213
.github/workflows/release-manual.yml
vendored
@@ -1,158 +1,79 @@
|
|||||||
name: Manual Release
|
name: Release
|
||||||
|
|
||||||
on:
|
on:
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
inputs:
|
inputs:
|
||||||
version:
|
version:
|
||||||
description: 'Version to release (e.g., 0.1.1)'
|
description: 'Version to release (e.g., 0.1.2)'
|
||||||
required: true
|
required: true
|
||||||
type: string
|
type: string
|
||||||
test_pypi:
|
|
||||||
description: 'Test on TestPyPI first'
|
|
||||||
required: false
|
|
||||||
type: boolean
|
|
||||||
default: true
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
validate-and-release:
|
update-version:
|
||||||
|
name: Update Version
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
permissions:
|
permissions:
|
||||||
contents: write
|
contents: write
|
||||||
actions: read
|
outputs:
|
||||||
|
commit-sha: ${{ steps.push.outputs.commit-sha }}
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
with:
|
|
||||||
token: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
|
|
||||||
- name: Check CI status
|
- name: Validate version
|
||||||
run: |
|
|
||||||
echo "ℹ️ This workflow will download build artifacts from the latest CI run."
|
|
||||||
echo " CI must have completed successfully on the current commit."
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
- name: Validate version format
|
|
||||||
run: |
|
run: |
|
||||||
if ! [[ "${{ inputs.version }}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
if ! [[ "${{ inputs.version }}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||||
echo "❌ Invalid version format. Use semantic versioning (e.g., 0.1.1)"
|
echo "❌ Invalid version format"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
echo "✅ Version format valid: ${{ inputs.version }}"
|
echo "✅ Version format valid"
|
||||||
|
|
||||||
- name: Check if version already exists
|
- name: Update versions and push
|
||||||
run: |
|
id: push
|
||||||
if git tag | grep -q "^v${{ inputs.version }}$"; then
|
|
||||||
echo "❌ Version v${{ inputs.version }} already exists!"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
echo "✅ Version is new"
|
|
||||||
|
|
||||||
- name: Set up Python
|
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version: '3.13'
|
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
run: |
|
|
||||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
||||||
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
|
|
||||||
|
|
||||||
- name: Update versions
|
|
||||||
run: |
|
run: |
|
||||||
./scripts/bump_version.sh ${{ inputs.version }}
|
./scripts/bump_version.sh ${{ inputs.version }}
|
||||||
git config user.name "GitHub Actions"
|
git config user.name "GitHub Actions"
|
||||||
git config user.email "actions@github.com"
|
git config user.email "actions@github.com"
|
||||||
git add packages/*/pyproject.toml
|
git add packages/*/pyproject.toml
|
||||||
git commit -m "chore: release v${{ inputs.version }}"
|
git commit -m "chore: release v${{ inputs.version }}"
|
||||||
|
git push origin main
|
||||||
|
|
||||||
|
COMMIT_SHA=$(git rev-parse HEAD)
|
||||||
|
echo "commit-sha=$COMMIT_SHA" >> $GITHUB_OUTPUT
|
||||||
|
echo "✅ Pushed version update: $COMMIT_SHA"
|
||||||
|
|
||||||
|
build-packages:
|
||||||
|
name: Build packages
|
||||||
|
needs: update-version
|
||||||
|
uses: ./.github/workflows/build-reusable.yml
|
||||||
|
with:
|
||||||
|
ref: ${{ needs.update-version.outputs.commit-sha }}
|
||||||
|
|
||||||
|
publish:
|
||||||
|
name: Publish and Release
|
||||||
|
needs: build-packages
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
contents: write
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
ref: ${{ needs.update-version.outputs.commit-sha }}
|
||||||
|
|
||||||
- name: Get CI run ID
|
- name: Download all artifacts
|
||||||
id: get-ci-run
|
uses: actions/download-artifact@v4
|
||||||
run: |
|
with:
|
||||||
# Get the latest successful CI run on the previous commit (before version bump)
|
path: dist-artifacts
|
||||||
COMMIT_SHA=$(git rev-parse HEAD~1)
|
|
||||||
RUN_ID=$(gh run list \
|
|
||||||
--workflow="CI - Build Multi-Platform Packages" \
|
|
||||||
--status=success \
|
|
||||||
--commit=$COMMIT_SHA \
|
|
||||||
--json databaseId \
|
|
||||||
--jq '.[0].databaseId')
|
|
||||||
|
|
||||||
if [ -z "$RUN_ID" ]; then
|
|
||||||
echo "❌ No successful CI run found for commit $COMMIT_SHA"
|
|
||||||
echo ""
|
|
||||||
echo "This usually means:"
|
|
||||||
echo "1. CI hasn't run on the latest commit yet"
|
|
||||||
echo "2. CI failed on the latest commit"
|
|
||||||
echo ""
|
|
||||||
echo "Please ensure CI passes on main branch before releasing."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "✅ Found CI run: $RUN_ID"
|
|
||||||
echo "run-id=$RUN_ID" >> $GITHUB_OUTPUT
|
|
||||||
env:
|
|
||||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
|
|
||||||
- name: Download artifacts from CI run
|
- name: Collect packages
|
||||||
run: |
|
run: |
|
||||||
echo "📦 Downloading artifacts from CI run ${{ steps.get-ci-run.outputs.run-id }}..."
|
mkdir -p dist
|
||||||
|
find dist-artifacts -name "*.whl" -exec cp {} dist/ \;
|
||||||
|
find dist-artifacts -name "*.tar.gz" -exec cp {} dist/ \;
|
||||||
|
|
||||||
# Download all artifacts (not just wheels-*)
|
echo "📦 Packages to publish:"
|
||||||
gh run download ${{ steps.get-ci-run.outputs.run-id }} \
|
ls -la dist/
|
||||||
--dir ./dist-downloads
|
|
||||||
|
|
||||||
# Consolidate all wheels into packages/*/dist/
|
|
||||||
mkdir -p packages/leann-core/dist
|
|
||||||
mkdir -p packages/leann-backend-hnsw/dist
|
|
||||||
mkdir -p packages/leann-backend-diskann/dist
|
|
||||||
mkdir -p packages/leann/dist
|
|
||||||
|
|
||||||
find ./dist-downloads -name "*.whl" -exec cp {} ./packages/ \;
|
|
||||||
|
|
||||||
# Move wheels to correct package directories
|
|
||||||
for wheel in packages/*.whl; do
|
|
||||||
if [[ $wheel == *"leann_core"* ]]; then
|
|
||||||
mv "$wheel" packages/leann-core/dist/
|
|
||||||
elif [[ $wheel == *"leann_backend_hnsw"* ]]; then
|
|
||||||
mv "$wheel" packages/leann-backend-hnsw/dist/
|
|
||||||
elif [[ $wheel == *"leann_backend_diskann"* ]]; then
|
|
||||||
mv "$wheel" packages/leann-backend-diskann/dist/
|
|
||||||
elif [[ $wheel == *"leann-"* ]] && [[ $wheel != *"backend"* ]] && [[ $wheel != *"core"* ]]; then
|
|
||||||
mv "$wheel" packages/leann/dist/
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
# List downloaded wheels
|
|
||||||
echo "✅ Downloaded wheels:"
|
|
||||||
find packages/*/dist -name "*.whl" -type f | sort
|
|
||||||
env:
|
|
||||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
|
|
||||||
- name: Test on TestPyPI (optional)
|
|
||||||
if: inputs.test_pypi
|
|
||||||
continue-on-error: true
|
|
||||||
env:
|
|
||||||
TWINE_USERNAME: __token__
|
|
||||||
TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }}
|
|
||||||
run: |
|
|
||||||
if [ -z "$TWINE_PASSWORD" ]; then
|
|
||||||
echo "⚠️ TEST_PYPI_API_TOKEN not configured, skipping TestPyPI upload"
|
|
||||||
echo " To enable TestPyPI testing, add TEST_PYPI_API_TOKEN to repository secrets"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
pip install twine
|
|
||||||
echo "📦 Uploading to TestPyPI..."
|
|
||||||
twine upload --repository testpypi packages/*/dist/* --verbose || {
|
|
||||||
echo "⚠️ TestPyPI upload failed, but continuing with release"
|
|
||||||
echo " This is optional and won't block the release"
|
|
||||||
exit 0
|
|
||||||
}
|
|
||||||
echo "✅ Test upload successful!"
|
|
||||||
echo "📋 Check packages at: https://test.pypi.org/user/your-username/"
|
|
||||||
echo ""
|
|
||||||
echo "To test installation:"
|
|
||||||
echo "pip install -i https://test.pypi.org/simple/ leann"
|
|
||||||
|
|
||||||
- name: Publish to PyPI
|
- name: Publish to PyPI
|
||||||
env:
|
env:
|
||||||
@@ -161,46 +82,22 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
if [ -z "$TWINE_PASSWORD" ]; then
|
if [ -z "$TWINE_PASSWORD" ]; then
|
||||||
echo "❌ PYPI_API_TOKEN not configured!"
|
echo "❌ PYPI_API_TOKEN not configured!"
|
||||||
echo " Please add PYPI_API_TOKEN to repository secrets"
|
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
pip install twine
|
pip install twine
|
||||||
echo "📦 Publishing to PyPI..."
|
twine upload dist/* --skip-existing --verbose
|
||||||
|
|
||||||
# Collect all wheels in one place
|
|
||||||
mkdir -p all_wheels
|
|
||||||
find packages/*/dist -name "*.whl" -exec cp {} all_wheels/ \;
|
|
||||||
find packages/*/dist -name "*.tar.gz" -exec cp {} all_wheels/ \;
|
|
||||||
|
|
||||||
echo "📋 Packages to publish:"
|
|
||||||
ls -la all_wheels/
|
|
||||||
|
|
||||||
# Upload to PyPI
|
|
||||||
twine upload all_wheels/* --skip-existing --verbose
|
|
||||||
|
|
||||||
echo "✅ Published to PyPI!"
|
echo "✅ Published to PyPI!"
|
||||||
echo "🎉 Check packages at: https://pypi.org/project/leann/"
|
|
||||||
|
|
||||||
- name: Create and push tag
|
- name: Create release
|
||||||
run: |
|
run: |
|
||||||
git tag "v${{ inputs.version }}"
|
git tag "v${{ inputs.version }}"
|
||||||
git push origin main
|
|
||||||
git push origin "v${{ inputs.version }}"
|
git push origin "v${{ inputs.version }}"
|
||||||
echo "✅ Tag v${{ inputs.version }} created and pushed"
|
|
||||||
|
gh release create "v${{ inputs.version }}" \
|
||||||
- name: Create GitHub Release
|
--title "Release v${{ inputs.version }}" \
|
||||||
uses: softprops/action-gh-release@v1
|
--notes "🚀 Released to PyPI: https://pypi.org/project/leann/${{ inputs.version }}/" \
|
||||||
with:
|
--latest
|
||||||
tag_name: v${{ inputs.version }}
|
env:
|
||||||
name: Release v${{ inputs.version }}
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
body: |
|
|
||||||
## 🚀 Release v${{ inputs.version }}
|
|
||||||
|
|
||||||
### What's Changed
|
|
||||||
See the [full changelog](https://github.com/${{ github.repository }}/compare/...v${{ inputs.version }})
|
|
||||||
|
|
||||||
### Installation
|
|
||||||
```bash
|
|
||||||
pip install leann==${{ inputs.version }}
|
|
||||||
```
|
|
||||||
@@ -94,7 +94,7 @@ ollama pull llama3.2:1b
|
|||||||
## Quick Start in 30s
|
## Quick Start in 30s
|
||||||
|
|
||||||
Our declarative API makes RAG as easy as writing a config file.
|
Our declarative API makes RAG as easy as writing a config file.
|
||||||
[Try in this ipynb file →](demo.ipynb)
|
[Try in this ipynb file →](demo.ipynb) [](https://colab.research.google.com/github/yichuan-w/LEANN/blob/main/demo.ipynb)
|
||||||
|
|
||||||
```python
|
```python
|
||||||
from leann.api import LeannBuilder, LeannSearcher, LeannChat
|
from leann.api import LeannBuilder, LeannSearcher, LeannChat
|
||||||
|
|||||||
108
docs/RELEASE.md
108
docs/RELEASE.md
@@ -1,100 +1,22 @@
|
|||||||
# Release Guide
|
# Release Guide
|
||||||
|
|
||||||
## 📋 Prerequisites
|
## Setup (One-time)
|
||||||
|
|
||||||
Before releasing, ensure:
|
Add `PYPI_API_TOKEN` to GitHub Secrets:
|
||||||
1. ✅ All code changes are committed and pushed
|
1. Get token: https://pypi.org/manage/account/token/
|
||||||
2. ✅ CI has passed on the latest commit (check [Actions](https://github.com/yichuan-w/LEANN/actions/workflows/ci.yml))
|
2. Add to secrets: Settings → Secrets → Actions → `PYPI_API_TOKEN`
|
||||||
3. ✅ You have determined the new version number
|
|
||||||
|
|
||||||
### Required: PyPI Configuration
|
## Release (One-click)
|
||||||
|
|
||||||
To enable PyPI publishing:
|
1. Go to: https://github.com/yichuan-w/LEANN/actions/workflows/release-manual.yml
|
||||||
1. Get a PyPI API token from https://pypi.org/manage/account/token/
|
2. Click "Run workflow"
|
||||||
2. Add it to repository secrets: Settings → Secrets → Actions → New repository secret
|
3. Enter version: `0.1.2`
|
||||||
- Name: `PYPI_API_TOKEN`
|
4. Click green "Run workflow" button
|
||||||
- Value: Your PyPI token (starts with `pypi-`)
|
|
||||||
|
|
||||||
### Optional: TestPyPI Configuration
|
That's it! The workflow will automatically:
|
||||||
|
- ✅ Update version in all packages
|
||||||
|
- ✅ Build all packages
|
||||||
|
- ✅ Publish to PyPI
|
||||||
|
- ✅ Create GitHub tag and release
|
||||||
|
|
||||||
To enable TestPyPI testing (recommended but not required):
|
Check progress: https://github.com/yichuan-w/LEANN/actions
|
||||||
1. Get a TestPyPI API token from https://test.pypi.org/manage/account/token/
|
|
||||||
2. Add it to repository secrets: Settings → Secrets → Actions → New repository secret
|
|
||||||
- Name: `TEST_PYPI_API_TOKEN`
|
|
||||||
- Value: Your TestPyPI token (starts with `pypi-`)
|
|
||||||
|
|
||||||
**Note**: TestPyPI testing is optional. If not configured, the release will skip TestPyPI and proceed.
|
|
||||||
|
|
||||||
## 🚀 Recommended: Manual Release Workflow
|
|
||||||
|
|
||||||
### Via GitHub UI (Most Reliable)
|
|
||||||
|
|
||||||
1. **Verify CI Status**: Check that the latest commit has a green checkmark ✅
|
|
||||||
2. Go to [Actions → Manual Release](https://github.com/yichuan-w/LEANN/actions/workflows/release-manual.yml)
|
|
||||||
3. Click "Run workflow"
|
|
||||||
4. Enter version (e.g., `0.1.1`)
|
|
||||||
5. Toggle "Test on TestPyPI first" if desired
|
|
||||||
6. Click "Run workflow"
|
|
||||||
|
|
||||||
**What happens:**
|
|
||||||
- ✅ Downloads pre-built packages from CI (no rebuild needed!)
|
|
||||||
- ✅ Updates all package versions
|
|
||||||
- ✅ Optionally tests on TestPyPI
|
|
||||||
- ✅ **Publishes directly to PyPI**
|
|
||||||
- ✅ Creates tag and GitHub release
|
|
||||||
|
|
||||||
### Via Command Line
|
|
||||||
|
|
||||||
```bash
|
|
||||||
gh workflow run release-manual.yml -f version=0.1.1 -f test_pypi=true
|
|
||||||
```
|
|
||||||
|
|
||||||
## ⚡ Quick Release (One-Line)
|
|
||||||
|
|
||||||
For experienced users who want the fastest path:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./scripts/release.sh 0.1.1
|
|
||||||
```
|
|
||||||
|
|
||||||
This script will:
|
|
||||||
1. Update all package versions
|
|
||||||
2. Commit and push changes
|
|
||||||
3. Create GitHub release
|
|
||||||
4. **Manual Release workflow will automatically publish to PyPI**
|
|
||||||
|
|
||||||
⚠️ **Note**: If CI fails, you'll need to manually fix and re-tag
|
|
||||||
|
|
||||||
## Manual Testing Before Release
|
|
||||||
|
|
||||||
For testing specific packages locally (especially DiskANN on macOS):
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Build specific package locally
|
|
||||||
./scripts/build_and_test.sh diskann # or hnsw, core, meta, all
|
|
||||||
|
|
||||||
# Test installation in a clean environment
|
|
||||||
python -m venv test_env
|
|
||||||
source test_env/bin/activate
|
|
||||||
pip install packages/*/dist/*.whl
|
|
||||||
|
|
||||||
# Upload to Test PyPI (optional)
|
|
||||||
./scripts/upload_to_pypi.sh test
|
|
||||||
|
|
||||||
# Upload to Production PyPI (use with caution)
|
|
||||||
./scripts/upload_to_pypi.sh prod
|
|
||||||
```
|
|
||||||
|
|
||||||
## First-time setup
|
|
||||||
|
|
||||||
1. Install GitHub CLI:
|
|
||||||
```bash
|
|
||||||
brew install gh
|
|
||||||
gh auth login
|
|
||||||
```
|
|
||||||
|
|
||||||
2. Set PyPI token in GitHub:
|
|
||||||
```bash
|
|
||||||
gh secret set PYPI_API_TOKEN
|
|
||||||
# Paste your PyPI token when prompted
|
|
||||||
```
|
|
||||||
@@ -4,8 +4,8 @@ build-backend = "scikit_build_core.build"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "leann-backend-diskann"
|
name = "leann-backend-diskann"
|
||||||
version = "0.1.2"
|
version = "0.1.6"
|
||||||
dependencies = ["leann-core==0.1.2", "numpy"]
|
dependencies = ["leann-core==0.1.6", "numpy"]
|
||||||
|
|
||||||
[tool.scikit-build]
|
[tool.scikit-build]
|
||||||
# Key: simplified CMake path
|
# Key: simplified CMake path
|
||||||
|
|||||||
@@ -6,10 +6,10 @@ build-backend = "scikit_build_core.build"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "leann-backend-hnsw"
|
name = "leann-backend-hnsw"
|
||||||
version = "0.1.2"
|
version = "0.1.6"
|
||||||
description = "Custom-built HNSW (Faiss) backend for the Leann toolkit."
|
description = "Custom-built HNSW (Faiss) backend for the Leann toolkit."
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"leann-core==0.1.2",
|
"leann-core==0.1.6",
|
||||||
"numpy",
|
"numpy",
|
||||||
"pyzmq>=23.0.0",
|
"pyzmq>=23.0.0",
|
||||||
"msgpack>=1.0.0",
|
"msgpack>=1.0.0",
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "leann-core"
|
name = "leann-core"
|
||||||
version = "0.1.2"
|
version = "0.1.6"
|
||||||
description = "Core API and plugin system for LEANN"
|
description = "Core API and plugin system for LEANN"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.9"
|
requires-python = ">=3.9"
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "leann"
|
name = "leann"
|
||||||
version = "0.1.2"
|
version = "0.1.6"
|
||||||
description = "LEANN - The smallest vector index in the world. RAG Everything with LEANN!"
|
description = "LEANN - The smallest vector index in the world. RAG Everything with LEANN!"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.9"
|
requires-python = ">=3.9"
|
||||||
|
|||||||
Reference in New Issue
Block a user