TBNilles/LEANN
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: true one-click automated release with multi-platform support

Browse Source
This commit is contained in:
Andy Lee
2025-07-24 19:30:44 -07:00
parent d45c013806
commit 32a374d094
2 changed files with 137 additions and 342 deletions
Download Patch File Download Diff File Expand all files Collapse all files

360
.github/workflows/release-manual.yml vendored
View File

@@ -1,241 +1,149 @@
name: Manual Release
name: Release
on:
workflow_dispatch:
inputs:
version:
description: 'Version to release (e.g., 0.1.1)'
description: 'Version to release (e.g., 0.1.2)'
required: true
type: string
test_pypi:
description: 'Test on TestPyPI first'
required: false
type: boolean
default: true
jobs:
validate-and-release:
update-version:
name: Update Version
runs-on: ubuntu-latest
permissions:
contents: write
actions: read
outputs:
commit-sha: ${{ steps.push.outputs.commit-sha }}
steps:
- uses: actions/checkout@v4
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Check CI status
run: |
echo "ℹ️ This workflow will download build artifacts from the latest CI run."
echo " CI must have completed successfully on the current commit."
echo ""
- name: Validate version format
- name: Validate version
run: |
if ! [[ "${{ inputs.version }}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
echo "❌ Invalid version format. Use semantic versioning (e.g., 0.1.1)"
echo "❌ Invalid version format"
exit 1
fi
echo "✅ Version format valid: ${{ inputs.version }}"
echo "✅ Version format valid"
- name: Check if version already exists
run: |
if git tag | grep -q "^v${{ inputs.version }}$"; then
echo "❌ Version v${{ inputs.version }} already exists!"
exit 1
fi
echo "✅ Version is new"
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.13'
- name: Install uv
run: |
curl -LsSf https://astral.sh/uv/install.sh | sh
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
- name: Update versions
- name: Update versions and push
id: push
run: |
./scripts/bump_version.sh ${{ inputs.version }}
git config user.name "GitHub Actions"
git config user.email "actions@github.com"
git add packages/*/pyproject.toml
git commit -m "chore: release v${{ inputs.version }}"
- name: Push version update
run: |
git push origin HEAD:main
echo "✅ Pushed version update to main branch"
git push origin main
COMMIT_SHA=$(git rev-parse HEAD)
echo "commit-sha=$COMMIT_SHA" >> $GITHUB_OUTPUT
id: push-version
- name: Trigger CI build
run: |
echo "🚀 Manually triggering CI for the new version..."
# Check if we have a PAT for triggering workflows
if [ -z "${{ secrets.WORKFLOW_PAT }}" ]; then
echo "⚠️ No WORKFLOW_PAT found. CI will be triggered by the push event."
echo " Note: If CI doesn't trigger automatically, you'll need to:"
echo " 1. Add a Personal Access Token with 'workflow' scope as WORKFLOW_PAT secret"
echo " 2. Or manually run the CI workflow after this release completes"
exit 0
fi
gh workflow run "CI - Build Multi-Platform Packages" \
--ref main \
-f publish=false
# Give GitHub a moment to register the new workflow run
sleep 5
env:
GH_TOKEN: ${{ secrets.WORKFLOW_PAT || secrets.GITHUB_TOKEN }}
- name: Wait for CI to complete
id: wait-for-ci
run: |
echo "⏳ Waiting for CI to build new version..."
COMMIT_SHA="${{ steps.push-version.outputs.commit-sha }}"
# First, wait a bit for CI to potentially start
echo "⏳ Waiting for CI to start..."
sleep 30
# Check if there's any CI run for this commit
CI_EXISTS=$(gh run list \
--workflow="CI - Build Multi-Platform Packages" \
--commit=$COMMIT_SHA \
--json databaseId \
--jq 'length')
if [ "$CI_EXISTS" -eq "0" ]; then
echo "⚠️ No CI run found for commit $COMMIT_SHA"
echo " This might be because:"
echo " 1. WORKFLOW_PAT is not configured"
echo " 2. CI hasn't started yet"
echo ""
echo " You can manually trigger CI after this release completes:"
echo " gh workflow run 'CI - Build Multi-Platform Packages' --ref main"
echo ""
echo " For now, we'll use the artifacts from the latest successful CI run."
# Get the latest successful CI run
LATEST_RUN=$(gh run list \
--workflow="CI - Build Multi-Platform Packages" \
--status=success \
--json databaseId \
--jq '.[0].databaseId')
if [ -z "$LATEST_RUN" ]; then
echo "❌ No successful CI runs found!"
exit 1
fi
echo "📦 Using artifacts from CI run: $LATEST_RUN"
echo "run-id=$LATEST_RUN" >> $GITHUB_OUTPUT
exit 0
fi
# Wait up to 20 minutes for CI to complete
for i in {1..40}; do
# First check if CI is running
RUNNING_ID=$(gh run list \
--workflow="CI - Build Multi-Platform Packages" \
--commit=$COMMIT_SHA \
--status=in_progress \
--json databaseId \
--jq '.[0].databaseId')
if [ ! -z "$RUNNING_ID" ]; then
echo "⏳ CI is running (ID: $RUNNING_ID)..."
fi
# Check if CI has completed
RUN_ID=$(gh run list \
--workflow="CI - Build Multi-Platform Packages" \
--commit=$COMMIT_SHA \
--json databaseId,status,conclusion \
--jq '.[] | select(.status == "completed" and .conclusion == "success") | .databaseId' | head -1)
if [ ! -z "$RUN_ID" ]; then
echo "✅ Found completed CI run: $RUN_ID"
echo "run-id=$RUN_ID" >> $GITHUB_OUTPUT
exit 0
fi
echo "⏳ Waiting for CI... (attempt $i/40)"
sleep 30
done
echo "❌ CI did not complete within 20 minutes"
exit 1
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
echo "✅ Pushed version update: $COMMIT_SHA"
build-packages:
name: Build packages
needs: update-version
strategy:
matrix:
include:
- os: ubuntu-latest
python: '3.11'
- os: macos-latest
python: '3.11'
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
with:
ref: ${{ needs.update-version.outputs.commit-sha }}
submodules: recursive
- name: Download artifacts from CI run
run: |
echo "📦 Downloading artifacts from CI run ${{ steps.wait-for-ci.outputs.run-id }}..."
# Download all artifacts (not just wheels-*)
gh run download ${{ steps.wait-for-ci.outputs.run-id }} \
--dir ./dist-downloads
# Consolidate all wheels into packages/*/dist/
mkdir -p packages/leann-core/dist
mkdir -p packages/leann-backend-hnsw/dist
mkdir -p packages/leann-backend-diskann/dist
mkdir -p packages/leann/dist
find ./dist-downloads -name "*.whl" -exec cp {} ./packages/ \;
# Move wheels to correct package directories
for wheel in packages/*.whl; do
if [[ $wheel == *"leann_core"* ]]; then
mv "$wheel" packages/leann-core/dist/
elif [[ $wheel == *"leann_backend_hnsw"* ]]; then
mv "$wheel" packages/leann-backend-hnsw/dist/
elif [[ $wheel == *"leann_backend_diskann"* ]]; then
mv "$wheel" packages/leann-backend-diskann/dist/
elif [[ $wheel == *"leann-"* ]] && [[ $wheel != *"backend"* ]] && [[ $wheel != *"core"* ]]; then
mv "$wheel" packages/leann/dist/
fi
done
# List downloaded wheels
echo "✅ Downloaded wheels:"
find packages/*/dist -name "*.whl" -type f | sort
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: Test on TestPyPI (optional)
if: inputs.test_pypi
continue-on-error: true
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }}
- name: Install uv
uses: astral-sh/setup-uv@v4
- name: Install system dependencies (Ubuntu)
if: runner.os == 'Linux'
run: |
if [ -z "$TWINE_PASSWORD" ]; then
echo "⚠️ TEST_PYPI_API_TOKEN not configured, skipping TestPyPI upload"
echo " To enable TestPyPI testing, add TEST_PYPI_API_TOKEN to repository secrets"
exit 0
sudo apt-get update
sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev
- name: Install system dependencies (macOS)
if: runner.os == 'macOS'
run: |
brew install llvm libomp boost protobuf zeromq
- name: Build packages
run: |
# Build core (platform independent)
if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
cd packages/leann-core
uv build
cd ../..
fi
pip install twine
echo "📦 Uploading to TestPyPI..."
twine upload --repository testpypi packages/*/dist/* --verbose || {
echo "⚠️ TestPyPI upload failed, but continuing with release"
echo " This is optional and won't block the release"
exit 0
}
echo "✅ Test upload successful!"
echo "📋 Check packages at: https://test.pypi.org/user/your-username/"
echo ""
echo "To test installation:"
echo "pip install -i https://test.pypi.org/simple/ leann"
# Build HNSW backend
cd packages/leann-backend-hnsw
uv pip install --system -r pyproject.toml --extra build
CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build
cd ../..
# Build DiskANN backend
cd packages/leann-backend-diskann
uv pip install --system -r pyproject.toml --extra build
CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build
cd ../..
# Build meta package (platform independent)
if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
cd packages/leann
uv build
cd ../..
fi
echo "📦 Built packages:"
find packages/*/dist -name "*.whl" -o -name "*.tar.gz" | sort
env:
CC: ${{ runner.os == 'macOS' && '$(brew --prefix llvm)/bin/clang' || '' }}
CXX: ${{ runner.os == 'macOS' && '$(brew --prefix llvm)/bin/clang++' || '' }}
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: packages-${{ matrix.os }}-${{ matrix.python }}
path: packages/*/dist/
publish:
name: Publish and Release
needs: build-packages
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- uses: actions/checkout@v4
with:
ref: ${{ needs.update-version.outputs.commit-sha }}
- name: Download all artifacts
uses: actions/download-artifact@v4
with:
path: dist-artifacts
- name: Collect packages
run: |
mkdir -p dist
find dist-artifacts -name "*.whl" -exec cp {} dist/ \;
find dist-artifacts -name "*.tar.gz" -exec cp {} dist/ \;
echo "📦 Packages to publish:"
ls -la dist/
- name: Publish to PyPI
env:
@@ -244,46 +152,22 @@ jobs:
run: |
if [ -z "$TWINE_PASSWORD" ]; then
echo "❌ PYPI_API_TOKEN not configured!"
echo " Please add PYPI_API_TOKEN to repository secrets"
exit 1
fi
pip install twine
echo "📦 Publishing to PyPI..."
# Collect all wheels in one place
mkdir -p all_wheels
find packages/*/dist -name "*.whl" -exec cp {} all_wheels/ \;
find packages/*/dist -name "*.tar.gz" -exec cp {} all_wheels/ \;
echo "📋 Packages to publish:"
ls -la all_wheels/
# Upload to PyPI
twine upload all_wheels/* --skip-existing --verbose
twine upload dist/* --skip-existing --verbose
echo "✅ Published to PyPI!"
echo "🎉 Check packages at: https://pypi.org/project/leann/"
- name: Create and push tag
- name: Create release
run: |
git tag "v${{ inputs.version }}"
git push origin main
git push origin "v${{ inputs.version }}"
echo "✅ Tag v${{ inputs.version }} created and pushed"
- name: Create GitHub Release
uses: softprops/action-gh-release@v1
with:
tag_name: v${{ inputs.version }}
name: Release v${{ inputs.version }}
body: |
## 🚀 Release v${{ inputs.version }}
### What's Changed
See the [full changelog](https://github.com/${{ github.repository }}/compare/...v${{ inputs.version }})
### Installation
```bash
pip install leann==${{ inputs.version }}
```
gh release create "v${{ inputs.version }}" \
--title "Release v${{ inputs.version }}" \
--notes "🚀 Released to PyPI: https://pypi.org/project/leann/${{ inputs.version }}/" \
--latest
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

119
docs/RELEASE.md
View File

@@ -1,111 +1,22 @@
# Release Guide
## Required: PyPI Configuration
## Setup (One-time)
Before releasing, ensure you have configured the PyPI API token:
Add `PYPI_API_TOKEN` to GitHub Secrets:
1. Get token: https://pypi.org/manage/account/token/
2. Add to secrets: Settings → Secrets → Actions → `PYPI_API_TOKEN`
1. Generate API token at https://pypi.org/manage/account/token/
2. Add as GitHub secret: `PYPI_API_TOKEN`
3. For full automation, also add a Personal Access Token:
- Create PAT at https://github.com/settings/tokens with `workflow` scope
- Add as GitHub secret: `WORKFLOW_PAT`
- This allows the release workflow to trigger CI builds automatically
## Release (One-click)
## 📋 Prerequisites
1. Go to: https://github.com/yichuan-w/LEANN/actions/workflows/release-manual.yml
2. Click "Run workflow"
3. Enter version: `0.1.2`
4. Click green "Run workflow" button
Before releasing, ensure:
1. ✅ All code changes are committed and pushed
2. ✅ CI has passed on the latest commit (check [Actions](https://github.com/yichuan-w/LEANN/actions/workflows/ci.yml))
3. ✅ You have determined the new version number
That's it! The workflow will automatically:
- ✅ Update version in all packages
- ✅ Build all packages
- ✅ Publish to PyPI
- ✅ Create GitHub tag and release
### Required: PyPI Configuration
To enable PyPI publishing:
1. Get a PyPI API token from https://pypi.org/manage/account/token/
2. Add it to repository secrets: Settings → Secrets → Actions → New repository secret
- Name: `PYPI_API_TOKEN`
- Value: Your PyPI token (starts with `pypi-`)
### Optional: TestPyPI Configuration
To enable TestPyPI testing (recommended but not required):
1. Get a TestPyPI API token from https://test.pypi.org/manage/account/token/
2. Add it to repository secrets: Settings → Secrets → Actions → New repository secret
- Name: `TEST_PYPI_API_TOKEN`
- Value: Your TestPyPI token (starts with `pypi-`)
**Note**: TestPyPI testing is optional. If not configured, the release will skip TestPyPI and proceed.
## 🚀 Recommended: Manual Release Workflow
### Via GitHub UI (Most Reliable)
1. **Verify CI Status**: Check that the latest commit has a green checkmark ✅
2. Go to [Actions → Manual Release](https://github.com/yichuan-w/LEANN/actions/workflows/release-manual.yml)
3. Click "Run workflow"
4. Enter version (e.g., `0.1.1`)
5. Toggle "Test on TestPyPI first" if desired
6. Click "Run workflow"
**What happens:**
- ✅ Downloads pre-built packages from CI (no rebuild needed!)
- ✅ Updates all package versions
- ✅ Optionally tests on TestPyPI
- ✅ **Publishes directly to PyPI**
- ✅ Creates tag and GitHub release
### Via Command Line
```bash
gh workflow run release-manual.yml -f version=0.1.1 -f test_pypi=true
```
## ⚡ Quick Release (One-Line)
For experienced users who want the fastest path:
```bash
./scripts/release.sh 0.1.1
```
This script will:
1. Update all package versions
2. Commit and push changes
3. Create GitHub release
4. **Manual Release workflow will automatically publish to PyPI**
⚠️ **Note**: If CI fails, you'll need to manually fix and re-tag
## Manual Testing Before Release
For testing specific packages locally (especially DiskANN on macOS):
```bash
# Build specific package locally
./scripts/build_and_test.sh diskann # or hnsw, core, meta, all
# Test installation in a clean environment
python -m venv test_env
source test_env/bin/activate
pip install packages/*/dist/*.whl
# Upload to Test PyPI (optional)
./scripts/upload_to_pypi.sh test
# Upload to Production PyPI (use with caution)
./scripts/upload_to_pypi.sh prod
```
## First-time setup
1. Install GitHub CLI:
```bash
brew install gh
gh auth login
```
2. Set PyPI token in GitHub:
```bash
gh secret set PYPI_API_TOKEN
# Paste your PyPI token when prompted
```
Check progress: https://github.com/yichuan-w/LEANN/actions