From 32a374d09429fa6f0d0bfe820163ad493b78f446 Mon Sep 17 00:00:00 2001 From: Andy Lee Date: Thu, 24 Jul 2025 19:30:44 -0700 Subject: [PATCH] feat: true one-click automated release with multi-platform support --- .github/workflows/release-manual.yml | 360 +++++++++------------------ docs/RELEASE.md | 119 ++------- 2 files changed, 137 insertions(+), 342 deletions(-) diff --git a/.github/workflows/release-manual.yml b/.github/workflows/release-manual.yml index d2da5ae..e0f9d47 100644 --- a/.github/workflows/release-manual.yml +++ b/.github/workflows/release-manual.yml @@ -1,241 +1,149 @@ -name: Manual Release +name: Release on: workflow_dispatch: inputs: version: - description: 'Version to release (e.g., 0.1.1)' + description: 'Version to release (e.g., 0.1.2)' required: true type: string - test_pypi: - description: 'Test on TestPyPI first' - required: false - type: boolean - default: true jobs: - validate-and-release: + update-version: + name: Update Version runs-on: ubuntu-latest - permissions: - contents: write - actions: read - + outputs: + commit-sha: ${{ steps.push.outputs.commit-sha }} + steps: - uses: actions/checkout@v4 with: token: ${{ secrets.GITHUB_TOKEN }} - - name: Check CI status - run: | - echo "ℹ️ This workflow will download build artifacts from the latest CI run." - echo " CI must have completed successfully on the current commit." - echo "" - - - name: Validate version format + - name: Validate version run: | if ! [[ "${{ inputs.version }}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "❌ Invalid version format. Use semantic versioning (e.g., 0.1.1)" + echo "❌ Invalid version format" exit 1 fi - echo "✅ Version format valid: ${{ inputs.version }}" + echo "✅ Version format valid" - - name: Check if version already exists - run: | - if git tag | grep -q "^v${{ inputs.version }}$"; then - echo "❌ Version v${{ inputs.version }} already exists!" - exit 1 - fi - echo "✅ Version is new" - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.13' - - - name: Install uv - run: | - curl -LsSf https://astral.sh/uv/install.sh | sh - echo "$HOME/.cargo/bin" >> $GITHUB_PATH - - - name: Update versions + - name: Update versions and push + id: push run: | ./scripts/bump_version.sh ${{ inputs.version }} git config user.name "GitHub Actions" git config user.email "actions@github.com" git add packages/*/pyproject.toml git commit -m "chore: release v${{ inputs.version }}" - - - name: Push version update - run: | - git push origin HEAD:main - echo "✅ Pushed version update to main branch" + git push origin main + COMMIT_SHA=$(git rev-parse HEAD) echo "commit-sha=$COMMIT_SHA" >> $GITHUB_OUTPUT - id: push-version - - - name: Trigger CI build - run: | - echo "🚀 Manually triggering CI for the new version..." - - # Check if we have a PAT for triggering workflows - if [ -z "${{ secrets.WORKFLOW_PAT }}" ]; then - echo "⚠️ No WORKFLOW_PAT found. CI will be triggered by the push event." - echo " Note: If CI doesn't trigger automatically, you'll need to:" - echo " 1. Add a Personal Access Token with 'workflow' scope as WORKFLOW_PAT secret" - echo " 2. Or manually run the CI workflow after this release completes" - exit 0 - fi - - gh workflow run "CI - Build Multi-Platform Packages" \ - --ref main \ - -f publish=false - - # Give GitHub a moment to register the new workflow run - sleep 5 - env: - GH_TOKEN: ${{ secrets.WORKFLOW_PAT || secrets.GITHUB_TOKEN }} - - - name: Wait for CI to complete - id: wait-for-ci - run: | - echo "⏳ Waiting for CI to build new version..." - COMMIT_SHA="${{ steps.push-version.outputs.commit-sha }}" - - # First, wait a bit for CI to potentially start - echo "⏳ Waiting for CI to start..." - sleep 30 - - # Check if there's any CI run for this commit - CI_EXISTS=$(gh run list \ - --workflow="CI - Build Multi-Platform Packages" \ - --commit=$COMMIT_SHA \ - --json databaseId \ - --jq 'length') - - if [ "$CI_EXISTS" -eq "0" ]; then - echo "⚠️ No CI run found for commit $COMMIT_SHA" - echo " This might be because:" - echo " 1. WORKFLOW_PAT is not configured" - echo " 2. CI hasn't started yet" - echo "" - echo " You can manually trigger CI after this release completes:" - echo " gh workflow run 'CI - Build Multi-Platform Packages' --ref main" - echo "" - echo " For now, we'll use the artifacts from the latest successful CI run." - - # Get the latest successful CI run - LATEST_RUN=$(gh run list \ - --workflow="CI - Build Multi-Platform Packages" \ - --status=success \ - --json databaseId \ - --jq '.[0].databaseId') - - if [ -z "$LATEST_RUN" ]; then - echo "❌ No successful CI runs found!" - exit 1 - fi - - echo "📦 Using artifacts from CI run: $LATEST_RUN" - echo "run-id=$LATEST_RUN" >> $GITHUB_OUTPUT - exit 0 - fi - - # Wait up to 20 minutes for CI to complete - for i in {1..40}; do - # First check if CI is running - RUNNING_ID=$(gh run list \ - --workflow="CI - Build Multi-Platform Packages" \ - --commit=$COMMIT_SHA \ - --status=in_progress \ - --json databaseId \ - --jq '.[0].databaseId') - - if [ ! -z "$RUNNING_ID" ]; then - echo "⏳ CI is running (ID: $RUNNING_ID)..." - fi - - # Check if CI has completed - RUN_ID=$(gh run list \ - --workflow="CI - Build Multi-Platform Packages" \ - --commit=$COMMIT_SHA \ - --json databaseId,status,conclusion \ - --jq '.[] | select(.status == "completed" and .conclusion == "success") | .databaseId' | head -1) - - if [ ! -z "$RUN_ID" ]; then - echo "✅ Found completed CI run: $RUN_ID" - echo "run-id=$RUN_ID" >> $GITHUB_OUTPUT - exit 0 - fi - - echo "⏳ Waiting for CI... (attempt $i/40)" - sleep 30 - done - - echo "❌ CI did not complete within 20 minutes" - exit 1 - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + echo "✅ Pushed version update: $COMMIT_SHA" + + build-packages: + name: Build packages + needs: update-version + strategy: + matrix: + include: + - os: ubuntu-latest + python: '3.11' + - os: macos-latest + python: '3.11' + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.update-version.outputs.commit-sha }} + submodules: recursive - - name: Download artifacts from CI run - run: | - echo "📦 Downloading artifacts from CI run ${{ steps.wait-for-ci.outputs.run-id }}..." - - # Download all artifacts (not just wheels-*) - gh run download ${{ steps.wait-for-ci.outputs.run-id }} \ - --dir ./dist-downloads - - # Consolidate all wheels into packages/*/dist/ - mkdir -p packages/leann-core/dist - mkdir -p packages/leann-backend-hnsw/dist - mkdir -p packages/leann-backend-diskann/dist - mkdir -p packages/leann/dist - - find ./dist-downloads -name "*.whl" -exec cp {} ./packages/ \; - - # Move wheels to correct package directories - for wheel in packages/*.whl; do - if [[ $wheel == *"leann_core"* ]]; then - mv "$wheel" packages/leann-core/dist/ - elif [[ $wheel == *"leann_backend_hnsw"* ]]; then - mv "$wheel" packages/leann-backend-hnsw/dist/ - elif [[ $wheel == *"leann_backend_diskann"* ]]; then - mv "$wheel" packages/leann-backend-diskann/dist/ - elif [[ $wheel == *"leann-"* ]] && [[ $wheel != *"backend"* ]] && [[ $wheel != *"core"* ]]; then - mv "$wheel" packages/leann/dist/ - fi - done - - # List downloaded wheels - echo "✅ Downloaded wheels:" - find packages/*/dist -name "*.whl" -type f | sort - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} - - name: Test on TestPyPI (optional) - if: inputs.test_pypi - continue-on-error: true - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }} + - name: Install uv + uses: astral-sh/setup-uv@v4 + + - name: Install system dependencies (Ubuntu) + if: runner.os == 'Linux' run: | - if [ -z "$TWINE_PASSWORD" ]; then - echo "⚠️ TEST_PYPI_API_TOKEN not configured, skipping TestPyPI upload" - echo " To enable TestPyPI testing, add TEST_PYPI_API_TOKEN to repository secrets" - exit 0 + sudo apt-get update + sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev + + - name: Install system dependencies (macOS) + if: runner.os == 'macOS' + run: | + brew install llvm libomp boost protobuf zeromq + + - name: Build packages + run: | + # Build core (platform independent) + if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then + cd packages/leann-core + uv build + cd ../.. fi - pip install twine - echo "📦 Uploading to TestPyPI..." - twine upload --repository testpypi packages/*/dist/* --verbose || { - echo "⚠️ TestPyPI upload failed, but continuing with release" - echo " This is optional and won't block the release" - exit 0 - } - echo "✅ Test upload successful!" - echo "📋 Check packages at: https://test.pypi.org/user/your-username/" - echo "" - echo "To test installation:" - echo "pip install -i https://test.pypi.org/simple/ leann" + # Build HNSW backend + cd packages/leann-backend-hnsw + uv pip install --system -r pyproject.toml --extra build + CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build + cd ../.. + + # Build DiskANN backend + cd packages/leann-backend-diskann + uv pip install --system -r pyproject.toml --extra build + CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build + cd ../.. + + # Build meta package (platform independent) + if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then + cd packages/leann + uv build + cd ../.. + fi + + echo "📦 Built packages:" + find packages/*/dist -name "*.whl" -o -name "*.tar.gz" | sort + env: + CC: ${{ runner.os == 'macOS' && '$(brew --prefix llvm)/bin/clang' || '' }} + CXX: ${{ runner.os == 'macOS' && '$(brew --prefix llvm)/bin/clang++' || '' }} + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: packages-${{ matrix.os }}-${{ matrix.python }} + path: packages/*/dist/ + + publish: + name: Publish and Release + needs: build-packages + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.update-version.outputs.commit-sha }} + + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: dist-artifacts + + - name: Collect packages + run: | + mkdir -p dist + find dist-artifacts -name "*.whl" -exec cp {} dist/ \; + find dist-artifacts -name "*.tar.gz" -exec cp {} dist/ \; + + echo "📦 Packages to publish:" + ls -la dist/ - name: Publish to PyPI env: @@ -244,46 +152,22 @@ jobs: run: | if [ -z "$TWINE_PASSWORD" ]; then echo "❌ PYPI_API_TOKEN not configured!" - echo " Please add PYPI_API_TOKEN to repository secrets" exit 1 fi pip install twine - echo "📦 Publishing to PyPI..." - - # Collect all wheels in one place - mkdir -p all_wheels - find packages/*/dist -name "*.whl" -exec cp {} all_wheels/ \; - find packages/*/dist -name "*.tar.gz" -exec cp {} all_wheels/ \; - - echo "📋 Packages to publish:" - ls -la all_wheels/ - - # Upload to PyPI - twine upload all_wheels/* --skip-existing --verbose + twine upload dist/* --skip-existing --verbose echo "✅ Published to PyPI!" - echo "🎉 Check packages at: https://pypi.org/project/leann/" - - name: Create and push tag + - name: Create release run: | git tag "v${{ inputs.version }}" - git push origin main git push origin "v${{ inputs.version }}" - echo "✅ Tag v${{ inputs.version }} created and pushed" - - - name: Create GitHub Release - uses: softprops/action-gh-release@v1 - with: - tag_name: v${{ inputs.version }} - name: Release v${{ inputs.version }} - body: | - ## 🚀 Release v${{ inputs.version }} - - ### What's Changed - See the [full changelog](https://github.com/${{ github.repository }}/compare/...v${{ inputs.version }}) - - ### Installation - ```bash - pip install leann==${{ inputs.version }} - ``` \ No newline at end of file + + gh release create "v${{ inputs.version }}" \ + --title "Release v${{ inputs.version }}" \ + --notes "🚀 Released to PyPI: https://pypi.org/project/leann/${{ inputs.version }}/" \ + --latest + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/docs/RELEASE.md b/docs/RELEASE.md index 69da211..40da945 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -1,111 +1,22 @@ # Release Guide -## Required: PyPI Configuration +## Setup (One-time) -Before releasing, ensure you have configured the PyPI API token: +Add `PYPI_API_TOKEN` to GitHub Secrets: +1. Get token: https://pypi.org/manage/account/token/ +2. Add to secrets: Settings → Secrets → Actions → `PYPI_API_TOKEN` -1. Generate API token at https://pypi.org/manage/account/token/ -2. Add as GitHub secret: `PYPI_API_TOKEN` -3. For full automation, also add a Personal Access Token: - - Create PAT at https://github.com/settings/tokens with `workflow` scope - - Add as GitHub secret: `WORKFLOW_PAT` - - This allows the release workflow to trigger CI builds automatically +## Release (One-click) -## 📋 Prerequisites +1. Go to: https://github.com/yichuan-w/LEANN/actions/workflows/release-manual.yml +2. Click "Run workflow" +3. Enter version: `0.1.2` +4. Click green "Run workflow" button -Before releasing, ensure: -1. ✅ All code changes are committed and pushed -2. ✅ CI has passed on the latest commit (check [Actions](https://github.com/yichuan-w/LEANN/actions/workflows/ci.yml)) -3. ✅ You have determined the new version number +That's it! The workflow will automatically: +- ✅ Update version in all packages +- ✅ Build all packages +- ✅ Publish to PyPI +- ✅ Create GitHub tag and release -### Required: PyPI Configuration - -To enable PyPI publishing: -1. Get a PyPI API token from https://pypi.org/manage/account/token/ -2. Add it to repository secrets: Settings → Secrets → Actions → New repository secret - - Name: `PYPI_API_TOKEN` - - Value: Your PyPI token (starts with `pypi-`) - -### Optional: TestPyPI Configuration - -To enable TestPyPI testing (recommended but not required): -1. Get a TestPyPI API token from https://test.pypi.org/manage/account/token/ -2. Add it to repository secrets: Settings → Secrets → Actions → New repository secret - - Name: `TEST_PYPI_API_TOKEN` - - Value: Your TestPyPI token (starts with `pypi-`) - -**Note**: TestPyPI testing is optional. If not configured, the release will skip TestPyPI and proceed. - -## 🚀 Recommended: Manual Release Workflow - -### Via GitHub UI (Most Reliable) - -1. **Verify CI Status**: Check that the latest commit has a green checkmark ✅ -2. Go to [Actions → Manual Release](https://github.com/yichuan-w/LEANN/actions/workflows/release-manual.yml) -3. Click "Run workflow" -4. Enter version (e.g., `0.1.1`) -5. Toggle "Test on TestPyPI first" if desired -6. Click "Run workflow" - -**What happens:** -- ✅ Downloads pre-built packages from CI (no rebuild needed!) -- ✅ Updates all package versions -- ✅ Optionally tests on TestPyPI -- ✅ **Publishes directly to PyPI** -- ✅ Creates tag and GitHub release - -### Via Command Line - -```bash -gh workflow run release-manual.yml -f version=0.1.1 -f test_pypi=true -``` - -## ⚡ Quick Release (One-Line) - -For experienced users who want the fastest path: - -```bash -./scripts/release.sh 0.1.1 -``` - -This script will: -1. Update all package versions -2. Commit and push changes -3. Create GitHub release -4. **Manual Release workflow will automatically publish to PyPI** - -⚠️ **Note**: If CI fails, you'll need to manually fix and re-tag - -## Manual Testing Before Release - -For testing specific packages locally (especially DiskANN on macOS): - -```bash -# Build specific package locally -./scripts/build_and_test.sh diskann # or hnsw, core, meta, all - -# Test installation in a clean environment -python -m venv test_env -source test_env/bin/activate -pip install packages/*/dist/*.whl - -# Upload to Test PyPI (optional) -./scripts/upload_to_pypi.sh test - -# Upload to Production PyPI (use with caution) -./scripts/upload_to_pypi.sh prod -``` - -## First-time setup - -1. Install GitHub CLI: - ```bash - brew install gh - gh auth login - ``` - -2. Set PyPI token in GitHub: - ```bash - gh secret set PYPI_API_TOKEN - # Paste your PyPI token when prompted - ``` \ No newline at end of file +Check progress: https://github.com/yichuan-w/LEANN/actions \ No newline at end of file