From 20f2aece08313ae1682534a313369f4adcdb7ed9 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Fri, 25 Jul 2025 02:05:11 +0000 Subject: [PATCH 01/16] chore: release v0.1.3 --- packages/leann-backend-diskann/pyproject.toml | 4 ++-- packages/leann-backend-hnsw/pyproject.toml | 4 ++-- packages/leann-core/pyproject.toml | 2 +- packages/leann/pyproject.toml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/leann-backend-diskann/pyproject.toml b/packages/leann-backend-diskann/pyproject.toml index c48b8ed..57282d7 100644 --- a/packages/leann-backend-diskann/pyproject.toml +++ b/packages/leann-backend-diskann/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "scikit_build_core.build" [project] name = "leann-backend-diskann" -version = "0.1.2" -dependencies = ["leann-core==0.1.2", "numpy"] +version = "0.1.3" +dependencies = ["leann-core==0.1.3", "numpy"] [tool.scikit-build] # Key: simplified CMake path diff --git a/packages/leann-backend-hnsw/pyproject.toml b/packages/leann-backend-hnsw/pyproject.toml index 14b0055..2157138 100644 --- a/packages/leann-backend-hnsw/pyproject.toml +++ b/packages/leann-backend-hnsw/pyproject.toml @@ -6,10 +6,10 @@ build-backend = "scikit_build_core.build" [project] name = "leann-backend-hnsw" -version = "0.1.2" +version = "0.1.3" description = "Custom-built HNSW (Faiss) backend for the Leann toolkit." dependencies = [ - "leann-core==0.1.2", + "leann-core==0.1.3", "numpy", "pyzmq>=23.0.0", "msgpack>=1.0.0", diff --git a/packages/leann-core/pyproject.toml b/packages/leann-core/pyproject.toml index b0c06f5..35afa56 100644 --- a/packages/leann-core/pyproject.toml +++ b/packages/leann-core/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "leann-core" -version = "0.1.2" +version = "0.1.3" description = "Core API and plugin system for LEANN" readme = "README.md" requires-python = ">=3.9" diff --git a/packages/leann/pyproject.toml b/packages/leann/pyproject.toml index 31844c4..bc7fc7a 100644 --- a/packages/leann/pyproject.toml +++ b/packages/leann/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "leann" -version = "0.1.2" +version = "0.1.3" description = "LEANN - The smallest vector index in the world. RAG Everything with LEANN!" readme = "README.md" requires-python = ">=3.9" From 8307555d547551bd468aa3ba51e6fa4155410d1d Mon Sep 17 00:00:00 2001 From: Andy Lee Date: Thu, 24 Jul 2025 19:21:32 -0700 Subject: [PATCH 02/16] fix: manually trigger CI after version push in release workflow --- .github/workflows/release-manual.yml | 29 ++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-manual.yml b/.github/workflows/release-manual.yml index 6310a2f..98b1ab8 100644 --- a/.github/workflows/release-manual.yml +++ b/.github/workflows/release-manual.yml @@ -72,6 +72,18 @@ jobs: COMMIT_SHA=$(git rev-parse HEAD) echo "commit-sha=$COMMIT_SHA" >> $GITHUB_OUTPUT id: push-version + + - name: Trigger CI build + run: | + echo "🚀 Manually triggering CI for the new version..." + gh workflow run "CI - Build Multi-Platform Packages" \ + --ref main \ + -f publish=false + + # Give GitHub a moment to register the new workflow run + sleep 5 + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Wait for CI to complete id: wait-for-ci @@ -81,11 +93,24 @@ jobs: # Wait up to 20 minutes for CI to complete for i in {1..40}; do + # First check if CI is running + RUNNING_ID=$(gh run list \ + --workflow="CI - Build Multi-Platform Packages" \ + --commit=$COMMIT_SHA \ + --status=in_progress \ + --json databaseId \ + --jq '.[0].databaseId') + + if [ ! -z "$RUNNING_ID" ]; then + echo "âŗ CI is running (ID: $RUNNING_ID)..." + fi + + # Check if CI has completed RUN_ID=$(gh run list \ --workflow="CI - Build Multi-Platform Packages" \ --commit=$COMMIT_SHA \ - --json databaseId,status \ - --jq '.[] | select(.status == "completed") | .databaseId' | head -1) + --json databaseId,status,conclusion \ + --jq '.[] | select(.status == "completed" and .conclusion == "success") | .databaseId' | head -1) if [ ! -z "$RUN_ID" ]; then echo "✅ Found completed CI run: $RUN_ID" From 9000a7083d295d43341d20e422b3abada0cde8bd Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Fri, 25 Jul 2025 02:23:36 +0000 Subject: [PATCH 03/16] chore: release v0.1.4 --- packages/leann-backend-diskann/pyproject.toml | 4 ++-- packages/leann-backend-hnsw/pyproject.toml | 4 ++-- packages/leann-core/pyproject.toml | 2 +- packages/leann/pyproject.toml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/leann-backend-diskann/pyproject.toml b/packages/leann-backend-diskann/pyproject.toml index 57282d7..b4d299e 100644 --- a/packages/leann-backend-diskann/pyproject.toml +++ b/packages/leann-backend-diskann/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "scikit_build_core.build" [project] name = "leann-backend-diskann" -version = "0.1.3" -dependencies = ["leann-core==0.1.3", "numpy"] +version = "0.1.4" +dependencies = ["leann-core==0.1.4", "numpy"] [tool.scikit-build] # Key: simplified CMake path diff --git a/packages/leann-backend-hnsw/pyproject.toml b/packages/leann-backend-hnsw/pyproject.toml index 2157138..ed52429 100644 --- a/packages/leann-backend-hnsw/pyproject.toml +++ b/packages/leann-backend-hnsw/pyproject.toml @@ -6,10 +6,10 @@ build-backend = "scikit_build_core.build" [project] name = "leann-backend-hnsw" -version = "0.1.3" +version = "0.1.4" description = "Custom-built HNSW (Faiss) backend for the Leann toolkit." dependencies = [ - "leann-core==0.1.3", + "leann-core==0.1.4", "numpy", "pyzmq>=23.0.0", "msgpack>=1.0.0", diff --git a/packages/leann-core/pyproject.toml b/packages/leann-core/pyproject.toml index 35afa56..ee0db23 100644 --- a/packages/leann-core/pyproject.toml +++ b/packages/leann-core/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "leann-core" -version = "0.1.3" +version = "0.1.4" description = "Core API and plugin system for LEANN" readme = "README.md" requires-python = ">=3.9" diff --git a/packages/leann/pyproject.toml b/packages/leann/pyproject.toml index bc7fc7a..2671852 100644 --- a/packages/leann/pyproject.toml +++ b/packages/leann/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "leann" -version = "0.1.3" +version = "0.1.4" description = "LEANN - The smallest vector index in the world. RAG Everything with LEANN!" readme = "README.md" requires-python = ">=3.9" From d45c0138061dfc48c7bed8364f69756b5e01fc5b Mon Sep 17 00:00:00 2001 From: Andy Lee Date: Thu, 24 Jul 2025 19:25:11 -0700 Subject: [PATCH 04/16] fix: handle workflow trigger permission gracefully --- .github/workflows/release-manual.yml | 51 +++++++++++++++++++++++++++- docs/RELEASE.md | 11 ++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-manual.yml b/.github/workflows/release-manual.yml index 98b1ab8..d2da5ae 100644 --- a/.github/workflows/release-manual.yml +++ b/.github/workflows/release-manual.yml @@ -76,6 +76,16 @@ jobs: - name: Trigger CI build run: | echo "🚀 Manually triggering CI for the new version..." + + # Check if we have a PAT for triggering workflows + if [ -z "${{ secrets.WORKFLOW_PAT }}" ]; then + echo "âš ī¸ No WORKFLOW_PAT found. CI will be triggered by the push event." + echo " Note: If CI doesn't trigger automatically, you'll need to:" + echo " 1. Add a Personal Access Token with 'workflow' scope as WORKFLOW_PAT secret" + echo " 2. Or manually run the CI workflow after this release completes" + exit 0 + fi + gh workflow run "CI - Build Multi-Platform Packages" \ --ref main \ -f publish=false @@ -83,7 +93,7 @@ jobs: # Give GitHub a moment to register the new workflow run sleep 5 env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.WORKFLOW_PAT || secrets.GITHUB_TOKEN }} - name: Wait for CI to complete id: wait-for-ci @@ -91,6 +101,45 @@ jobs: echo "âŗ Waiting for CI to build new version..." COMMIT_SHA="${{ steps.push-version.outputs.commit-sha }}" + # First, wait a bit for CI to potentially start + echo "âŗ Waiting for CI to start..." + sleep 30 + + # Check if there's any CI run for this commit + CI_EXISTS=$(gh run list \ + --workflow="CI - Build Multi-Platform Packages" \ + --commit=$COMMIT_SHA \ + --json databaseId \ + --jq 'length') + + if [ "$CI_EXISTS" -eq "0" ]; then + echo "âš ī¸ No CI run found for commit $COMMIT_SHA" + echo " This might be because:" + echo " 1. WORKFLOW_PAT is not configured" + echo " 2. CI hasn't started yet" + echo "" + echo " You can manually trigger CI after this release completes:" + echo " gh workflow run 'CI - Build Multi-Platform Packages' --ref main" + echo "" + echo " For now, we'll use the artifacts from the latest successful CI run." + + # Get the latest successful CI run + LATEST_RUN=$(gh run list \ + --workflow="CI - Build Multi-Platform Packages" \ + --status=success \ + --json databaseId \ + --jq '.[0].databaseId') + + if [ -z "$LATEST_RUN" ]; then + echo "❌ No successful CI runs found!" + exit 1 + fi + + echo "đŸ“Ļ Using artifacts from CI run: $LATEST_RUN" + echo "run-id=$LATEST_RUN" >> $GITHUB_OUTPUT + exit 0 + fi + # Wait up to 20 minutes for CI to complete for i in {1..40}; do # First check if CI is running diff --git a/docs/RELEASE.md b/docs/RELEASE.md index 985a282..69da211 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -1,5 +1,16 @@ # Release Guide +## Required: PyPI Configuration + +Before releasing, ensure you have configured the PyPI API token: + +1. Generate API token at https://pypi.org/manage/account/token/ +2. Add as GitHub secret: `PYPI_API_TOKEN` +3. For full automation, also add a Personal Access Token: + - Create PAT at https://github.com/settings/tokens with `workflow` scope + - Add as GitHub secret: `WORKFLOW_PAT` + - This allows the release workflow to trigger CI builds automatically + ## 📋 Prerequisites Before releasing, ensure: From 32a374d09429fa6f0d0bfe820163ad493b78f446 Mon Sep 17 00:00:00 2001 From: Andy Lee Date: Thu, 24 Jul 2025 19:30:44 -0700 Subject: [PATCH 05/16] feat: true one-click automated release with multi-platform support --- .github/workflows/release-manual.yml | 360 +++++++++------------------ docs/RELEASE.md | 119 ++------- 2 files changed, 137 insertions(+), 342 deletions(-) diff --git a/.github/workflows/release-manual.yml b/.github/workflows/release-manual.yml index d2da5ae..e0f9d47 100644 --- a/.github/workflows/release-manual.yml +++ b/.github/workflows/release-manual.yml @@ -1,241 +1,149 @@ -name: Manual Release +name: Release on: workflow_dispatch: inputs: version: - description: 'Version to release (e.g., 0.1.1)' + description: 'Version to release (e.g., 0.1.2)' required: true type: string - test_pypi: - description: 'Test on TestPyPI first' - required: false - type: boolean - default: true jobs: - validate-and-release: + update-version: + name: Update Version runs-on: ubuntu-latest - permissions: - contents: write - actions: read - + outputs: + commit-sha: ${{ steps.push.outputs.commit-sha }} + steps: - uses: actions/checkout@v4 with: token: ${{ secrets.GITHUB_TOKEN }} - - name: Check CI status - run: | - echo "â„šī¸ This workflow will download build artifacts from the latest CI run." - echo " CI must have completed successfully on the current commit." - echo "" - - - name: Validate version format + - name: Validate version run: | if ! [[ "${{ inputs.version }}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "❌ Invalid version format. Use semantic versioning (e.g., 0.1.1)" + echo "❌ Invalid version format" exit 1 fi - echo "✅ Version format valid: ${{ inputs.version }}" + echo "✅ Version format valid" - - name: Check if version already exists - run: | - if git tag | grep -q "^v${{ inputs.version }}$"; then - echo "❌ Version v${{ inputs.version }} already exists!" - exit 1 - fi - echo "✅ Version is new" - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.13' - - - name: Install uv - run: | - curl -LsSf https://astral.sh/uv/install.sh | sh - echo "$HOME/.cargo/bin" >> $GITHUB_PATH - - - name: Update versions + - name: Update versions and push + id: push run: | ./scripts/bump_version.sh ${{ inputs.version }} git config user.name "GitHub Actions" git config user.email "actions@github.com" git add packages/*/pyproject.toml git commit -m "chore: release v${{ inputs.version }}" - - - name: Push version update - run: | - git push origin HEAD:main - echo "✅ Pushed version update to main branch" + git push origin main + COMMIT_SHA=$(git rev-parse HEAD) echo "commit-sha=$COMMIT_SHA" >> $GITHUB_OUTPUT - id: push-version - - - name: Trigger CI build - run: | - echo "🚀 Manually triggering CI for the new version..." - - # Check if we have a PAT for triggering workflows - if [ -z "${{ secrets.WORKFLOW_PAT }}" ]; then - echo "âš ī¸ No WORKFLOW_PAT found. CI will be triggered by the push event." - echo " Note: If CI doesn't trigger automatically, you'll need to:" - echo " 1. Add a Personal Access Token with 'workflow' scope as WORKFLOW_PAT secret" - echo " 2. Or manually run the CI workflow after this release completes" - exit 0 - fi - - gh workflow run "CI - Build Multi-Platform Packages" \ - --ref main \ - -f publish=false - - # Give GitHub a moment to register the new workflow run - sleep 5 - env: - GH_TOKEN: ${{ secrets.WORKFLOW_PAT || secrets.GITHUB_TOKEN }} - - - name: Wait for CI to complete - id: wait-for-ci - run: | - echo "âŗ Waiting for CI to build new version..." - COMMIT_SHA="${{ steps.push-version.outputs.commit-sha }}" - - # First, wait a bit for CI to potentially start - echo "âŗ Waiting for CI to start..." - sleep 30 - - # Check if there's any CI run for this commit - CI_EXISTS=$(gh run list \ - --workflow="CI - Build Multi-Platform Packages" \ - --commit=$COMMIT_SHA \ - --json databaseId \ - --jq 'length') - - if [ "$CI_EXISTS" -eq "0" ]; then - echo "âš ī¸ No CI run found for commit $COMMIT_SHA" - echo " This might be because:" - echo " 1. WORKFLOW_PAT is not configured" - echo " 2. CI hasn't started yet" - echo "" - echo " You can manually trigger CI after this release completes:" - echo " gh workflow run 'CI - Build Multi-Platform Packages' --ref main" - echo "" - echo " For now, we'll use the artifacts from the latest successful CI run." - - # Get the latest successful CI run - LATEST_RUN=$(gh run list \ - --workflow="CI - Build Multi-Platform Packages" \ - --status=success \ - --json databaseId \ - --jq '.[0].databaseId') - - if [ -z "$LATEST_RUN" ]; then - echo "❌ No successful CI runs found!" - exit 1 - fi - - echo "đŸ“Ļ Using artifacts from CI run: $LATEST_RUN" - echo "run-id=$LATEST_RUN" >> $GITHUB_OUTPUT - exit 0 - fi - - # Wait up to 20 minutes for CI to complete - for i in {1..40}; do - # First check if CI is running - RUNNING_ID=$(gh run list \ - --workflow="CI - Build Multi-Platform Packages" \ - --commit=$COMMIT_SHA \ - --status=in_progress \ - --json databaseId \ - --jq '.[0].databaseId') - - if [ ! -z "$RUNNING_ID" ]; then - echo "âŗ CI is running (ID: $RUNNING_ID)..." - fi - - # Check if CI has completed - RUN_ID=$(gh run list \ - --workflow="CI - Build Multi-Platform Packages" \ - --commit=$COMMIT_SHA \ - --json databaseId,status,conclusion \ - --jq '.[] | select(.status == "completed" and .conclusion == "success") | .databaseId' | head -1) - - if [ ! -z "$RUN_ID" ]; then - echo "✅ Found completed CI run: $RUN_ID" - echo "run-id=$RUN_ID" >> $GITHUB_OUTPUT - exit 0 - fi - - echo "âŗ Waiting for CI... (attempt $i/40)" - sleep 30 - done - - echo "❌ CI did not complete within 20 minutes" - exit 1 - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + echo "✅ Pushed version update: $COMMIT_SHA" + + build-packages: + name: Build packages + needs: update-version + strategy: + matrix: + include: + - os: ubuntu-latest + python: '3.11' + - os: macos-latest + python: '3.11' + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.update-version.outputs.commit-sha }} + submodules: recursive - - name: Download artifacts from CI run - run: | - echo "đŸ“Ļ Downloading artifacts from CI run ${{ steps.wait-for-ci.outputs.run-id }}..." - - # Download all artifacts (not just wheels-*) - gh run download ${{ steps.wait-for-ci.outputs.run-id }} \ - --dir ./dist-downloads - - # Consolidate all wheels into packages/*/dist/ - mkdir -p packages/leann-core/dist - mkdir -p packages/leann-backend-hnsw/dist - mkdir -p packages/leann-backend-diskann/dist - mkdir -p packages/leann/dist - - find ./dist-downloads -name "*.whl" -exec cp {} ./packages/ \; - - # Move wheels to correct package directories - for wheel in packages/*.whl; do - if [[ $wheel == *"leann_core"* ]]; then - mv "$wheel" packages/leann-core/dist/ - elif [[ $wheel == *"leann_backend_hnsw"* ]]; then - mv "$wheel" packages/leann-backend-hnsw/dist/ - elif [[ $wheel == *"leann_backend_diskann"* ]]; then - mv "$wheel" packages/leann-backend-diskann/dist/ - elif [[ $wheel == *"leann-"* ]] && [[ $wheel != *"backend"* ]] && [[ $wheel != *"core"* ]]; then - mv "$wheel" packages/leann/dist/ - fi - done - - # List downloaded wheels - echo "✅ Downloaded wheels:" - find packages/*/dist -name "*.whl" -type f | sort - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} - - name: Test on TestPyPI (optional) - if: inputs.test_pypi - continue-on-error: true - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }} + - name: Install uv + uses: astral-sh/setup-uv@v4 + + - name: Install system dependencies (Ubuntu) + if: runner.os == 'Linux' run: | - if [ -z "$TWINE_PASSWORD" ]; then - echo "âš ī¸ TEST_PYPI_API_TOKEN not configured, skipping TestPyPI upload" - echo " To enable TestPyPI testing, add TEST_PYPI_API_TOKEN to repository secrets" - exit 0 + sudo apt-get update + sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev + + - name: Install system dependencies (macOS) + if: runner.os == 'macOS' + run: | + brew install llvm libomp boost protobuf zeromq + + - name: Build packages + run: | + # Build core (platform independent) + if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then + cd packages/leann-core + uv build + cd ../.. fi - pip install twine - echo "đŸ“Ļ Uploading to TestPyPI..." - twine upload --repository testpypi packages/*/dist/* --verbose || { - echo "âš ī¸ TestPyPI upload failed, but continuing with release" - echo " This is optional and won't block the release" - exit 0 - } - echo "✅ Test upload successful!" - echo "📋 Check packages at: https://test.pypi.org/user/your-username/" - echo "" - echo "To test installation:" - echo "pip install -i https://test.pypi.org/simple/ leann" + # Build HNSW backend + cd packages/leann-backend-hnsw + uv pip install --system -r pyproject.toml --extra build + CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build + cd ../.. + + # Build DiskANN backend + cd packages/leann-backend-diskann + uv pip install --system -r pyproject.toml --extra build + CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build + cd ../.. + + # Build meta package (platform independent) + if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then + cd packages/leann + uv build + cd ../.. + fi + + echo "đŸ“Ļ Built packages:" + find packages/*/dist -name "*.whl" -o -name "*.tar.gz" | sort + env: + CC: ${{ runner.os == 'macOS' && '$(brew --prefix llvm)/bin/clang' || '' }} + CXX: ${{ runner.os == 'macOS' && '$(brew --prefix llvm)/bin/clang++' || '' }} + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: packages-${{ matrix.os }}-${{ matrix.python }} + path: packages/*/dist/ + + publish: + name: Publish and Release + needs: build-packages + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.update-version.outputs.commit-sha }} + + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: dist-artifacts + + - name: Collect packages + run: | + mkdir -p dist + find dist-artifacts -name "*.whl" -exec cp {} dist/ \; + find dist-artifacts -name "*.tar.gz" -exec cp {} dist/ \; + + echo "đŸ“Ļ Packages to publish:" + ls -la dist/ - name: Publish to PyPI env: @@ -244,46 +152,22 @@ jobs: run: | if [ -z "$TWINE_PASSWORD" ]; then echo "❌ PYPI_API_TOKEN not configured!" - echo " Please add PYPI_API_TOKEN to repository secrets" exit 1 fi pip install twine - echo "đŸ“Ļ Publishing to PyPI..." - - # Collect all wheels in one place - mkdir -p all_wheels - find packages/*/dist -name "*.whl" -exec cp {} all_wheels/ \; - find packages/*/dist -name "*.tar.gz" -exec cp {} all_wheels/ \; - - echo "📋 Packages to publish:" - ls -la all_wheels/ - - # Upload to PyPI - twine upload all_wheels/* --skip-existing --verbose + twine upload dist/* --skip-existing --verbose echo "✅ Published to PyPI!" - echo "🎉 Check packages at: https://pypi.org/project/leann/" - - name: Create and push tag + - name: Create release run: | git tag "v${{ inputs.version }}" - git push origin main git push origin "v${{ inputs.version }}" - echo "✅ Tag v${{ inputs.version }} created and pushed" - - - name: Create GitHub Release - uses: softprops/action-gh-release@v1 - with: - tag_name: v${{ inputs.version }} - name: Release v${{ inputs.version }} - body: | - ## 🚀 Release v${{ inputs.version }} - - ### What's Changed - See the [full changelog](https://github.com/${{ github.repository }}/compare/...v${{ inputs.version }}) - - ### Installation - ```bash - pip install leann==${{ inputs.version }} - ``` \ No newline at end of file + + gh release create "v${{ inputs.version }}" \ + --title "Release v${{ inputs.version }}" \ + --notes "🚀 Released to PyPI: https://pypi.org/project/leann/${{ inputs.version }}/" \ + --latest + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/docs/RELEASE.md b/docs/RELEASE.md index 69da211..40da945 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -1,111 +1,22 @@ # Release Guide -## Required: PyPI Configuration +## Setup (One-time) -Before releasing, ensure you have configured the PyPI API token: +Add `PYPI_API_TOKEN` to GitHub Secrets: +1. Get token: https://pypi.org/manage/account/token/ +2. Add to secrets: Settings → Secrets → Actions → `PYPI_API_TOKEN` -1. Generate API token at https://pypi.org/manage/account/token/ -2. Add as GitHub secret: `PYPI_API_TOKEN` -3. For full automation, also add a Personal Access Token: - - Create PAT at https://github.com/settings/tokens with `workflow` scope - - Add as GitHub secret: `WORKFLOW_PAT` - - This allows the release workflow to trigger CI builds automatically +## Release (One-click) -## 📋 Prerequisites +1. Go to: https://github.com/yichuan-w/LEANN/actions/workflows/release-manual.yml +2. Click "Run workflow" +3. Enter version: `0.1.2` +4. Click green "Run workflow" button -Before releasing, ensure: -1. ✅ All code changes are committed and pushed -2. ✅ CI has passed on the latest commit (check [Actions](https://github.com/yichuan-w/LEANN/actions/workflows/ci.yml)) -3. ✅ You have determined the new version number +That's it! The workflow will automatically: +- ✅ Update version in all packages +- ✅ Build all packages +- ✅ Publish to PyPI +- ✅ Create GitHub tag and release -### Required: PyPI Configuration - -To enable PyPI publishing: -1. Get a PyPI API token from https://pypi.org/manage/account/token/ -2. Add it to repository secrets: Settings → Secrets → Actions → New repository secret - - Name: `PYPI_API_TOKEN` - - Value: Your PyPI token (starts with `pypi-`) - -### Optional: TestPyPI Configuration - -To enable TestPyPI testing (recommended but not required): -1. Get a TestPyPI API token from https://test.pypi.org/manage/account/token/ -2. Add it to repository secrets: Settings → Secrets → Actions → New repository secret - - Name: `TEST_PYPI_API_TOKEN` - - Value: Your TestPyPI token (starts with `pypi-`) - -**Note**: TestPyPI testing is optional. If not configured, the release will skip TestPyPI and proceed. - -## 🚀 Recommended: Manual Release Workflow - -### Via GitHub UI (Most Reliable) - -1. **Verify CI Status**: Check that the latest commit has a green checkmark ✅ -2. Go to [Actions → Manual Release](https://github.com/yichuan-w/LEANN/actions/workflows/release-manual.yml) -3. Click "Run workflow" -4. Enter version (e.g., `0.1.1`) -5. Toggle "Test on TestPyPI first" if desired -6. Click "Run workflow" - -**What happens:** -- ✅ Downloads pre-built packages from CI (no rebuild needed!) -- ✅ Updates all package versions -- ✅ Optionally tests on TestPyPI -- ✅ **Publishes directly to PyPI** -- ✅ Creates tag and GitHub release - -### Via Command Line - -```bash -gh workflow run release-manual.yml -f version=0.1.1 -f test_pypi=true -``` - -## ⚡ Quick Release (One-Line) - -For experienced users who want the fastest path: - -```bash -./scripts/release.sh 0.1.1 -``` - -This script will: -1. Update all package versions -2. Commit and push changes -3. Create GitHub release -4. **Manual Release workflow will automatically publish to PyPI** - -âš ī¸ **Note**: If CI fails, you'll need to manually fix and re-tag - -## Manual Testing Before Release - -For testing specific packages locally (especially DiskANN on macOS): - -```bash -# Build specific package locally -./scripts/build_and_test.sh diskann # or hnsw, core, meta, all - -# Test installation in a clean environment -python -m venv test_env -source test_env/bin/activate -pip install packages/*/dist/*.whl - -# Upload to Test PyPI (optional) -./scripts/upload_to_pypi.sh test - -# Upload to Production PyPI (use with caution) -./scripts/upload_to_pypi.sh prod -``` - -## First-time setup - -1. Install GitHub CLI: - ```bash - brew install gh - gh auth login - ``` - -2. Set PyPI token in GitHub: - ```bash - gh secret set PYPI_API_TOKEN - # Paste your PyPI token when prompted - ``` \ No newline at end of file +Check progress: https://github.com/yichuan-w/LEANN/actions \ No newline at end of file From e8d2ecab031e78687acdfd2f57c41dd03ea4fe96 Mon Sep 17 00:00:00 2001 From: Andy Lee Date: Thu, 24 Jul 2025 19:35:12 -0700 Subject: [PATCH 06/16] refactor: use reusable workflow to avoid code duplication --- .github/workflows/build-and-publish.yml | 257 +----------------------- .github/workflows/build-reusable.yml | 92 +++++++++ .github/workflows/release-manual.yml | 76 +------ 3 files changed, 98 insertions(+), 327 deletions(-) create mode 100644 .github/workflows/build-reusable.yml diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index c19ed2f..d8689ff 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -1,262 +1,11 @@ -name: CI - Build Multi-Platform Packages +name: CI on: push: branches: [ main ] pull_request: branches: [ main ] - workflow_dispatch: - inputs: - publish: - description: 'Publish to PyPI (only use for emergency fixes)' - required: true - default: 'false' - type: choice - options: - - 'false' - - 'test' - - 'prod' jobs: - # Build pure Python package: leann-core - build-core: - name: Build leann-core - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Install uv - uses: astral-sh/setup-uv@v4 - - - name: Install build dependencies - run: | - uv pip install --system build twine - - - name: Build package - run: | - cd packages/leann-core - uv build - - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: leann-core-dist - path: packages/leann-core/dist/ - - # Build binary package: leann-backend-hnsw (default backend) - build-hnsw: - name: Build leann-backend-hnsw - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - python-version: ['3.9', '3.10', '3.11', '3.12'] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install uv - uses: astral-sh/setup-uv@v4 - - - name: Install system dependencies (Ubuntu) - if: runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get install -y libomp-dev libboost-all-dev libzmq3-dev \ - pkg-config libopenblas-dev patchelf - - - name: Install system dependencies (macOS) - if: runner.os == 'macOS' - run: | - brew install libomp boost zeromq - - - name: Install build dependencies - run: | - uv pip install --system scikit-build-core numpy swig - uv pip install --system auditwheel delocate - - - name: Build wheel - run: | - cd packages/leann-backend-hnsw - uv build --wheel --python python - - - name: Repair wheel (Linux) - if: runner.os == 'Linux' - run: | - cd packages/leann-backend-hnsw - auditwheel repair dist/*.whl -w dist_repaired - rm -rf dist - mv dist_repaired dist - - - name: Repair wheel (macOS) - if: runner.os == 'macOS' - run: | - cd packages/leann-backend-hnsw - delocate-wheel -w dist_repaired -v dist/*.whl - rm -rf dist - mv dist_repaired dist - - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: hnsw-${{ matrix.os }}-py${{ matrix.python-version }} - path: packages/leann-backend-hnsw/dist/ - - # Build binary package: leann-backend-diskann (multi-platform) - build-diskann: - name: Build leann-backend-diskann - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - python-version: ['3.9', '3.10', '3.11', '3.12'] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install uv - uses: astral-sh/setup-uv@v4 - - - name: Install system dependencies (Ubuntu) - if: runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get install -y libomp-dev libboost-all-dev libaio-dev libzmq3-dev \ - protobuf-compiler libprotobuf-dev libabsl-dev patchelf - - # Install Intel MKL using Intel's installer - wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/79153e0f-74d7-45af-b8c2-258941adf58a/intel-onemkl-2025.0.0.940.sh - sudo sh intel-onemkl-2025.0.0.940.sh -a --components intel.oneapi.lin.mkl.devel --action install --eula accept -s - source /opt/intel/oneapi/setvars.sh - echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH" >> $GITHUB_ENV - - - name: Install system dependencies (macOS) - if: runner.os == 'macOS' - run: | - brew install libomp boost zeromq protobuf - # MKL is not available on Homebrew, but DiskANN can work without it - - - name: Install build dependencies - run: | - uv pip install --system scikit-build-core numpy Cython pybind11 - if [[ "$RUNNER_OS" == "Linux" ]]; then - uv pip install --system auditwheel - else - uv pip install --system delocate - fi - - - name: Build wheel - run: | - cd packages/leann-backend-diskann - uv build --wheel --python python - - - name: Repair wheel (Linux) - if: runner.os == 'Linux' - run: | - cd packages/leann-backend-diskann - auditwheel repair dist/*.whl -w dist_repaired - rm -rf dist - mv dist_repaired dist - - - name: Repair wheel (macOS) - if: runner.os == 'macOS' - run: | - cd packages/leann-backend-diskann - delocate-wheel -w dist_repaired -v dist/*.whl - rm -rf dist - mv dist_repaired dist - - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: diskann-${{ matrix.os }}-py${{ matrix.python-version }} - path: packages/leann-backend-diskann/dist/ - - # Build meta-package: leann (build last) - build-meta: - name: Build leann meta-package - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Install uv - uses: astral-sh/setup-uv@v4 - - - name: Install build dependencies - run: | - uv pip install --system build - - - name: Build package - run: | - cd packages/leann - uv build - - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: leann-meta-dist - path: packages/leann/dist/ - - # Publish to PyPI (only for emergency fixes or manual triggers) - publish: - name: Publish to PyPI (Emergency) - needs: [build-core, build-hnsw, build-diskann, build-meta] - runs-on: ubuntu-latest - if: github.event_name == 'workflow_dispatch' && github.event.inputs.publish != 'false' - - steps: - - name: Download all artifacts - uses: actions/download-artifact@v4 - with: - path: dist - - - name: Flatten directory structure - run: | - mkdir -p all_wheels - find dist -name "*.whl" -exec cp {} all_wheels/ \; - find dist -name "*.tar.gz" -exec cp {} all_wheels/ \; - - - name: Show what will be published - run: | - echo "đŸ“Ļ Packages to be published:" - ls -la all_wheels/ - - - name: Publish to Test PyPI - if: github.event.inputs.publish == 'test' - uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.TEST_PYPI_API_TOKEN }} - repository-url: https://test.pypi.org/legacy/ - packages-dir: all_wheels/ - skip-existing: true - - - name: Publish to PyPI - if: github.event.inputs.publish == 'prod' - uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.PYPI_API_TOKEN }} - packages-dir: all_wheels/ - skip-existing: true \ No newline at end of file + build: + uses: ./.github/workflows/build-reusable.yml \ No newline at end of file diff --git a/.github/workflows/build-reusable.yml b/.github/workflows/build-reusable.yml new file mode 100644 index 0000000..76fbf2d --- /dev/null +++ b/.github/workflows/build-reusable.yml @@ -0,0 +1,92 @@ +name: Reusable Build + +on: + workflow_call: + inputs: + ref: + description: 'Git ref to build' + required: false + type: string + default: '' + +jobs: + build: + name: Build ${{ matrix.os }} + strategy: + matrix: + include: + - os: ubuntu-latest + python: '3.11' + - os: macos-latest + python: '3.11' + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + submodules: recursive + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + + - name: Install uv + uses: astral-sh/setup-uv@v4 + + - name: Install system dependencies (Ubuntu) + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev + + - name: Install system dependencies (macOS) + if: runner.os == 'macOS' + run: | + brew install llvm libomp boost protobuf zeromq + + - name: Build packages + run: | + # Build core (platform independent) + if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then + cd packages/leann-core + uv build + cd ../.. + fi + + # Build HNSW backend + cd packages/leann-backend-hnsw + uv pip install --system -r pyproject.toml --extra build + if [ "${{ matrix.os }}" == "macos-latest" ]; then + CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build + else + uv build + fi + cd ../.. + + # Build DiskANN backend + cd packages/leann-backend-diskann + uv pip install --system -r pyproject.toml --extra build + if [ "${{ matrix.os }}" == "macos-latest" ]; then + CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build + else + uv build + fi + cd ../.. + + # Build meta package (platform independent) + if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then + cd packages/leann + uv build + cd ../.. + fi + + echo "đŸ“Ļ Built packages:" + find packages/*/dist -name "*.whl" -o -name "*.tar.gz" | sort + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: packages-${{ matrix.os }}-${{ matrix.python }} + path: packages/*/dist/ \ No newline at end of file diff --git a/.github/workflows/release-manual.yml b/.github/workflows/release-manual.yml index e0f9d47..27d0f25 100644 --- a/.github/workflows/release-manual.yml +++ b/.github/workflows/release-manual.yml @@ -45,79 +45,9 @@ jobs: build-packages: name: Build packages needs: update-version - strategy: - matrix: - include: - - os: ubuntu-latest - python: '3.11' - - os: macos-latest - python: '3.11' - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ needs.update-version.outputs.commit-sha }} - submodules: recursive - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python }} - - - name: Install uv - uses: astral-sh/setup-uv@v4 - - - name: Install system dependencies (Ubuntu) - if: runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev - - - name: Install system dependencies (macOS) - if: runner.os == 'macOS' - run: | - brew install llvm libomp boost protobuf zeromq - - - name: Build packages - run: | - # Build core (platform independent) - if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then - cd packages/leann-core - uv build - cd ../.. - fi - - # Build HNSW backend - cd packages/leann-backend-hnsw - uv pip install --system -r pyproject.toml --extra build - CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build - cd ../.. - - # Build DiskANN backend - cd packages/leann-backend-diskann - uv pip install --system -r pyproject.toml --extra build - CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build - cd ../.. - - # Build meta package (platform independent) - if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then - cd packages/leann - uv build - cd ../.. - fi - - echo "đŸ“Ļ Built packages:" - find packages/*/dist -name "*.whl" -o -name "*.tar.gz" | sort - env: - CC: ${{ runner.os == 'macOS' && '$(brew --prefix llvm)/bin/clang' || '' }} - CXX: ${{ runner.os == 'macOS' && '$(brew --prefix llvm)/bin/clang++' || '' }} - - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: packages-${{ matrix.os }}-${{ matrix.python }} - path: packages/*/dist/ + uses: ./.github/workflows/build-reusable.yml + with: + ref: ${{ needs.update-version.outputs.commit-sha }} publish: name: Publish and Release From 7d73c2c8032501705ce7974b42ce2504430f4fbc Mon Sep 17 00:00:00 2001 From: Andy Lee Date: Thu, 24 Jul 2025 19:43:23 -0700 Subject: [PATCH 07/16] fix: remove invalid --extra build flag from build commands --- .github/workflows/build-reusable.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build-reusable.yml b/.github/workflows/build-reusable.yml index 76fbf2d..decf872 100644 --- a/.github/workflows/build-reusable.yml +++ b/.github/workflows/build-reusable.yml @@ -57,7 +57,6 @@ jobs: # Build HNSW backend cd packages/leann-backend-hnsw - uv pip install --system -r pyproject.toml --extra build if [ "${{ matrix.os }}" == "macos-latest" ]; then CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build else @@ -67,7 +66,6 @@ jobs: # Build DiskANN backend cd packages/leann-backend-diskann - uv pip install --system -r pyproject.toml --extra build if [ "${{ matrix.os }}" == "macos-latest" ]; then CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build else From f0a2ef96b417664c525e2ad9eb193ce95f442010 Mon Sep 17 00:00:00 2001 From: Andy Lee Date: Thu, 24 Jul 2025 19:49:38 -0700 Subject: [PATCH 08/16] fix: restore complete build configuration from working version --- .github/workflows/build-reusable.yml | 87 +++++++++++++++++++++++++--- 1 file changed, 80 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-reusable.yml b/.github/workflows/build-reusable.yml index decf872..c2e7958 100644 --- a/.github/workflows/build-reusable.yml +++ b/.github/workflows/build-reusable.yml @@ -11,14 +11,26 @@ on: jobs: build: - name: Build ${{ matrix.os }} + name: Build ${{ matrix.os }} Python ${{ matrix.python }} strategy: matrix: include: + - os: ubuntu-latest + python: '3.9' + - os: ubuntu-latest + python: '3.10' - os: ubuntu-latest python: '3.11' + - os: ubuntu-latest + python: '3.12' + - os: macos-latest + python: '3.9' + - os: macos-latest + python: '3.10' - os: macos-latest python: '3.11' + - os: macos-latest + python: '3.12' runs-on: ${{ matrix.os }} steps: @@ -39,13 +51,30 @@ jobs: if: runner.os == 'Linux' run: | sudo apt-get update - sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev + sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \ + pkg-config libopenblas-dev patchelf libabsl-dev libaio-dev libprotobuf-dev + + # Install Intel MKL for DiskANN + wget -q https://registrationcenter-download.intel.com/akdlm/IRC_NAS/79153e0f-74d7-45af-b8c2-258941adf58a/intel-onemkl-2025.0.0.940.sh + sudo sh intel-onemkl-2025.0.0.940.sh -a --components intel.oneapi.lin.mkl.devel --action install --eula accept -s + source /opt/intel/oneapi/setvars.sh + echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH" >> $GITHUB_ENV - name: Install system dependencies (macOS) if: runner.os == 'macOS' run: | brew install llvm libomp boost protobuf zeromq + - name: Install build dependencies + run: | + uv pip install --system scikit-build-core numpy swig Cython pybind11 + if [[ "$RUNNER_OS" == "Linux" ]]; then + uv pip install --system auditwheel + else + uv pip install --system delocate + fi + - name: Build packages run: | # Build core (platform independent) @@ -58,18 +87,18 @@ jobs: # Build HNSW backend cd packages/leann-backend-hnsw if [ "${{ matrix.os }}" == "macos-latest" ]; then - CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build + CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build --wheel --python python else - uv build + uv build --wheel --python python fi cd ../.. # Build DiskANN backend cd packages/leann-backend-diskann if [ "${{ matrix.os }}" == "macos-latest" ]; then - CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build + CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build --wheel --python python else - uv build + uv build --wheel --python python fi cd ../.. @@ -79,12 +108,56 @@ jobs: uv build cd ../.. fi + + - name: Repair wheels (Linux) + if: runner.os == 'Linux' + run: | + # Repair HNSW wheel + cd packages/leann-backend-hnsw + if [ -d dist ]; then + auditwheel repair dist/*.whl -w dist_repaired + rm -rf dist + mv dist_repaired dist + fi + cd ../.. + # Repair DiskANN wheel + cd packages/leann-backend-diskann + if [ -d dist ]; then + auditwheel repair dist/*.whl -w dist_repaired + rm -rf dist + mv dist_repaired dist + fi + cd ../.. + + - name: Repair wheels (macOS) + if: runner.os == 'macOS' + run: | + # Repair HNSW wheel + cd packages/leann-backend-hnsw + if [ -d dist ]; then + delocate-wheel -w dist_repaired -v dist/*.whl + rm -rf dist + mv dist_repaired dist + fi + cd ../.. + + # Repair DiskANN wheel + cd packages/leann-backend-diskann + if [ -d dist ]; then + delocate-wheel -w dist_repaired -v dist/*.whl + rm -rf dist + mv dist_repaired dist + fi + cd ../.. + + - name: List built packages + run: | echo "đŸ“Ļ Built packages:" find packages/*/dist -name "*.whl" -o -name "*.tar.gz" | sort - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: packages-${{ matrix.os }}-${{ matrix.python }} + name: packages-${{ matrix.os }}-py${{ matrix.python }} path: packages/*/dist/ \ No newline at end of file From d8b4ea75644010812479690e13a1a084d12aad85 Mon Sep 17 00:00:00 2001 From: Andy Lee Date: Thu, 24 Jul 2025 20:55:24 -0700 Subject: [PATCH 09/16] fix: add write permissions for GitHub Actions to push commits --- .github/workflows/release-manual.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-manual.yml b/.github/workflows/release-manual.yml index 27d0f25..63d4e7c 100644 --- a/.github/workflows/release-manual.yml +++ b/.github/workflows/release-manual.yml @@ -12,13 +12,13 @@ jobs: update-version: name: Update Version runs-on: ubuntu-latest + permissions: + contents: write outputs: commit-sha: ${{ steps.push.outputs.commit-sha }} steps: - uses: actions/checkout@v4 - with: - token: ${{ secrets.GITHUB_TOKEN }} - name: Validate version run: | From ed27a127d561beeda05273ed6f59dbea0a08f37d Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Fri, 25 Jul 2025 04:00:54 +0000 Subject: [PATCH 10/16] chore: release v0.1.5 --- packages/leann-backend-diskann/pyproject.toml | 4 ++-- packages/leann-backend-hnsw/pyproject.toml | 4 ++-- packages/leann-core/pyproject.toml | 2 +- packages/leann/pyproject.toml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/leann-backend-diskann/pyproject.toml b/packages/leann-backend-diskann/pyproject.toml index b4d299e..1794dcd 100644 --- a/packages/leann-backend-diskann/pyproject.toml +++ b/packages/leann-backend-diskann/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "scikit_build_core.build" [project] name = "leann-backend-diskann" -version = "0.1.4" -dependencies = ["leann-core==0.1.4", "numpy"] +version = "0.1.5" +dependencies = ["leann-core==0.1.5", "numpy"] [tool.scikit-build] # Key: simplified CMake path diff --git a/packages/leann-backend-hnsw/pyproject.toml b/packages/leann-backend-hnsw/pyproject.toml index ed52429..01f0f5b 100644 --- a/packages/leann-backend-hnsw/pyproject.toml +++ b/packages/leann-backend-hnsw/pyproject.toml @@ -6,10 +6,10 @@ build-backend = "scikit_build_core.build" [project] name = "leann-backend-hnsw" -version = "0.1.4" +version = "0.1.5" description = "Custom-built HNSW (Faiss) backend for the Leann toolkit." dependencies = [ - "leann-core==0.1.4", + "leann-core==0.1.5", "numpy", "pyzmq>=23.0.0", "msgpack>=1.0.0", diff --git a/packages/leann-core/pyproject.toml b/packages/leann-core/pyproject.toml index ee0db23..667d8e9 100644 --- a/packages/leann-core/pyproject.toml +++ b/packages/leann-core/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "leann-core" -version = "0.1.4" +version = "0.1.5" description = "Core API and plugin system for LEANN" readme = "README.md" requires-python = ">=3.9" diff --git a/packages/leann/pyproject.toml b/packages/leann/pyproject.toml index 2671852..81dd843 100644 --- a/packages/leann/pyproject.toml +++ b/packages/leann/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "leann" -version = "0.1.4" +version = "0.1.5" description = "LEANN - The smallest vector index in the world. RAG Everything with LEANN!" readme = "README.md" requires-python = ">=3.9" From a6aec68f321441ded57b8c5bce759841de934099 Mon Sep 17 00:00:00 2001 From: Andy Lee Date: Thu, 24 Jul 2025 21:26:28 -0700 Subject: [PATCH 11/16] fix: use manylinux2014 for better Linux compatibility - Change auditwheel --plat to manylinux2014_x86_64 - This ensures wheels work on Ubuntu 16.04+ instead of requiring 24.04+ - Fixes compatibility issues for users on Ubuntu 22.04 and similar systems --- .github/workflows/build-reusable.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-reusable.yml b/.github/workflows/build-reusable.yml index c2e7958..0d6539e 100644 --- a/.github/workflows/build-reusable.yml +++ b/.github/workflows/build-reusable.yml @@ -112,19 +112,19 @@ jobs: - name: Repair wheels (Linux) if: runner.os == 'Linux' run: | - # Repair HNSW wheel + # Repair HNSW wheel - use manylinux2014 for better compatibility cd packages/leann-backend-hnsw if [ -d dist ]; then - auditwheel repair dist/*.whl -w dist_repaired + auditwheel repair dist/*.whl -w dist_repaired --plat manylinux2014_x86_64 rm -rf dist mv dist_repaired dist fi cd ../.. - # Repair DiskANN wheel + # Repair DiskANN wheel - use manylinux2014 for better compatibility cd packages/leann-backend-diskann if [ -d dist ]; then - auditwheel repair dist/*.whl -w dist_repaired + auditwheel repair dist/*.whl -w dist_repaired --plat manylinux2014_x86_64 rm -rf dist mv dist_repaired dist fi From 166986d5e6457fe9f9ee5a4243b9924590431f79 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Fri, 25 Jul 2025 04:30:07 +0000 Subject: [PATCH 12/16] chore: release v0.1.6 --- packages/leann-backend-diskann/pyproject.toml | 4 ++-- packages/leann-backend-hnsw/pyproject.toml | 4 ++-- packages/leann-core/pyproject.toml | 2 +- packages/leann/pyproject.toml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/leann-backend-diskann/pyproject.toml b/packages/leann-backend-diskann/pyproject.toml index 1794dcd..0384fbf 100644 --- a/packages/leann-backend-diskann/pyproject.toml +++ b/packages/leann-backend-diskann/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "scikit_build_core.build" [project] name = "leann-backend-diskann" -version = "0.1.5" -dependencies = ["leann-core==0.1.5", "numpy"] +version = "0.1.6" +dependencies = ["leann-core==0.1.6", "numpy"] [tool.scikit-build] # Key: simplified CMake path diff --git a/packages/leann-backend-hnsw/pyproject.toml b/packages/leann-backend-hnsw/pyproject.toml index 01f0f5b..a977f1d 100644 --- a/packages/leann-backend-hnsw/pyproject.toml +++ b/packages/leann-backend-hnsw/pyproject.toml @@ -6,10 +6,10 @@ build-backend = "scikit_build_core.build" [project] name = "leann-backend-hnsw" -version = "0.1.5" +version = "0.1.6" description = "Custom-built HNSW (Faiss) backend for the Leann toolkit." dependencies = [ - "leann-core==0.1.5", + "leann-core==0.1.6", "numpy", "pyzmq>=23.0.0", "msgpack>=1.0.0", diff --git a/packages/leann-core/pyproject.toml b/packages/leann-core/pyproject.toml index 667d8e9..852b75e 100644 --- a/packages/leann-core/pyproject.toml +++ b/packages/leann-core/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "leann-core" -version = "0.1.5" +version = "0.1.6" description = "Core API and plugin system for LEANN" readme = "README.md" requires-python = ">=3.9" diff --git a/packages/leann/pyproject.toml b/packages/leann/pyproject.toml index 81dd843..ce494fa 100644 --- a/packages/leann/pyproject.toml +++ b/packages/leann/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "leann" -version = "0.1.5" +version = "0.1.6" description = "LEANN - The smallest vector index in the world. RAG Everything with LEANN!" readme = "README.md" requires-python = ">=3.9" From 2dd59c4ba130ed64f14cb9f85c4261ff0e3442ec Mon Sep 17 00:00:00 2001 From: Andy Lee Date: Thu, 24 Jul 2025 21:44:04 -0700 Subject: [PATCH 13/16] fix: let auditwheel auto-detect manylinux platform tag - Remove --plat manylinux2014_x86_64 flag that was causing build failures - Let auditwheel automatically determine the appropriate manylinux tag - Add auditwheel show command to display compatibility info - This fixes the 'too-recent versioned symbols' error --- .github/workflows/build-reusable.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-reusable.yml b/.github/workflows/build-reusable.yml index 0d6539e..dfdbea2 100644 --- a/.github/workflows/build-reusable.yml +++ b/.github/workflows/build-reusable.yml @@ -112,19 +112,25 @@ jobs: - name: Repair wheels (Linux) if: runner.os == 'Linux' run: | - # Repair HNSW wheel - use manylinux2014 for better compatibility + # Repair HNSW wheel cd packages/leann-backend-hnsw if [ -d dist ]; then - auditwheel repair dist/*.whl -w dist_repaired --plat manylinux2014_x86_64 + # Show what platform auditwheel will use + auditwheel show dist/*.whl || true + # Let auditwheel auto-detect the appropriate manylinux tag + auditwheel repair dist/*.whl -w dist_repaired rm -rf dist mv dist_repaired dist fi cd ../.. - # Repair DiskANN wheel - use manylinux2014 for better compatibility + # Repair DiskANN wheel cd packages/leann-backend-diskann if [ -d dist ]; then - auditwheel repair dist/*.whl -w dist_repaired --plat manylinux2014_x86_64 + # Show what platform auditwheel will use + auditwheel show dist/*.whl || true + # Let auditwheel auto-detect the appropriate manylinux tag + auditwheel repair dist/*.whl -w dist_repaired rm -rf dist mv dist_repaired dist fi From e64b599276ecd9c6ae687a9dca53c7a5776fbb01 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Fri, 25 Jul 2025 04:47:57 +0000 Subject: [PATCH 14/16] chore: release v0.1.7 --- packages/leann-backend-diskann/pyproject.toml | 4 ++-- packages/leann-backend-hnsw/pyproject.toml | 4 ++-- packages/leann-core/pyproject.toml | 2 +- packages/leann/pyproject.toml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/leann-backend-diskann/pyproject.toml b/packages/leann-backend-diskann/pyproject.toml index 0384fbf..8622bec 100644 --- a/packages/leann-backend-diskann/pyproject.toml +++ b/packages/leann-backend-diskann/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "scikit_build_core.build" [project] name = "leann-backend-diskann" -version = "0.1.6" -dependencies = ["leann-core==0.1.6", "numpy"] +version = "0.1.7" +dependencies = ["leann-core==0.1.7", "numpy"] [tool.scikit-build] # Key: simplified CMake path diff --git a/packages/leann-backend-hnsw/pyproject.toml b/packages/leann-backend-hnsw/pyproject.toml index a977f1d..52fbacb 100644 --- a/packages/leann-backend-hnsw/pyproject.toml +++ b/packages/leann-backend-hnsw/pyproject.toml @@ -6,10 +6,10 @@ build-backend = "scikit_build_core.build" [project] name = "leann-backend-hnsw" -version = "0.1.6" +version = "0.1.7" description = "Custom-built HNSW (Faiss) backend for the Leann toolkit." dependencies = [ - "leann-core==0.1.6", + "leann-core==0.1.7", "numpy", "pyzmq>=23.0.0", "msgpack>=1.0.0", diff --git a/packages/leann-core/pyproject.toml b/packages/leann-core/pyproject.toml index 852b75e..6dc85b4 100644 --- a/packages/leann-core/pyproject.toml +++ b/packages/leann-core/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "leann-core" -version = "0.1.6" +version = "0.1.7" description = "Core API and plugin system for LEANN" readme = "README.md" requires-python = ">=3.9" diff --git a/packages/leann/pyproject.toml b/packages/leann/pyproject.toml index ce494fa..c50c996 100644 --- a/packages/leann/pyproject.toml +++ b/packages/leann/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "leann" -version = "0.1.6" +version = "0.1.7" description = "LEANN - The smallest vector index in the world. RAG Everything with LEANN!" readme = "README.md" requires-python = ">=3.9" From 6df7893173ee0f7a1b1650e23b38a7996df3eaf7 Mon Sep 17 00:00:00 2001 From: Andy Lee Date: Fri, 25 Jul 2025 00:08:42 -0700 Subject: [PATCH 15/16] feat: use manylinux2014 containers for better Linux compatibility - Add manylinux2014 Docker containers for Linux builds - This will generate wheels compatible with older Linux systems (CentOS 7+, Ubuntu 16.04+) - Separate build logic for container vs regular environments - Install appropriate system dependencies for yum-based manylinux environment - Use pip instead of uv in containers for better compatibility - Fix Python version format for manylinux container paths --- .github/workflows/build-reusable.yml | 88 +++++++++++++++++++++++----- 1 file changed, 74 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build-reusable.yml b/.github/workflows/build-reusable.yml index dfdbea2..6c804b7 100644 --- a/.github/workflows/build-reusable.yml +++ b/.github/workflows/build-reusable.yml @@ -17,21 +17,30 @@ jobs: include: - os: ubuntu-latest python: '3.9' + container: 'quay.io/pypa/manylinux2014_x86_64' - os: ubuntu-latest python: '3.10' + container: 'quay.io/pypa/manylinux2014_x86_64' - os: ubuntu-latest python: '3.11' + container: 'quay.io/pypa/manylinux2014_x86_64' - os: ubuntu-latest python: '3.12' + container: 'quay.io/pypa/manylinux2014_x86_64' - os: macos-latest python: '3.9' + container: '' - os: macos-latest python: '3.10' + container: '' - os: macos-latest python: '3.11' + container: '' - os: macos-latest python: '3.12' + container: '' runs-on: ${{ matrix.os }} + container: ${{ matrix.container }} steps: - uses: actions/checkout@v4 @@ -39,16 +48,35 @@ jobs: ref: ${{ inputs.ref }} submodules: recursive - - name: Setup Python + - name: Setup Python (macOS and regular Ubuntu) + if: matrix.container == '' uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - - name: Install uv + - name: Setup Python (manylinux container) + if: matrix.container != '' + run: | + # Use the pre-installed Python version in manylinux container + # Convert Python version format (3.9 -> 39, 3.10 -> 310, etc.) + PY_VER=$(echo "${{ matrix.python }}" | sed 's/\.//g') + /opt/python/cp${PY_VER}-*/bin/python -m pip install --upgrade pip + # Create symlinks for convenience + ln -sf /opt/python/cp${PY_VER}-*/bin/python /usr/local/bin/python + ln -sf /opt/python/cp${PY_VER}-*/bin/pip /usr/local/bin/pip + + - name: Install uv (macOS and regular Ubuntu) + if: matrix.container == '' uses: astral-sh/setup-uv@v4 - - name: Install system dependencies (Ubuntu) - if: runner.os == 'Linux' + - name: Install uv (manylinux container) + if: matrix.container != '' + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + + - name: Install system dependencies (Ubuntu - regular) + if: runner.os == 'Linux' && matrix.container == '' run: | sudo apt-get update sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \ @@ -61,6 +89,17 @@ jobs: echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV echo "LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH" >> $GITHUB_ENV + - name: Install system dependencies (manylinux container) + if: runner.os == 'Linux' && matrix.container != '' + run: | + # manylinux2014 uses yum instead of apt + yum install -y epel-release + yum install -y boost-devel protobuf-compiler zeromq-devel \ + pkg-config openblas-devel libaio-devel protobuf-devel + + # Build tools are pre-installed in manylinux + # MKL is more complex in container, skip for now and use OpenBLAS + - name: Install system dependencies (macOS) if: runner.os == 'macOS' run: | @@ -68,44 +107,65 @@ jobs: - name: Install build dependencies run: | - uv pip install --system scikit-build-core numpy swig Cython pybind11 - if [[ "$RUNNER_OS" == "Linux" ]]; then - uv pip install --system auditwheel + if [[ -n "${{ matrix.container }}" ]]; then + # In manylinux container, use regular pip + pip install scikit-build-core numpy swig Cython pybind11 auditwheel else - uv pip install --system delocate + # Regular environment, use uv + uv pip install --system scikit-build-core numpy swig Cython pybind11 + if [[ "$RUNNER_OS" == "Linux" ]]; then + uv pip install --system auditwheel + else + uv pip install --system delocate + fi fi - name: Build packages run: | + # Choose build command based on environment + if [[ -n "${{ matrix.container }}" ]]; then + BUILD_CMD="pip wheel . --no-deps -w dist" + else + BUILD_CMD="uv build --wheel --python python" + fi + # Build core (platform independent) if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then cd packages/leann-core - uv build + if [[ -n "${{ matrix.container }}" ]]; then + pip wheel . --no-deps -w dist + else + uv build + fi cd ../.. fi # Build HNSW backend cd packages/leann-backend-hnsw if [ "${{ matrix.os }}" == "macos-latest" ]; then - CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build --wheel --python python + CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ $BUILD_CMD else - uv build --wheel --python python + eval $BUILD_CMD fi cd ../.. # Build DiskANN backend cd packages/leann-backend-diskann if [ "${{ matrix.os }}" == "macos-latest" ]; then - CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build --wheel --python python + CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ $BUILD_CMD else - uv build --wheel --python python + eval $BUILD_CMD fi cd ../.. # Build meta package (platform independent) if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then cd packages/leann - uv build + if [[ -n "${{ matrix.container }}" ]]; then + pip wheel . --no-deps -w dist + else + uv build + fi cd ../.. fi From 3603cd5034dea817b2e415becdc084606c402917 Mon Sep 17 00:00:00 2001 From: Andy Lee Date: Fri, 25 Jul 2025 00:12:05 -0700 Subject: [PATCH 16/16] fix: downgrade GitHub Actions versions for manylinux2014 compatibility - Use actions/checkout@v3 instead of v4 (Node.js 16 vs 20) - Use actions/setup-python@v4 instead of v5 - Use actions/upload-artifact@v3 and download-artifact@v3 - This fixes GLIBC version errors in manylinux2014 containers - manylinux2014 (CentOS 7) has glibc 2.17 but Node.js 20 needs 2.25+ --- .github/workflows/build-reusable.yml | 7 ++++--- .github/workflows/release-manual.yml | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-reusable.yml b/.github/workflows/build-reusable.yml index 6c804b7..77c79b0 100644 --- a/.github/workflows/build-reusable.yml +++ b/.github/workflows/build-reusable.yml @@ -43,14 +43,15 @@ jobs: container: ${{ matrix.container }} steps: - - uses: actions/checkout@v4 + # Use v3 for manylinux2014 compatibility (Node.js 16 instead of 20) + - uses: actions/checkout@v3 with: ref: ${{ inputs.ref }} submodules: recursive - name: Setup Python (macOS and regular Ubuntu) if: matrix.container == '' - uses: actions/setup-python@v5 + uses: actions/setup-python@v4 # v4 for better compatibility with: python-version: ${{ matrix.python }} @@ -223,7 +224,7 @@ jobs: find packages/*/dist -name "*.whl" -o -name "*.tar.gz" | sort - name: Upload artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 # v3 for manylinux2014 compatibility with: name: packages-${{ matrix.os }}-py${{ matrix.python }} path: packages/*/dist/ \ No newline at end of file diff --git a/.github/workflows/release-manual.yml b/.github/workflows/release-manual.yml index 63d4e7c..6623f44 100644 --- a/.github/workflows/release-manual.yml +++ b/.github/workflows/release-manual.yml @@ -18,7 +18,7 @@ jobs: commit-sha: ${{ steps.push.outputs.commit-sha }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v3 # Compatibility with manylinux2014 - name: Validate version run: | @@ -57,12 +57,12 @@ jobs: contents: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v3 # Consistency with build workflow with: ref: ${{ needs.update-version.outputs.commit-sha }} - name: Download all artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v3 # Match upload-artifact version with: path: dist-artifacts