diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index dd1a5d7..c19ed2f 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -1,15 +1,14 @@ -name: Build and Publish to PyPI +name: CI - Build Multi-Platform Packages on: - release: - types: [published] push: - tags: - - 'v*' + branches: [ main ] + pull_request: + branches: [ main ] workflow_dispatch: inputs: publish: - description: 'Publish to PyPI' + description: 'Publish to PyPI (only use for emergency fixes)' required: true default: 'false' type: choice @@ -221,12 +220,12 @@ jobs: name: leann-meta-dist path: packages/leann/dist/ - # Publish to PyPI + # Publish to PyPI (only for emergency fixes or manual triggers) publish: - name: Publish to PyPI + name: Publish to PyPI (Emergency) needs: [build-core, build-hnsw, build-diskann, build-meta] runs-on: ubuntu-latest - if: github.event_name == 'release' || github.event.inputs.publish != 'false' + if: github.event_name == 'workflow_dispatch' && github.event.inputs.publish != 'false' steps: - name: Download all artifacts @@ -240,17 +239,24 @@ jobs: find dist -name "*.whl" -exec cp {} all_wheels/ \; find dist -name "*.tar.gz" -exec cp {} all_wheels/ \; + - name: Show what will be published + run: | + echo "📦 Packages to be published:" + ls -la all_wheels/ + - name: Publish to Test PyPI - if: github.event.inputs.publish == 'test' || github.event_name == 'workflow_dispatch' + if: github.event.inputs.publish == 'test' uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.TEST_PYPI_API_TOKEN }} repository-url: https://test.pypi.org/legacy/ packages-dir: all_wheels/ + skip-existing: true - name: Publish to PyPI - if: github.event_name == 'release' || github.event.inputs.publish == 'prod' + if: github.event.inputs.publish == 'prod' uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.PYPI_API_TOKEN }} - packages-dir: all_wheels/ \ No newline at end of file + packages-dir: all_wheels/ + skip-existing: true \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 9aa4b72..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,110 +0,0 @@ -name: CI - Build and Test - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - build-test: - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] - - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install uv - run: | - curl -LsSf https://astral.sh/uv/install.sh | sh - echo "$HOME/.cargo/bin" >> $GITHUB_PATH - - - name: Install system dependencies (Ubuntu) - if: runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get install -y libomp-dev libboost-all-dev libzmq3-dev \ - pkg-config libopenblas-dev patchelf \ - libaio-dev protobuf-compiler libprotobuf-dev libabsl-dev - - # Install Intel MKL for DiskANN - wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/79153e0f-74d7-45af-b8c2-258941adf58a/intel-onemkl-2025.0.0.940.sh - sudo sh intel-onemkl-2025.0.0.940.sh -a --components intel.oneapi.lin.mkl.devel --action install --eula accept -s - source /opt/intel/oneapi/setvars.sh - echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH" >> $GITHUB_ENV - - - name: Install system dependencies (macOS) - if: runner.os == 'macOS' - run: | - brew install libomp boost zeromq protobuf - - - name: Build all packages - run: | - echo "🔨 Building on ${{ matrix.os }} with Python ${{ matrix.python-version }}..." - export UV_SYSTEM_PYTHON=1 - - # Verify Python version - python --version - which python - - # Build each package - for pkg in leann-core leann-backend-hnsw leann-backend-diskann leann; do - echo "Building $pkg..." - cd packages/$pkg - rm -rf dist/ build/ _skbuild/ - # Use explicit python interpreter - uv build --wheel --python python - if [ ! -f dist/*.whl ]; then - echo "❌ Failed to build $pkg!" - exit 1 - fi - echo "✅ $pkg built successfully" - cd ../.. - done - - - name: Install and test packages - run: | - # Create clean test environment - python -m venv test_env - if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "win32" ]]; then - source test_env/Scripts/activate - else - source test_env/bin/activate - fi - - # Install built packages - pip install packages/*/dist/*.whl - - # Basic import test - python -c "import leann; print('✅ LEANN imported successfully')" - python -c "import leann_backend_hnsw; print('✅ HNSW backend imported')" - python -c "import leann_backend_diskann; print('✅ DiskANN backend imported')" - - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: wheels-${{ matrix.os }}-py${{ matrix.python-version }} - path: packages/*/dist/*.whl - retention-days: 7 - - # Summary job to ensure all builds pass - ci-success: - needs: build-test - runs-on: ubuntu-latest - steps: - - name: CI Success - run: | - echo "✅ All CI builds passed!" - echo "Ready for manual release when needed." \ No newline at end of file diff --git a/.github/workflows/release-manual.yml b/.github/workflows/release-manual.yml index d9159c8..6310a2f 100644 --- a/.github/workflows/release-manual.yml +++ b/.github/workflows/release-manual.yml @@ -65,41 +65,49 @@ jobs: git add packages/*/pyproject.toml git commit -m "chore: release v${{ inputs.version }}" - - name: Get CI run ID - id: get-ci-run + - name: Push version update run: | - # Get the latest successful CI run on the previous commit (before version bump) - COMMIT_SHA=$(git rev-parse HEAD~1) - RUN_ID=$(gh run list \ - --workflow="CI - Build and Test" \ - --status=success \ - --commit=$COMMIT_SHA \ - --json databaseId \ - --jq '.[0].databaseId') + git push origin HEAD:main + echo "✅ Pushed version update to main branch" + COMMIT_SHA=$(git rev-parse HEAD) + echo "commit-sha=$COMMIT_SHA" >> $GITHUB_OUTPUT + id: push-version - if [ -z "$RUN_ID" ]; then - echo "❌ No successful CI run found for commit $COMMIT_SHA" - echo "" - echo "This usually means:" - echo "1. CI hasn't run on the latest commit yet" - echo "2. CI failed on the latest commit" - echo "" - echo "Please ensure CI passes on main branch before releasing." - exit 1 - fi + - name: Wait for CI to complete + id: wait-for-ci + run: | + echo "⏳ Waiting for CI to build new version..." + COMMIT_SHA="${{ steps.push-version.outputs.commit-sha }}" - echo "✅ Found CI run: $RUN_ID" - echo "run-id=$RUN_ID" >> $GITHUB_OUTPUT + # Wait up to 20 minutes for CI to complete + for i in {1..40}; do + RUN_ID=$(gh run list \ + --workflow="CI - Build Multi-Platform Packages" \ + --commit=$COMMIT_SHA \ + --json databaseId,status \ + --jq '.[] | select(.status == "completed") | .databaseId' | head -1) + + if [ ! -z "$RUN_ID" ]; then + echo "✅ Found completed CI run: $RUN_ID" + echo "run-id=$RUN_ID" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "⏳ Waiting for CI... (attempt $i/40)" + sleep 30 + done + + echo "❌ CI did not complete within 20 minutes" + exit 1 env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Download artifacts from CI + - name: Download artifacts from CI run run: | - echo "📦 Downloading artifacts from CI run ${{ steps.get-ci-run.outputs.run-id }}..." + echo "📦 Downloading artifacts from CI run ${{ steps.wait-for-ci.outputs.run-id }}..." - # Download all wheel artifacts - gh run download ${{ steps.get-ci-run.outputs.run-id }} \ - --pattern "wheels-*" \ + # Download all artifacts (not just wheels-*) + gh run download ${{ steps.wait-for-ci.outputs.run-id }} \ --dir ./dist-downloads # Consolidate all wheels into packages/*/dist/ @@ -131,19 +139,58 @@ jobs: - name: Test on TestPyPI (optional) if: inputs.test_pypi + continue-on-error: true env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }} run: | + if [ -z "$TWINE_PASSWORD" ]; then + echo "⚠️ TEST_PYPI_API_TOKEN not configured, skipping TestPyPI upload" + echo " To enable TestPyPI testing, add TEST_PYPI_API_TOKEN to repository secrets" + exit 0 + fi + pip install twine echo "📦 Uploading to TestPyPI..." - twine upload --repository testpypi packages/*/dist/* --verbose + twine upload --repository testpypi packages/*/dist/* --verbose || { + echo "⚠️ TestPyPI upload failed, but continuing with release" + echo " This is optional and won't block the release" + exit 0 + } echo "✅ Test upload successful!" echo "📋 Check packages at: https://test.pypi.org/user/your-username/" echo "" echo "To test installation:" echo "pip install -i https://test.pypi.org/simple/ leann" + - name: Publish to PyPI + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + run: | + if [ -z "$TWINE_PASSWORD" ]; then + echo "❌ PYPI_API_TOKEN not configured!" + echo " Please add PYPI_API_TOKEN to repository secrets" + exit 1 + fi + + pip install twine + echo "📦 Publishing to PyPI..." + + # Collect all wheels in one place + mkdir -p all_wheels + find packages/*/dist -name "*.whl" -exec cp {} all_wheels/ \; + find packages/*/dist -name "*.tar.gz" -exec cp {} all_wheels/ \; + + echo "📋 Packages to publish:" + ls -la all_wheels/ + + # Upload to PyPI + twine upload all_wheels/* --skip-existing --verbose + + echo "✅ Published to PyPI!" + echo "🎉 Check packages at: https://pypi.org/project/leann/" + - name: Create and push tag run: | git tag "v${{ inputs.version }}" @@ -165,19 +212,4 @@ jobs: ### Installation ```bash pip install leann==${{ inputs.version }} - ``` - - ### Test Installation (if using TestPyPI) - ```bash - pip install -i https://test.pypi.org/simple/ leann==${{ inputs.version }} - ``` - draft: false - prerelease: false - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Trigger PyPI publish - run: | - echo "🚀 Triggering PyPI publish workflow..." - # The existing build-and-publish.yml will be triggered by the tag push - echo "✅ Release process completed! The publish workflow will run automatically." \ No newline at end of file + ``` \ No newline at end of file diff --git a/docs/RELEASE.md b/docs/RELEASE.md index 0bc704a..985a282 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -7,6 +7,24 @@ Before releasing, ensure: 2. ✅ CI has passed on the latest commit (check [Actions](https://github.com/yichuan-w/LEANN/actions/workflows/ci.yml)) 3. ✅ You have determined the new version number +### Required: PyPI Configuration + +To enable PyPI publishing: +1. Get a PyPI API token from https://pypi.org/manage/account/token/ +2. Add it to repository secrets: Settings → Secrets → Actions → New repository secret + - Name: `PYPI_API_TOKEN` + - Value: Your PyPI token (starts with `pypi-`) + +### Optional: TestPyPI Configuration + +To enable TestPyPI testing (recommended but not required): +1. Get a TestPyPI API token from https://test.pypi.org/manage/account/token/ +2. Add it to repository secrets: Settings → Secrets → Actions → New repository secret + - Name: `TEST_PYPI_API_TOKEN` + - Value: Your TestPyPI token (starts with `pypi-`) + +**Note**: TestPyPI testing is optional. If not configured, the release will skip TestPyPI and proceed. + ## 🚀 Recommended: Manual Release Workflow ### Via GitHub UI (Most Reliable) @@ -19,12 +37,11 @@ Before releasing, ensure: 6. Click "Run workflow" **What happens:** -- ✅ Validates version format - ✅ Downloads pre-built packages from CI (no rebuild needed!) - ✅ Updates all package versions - ✅ Optionally tests on TestPyPI +- ✅ **Publishes directly to PyPI** - ✅ Creates tag and GitHub release -- ✅ Automatically triggers PyPI publish ### Via Command Line @@ -44,7 +61,7 @@ This script will: 1. Update all package versions 2. Commit and push changes 3. Create GitHub release -4. CI automatically builds and publishes to PyPI +4. **Manual Release workflow will automatically publish to PyPI** ⚠️ **Note**: If CI fails, you'll need to manually fix and re-tag diff --git a/packages/leann-backend-diskann/pyproject.toml b/packages/leann-backend-diskann/pyproject.toml index 24637cc..c48b8ed 100644 --- a/packages/leann-backend-diskann/pyproject.toml +++ b/packages/leann-backend-diskann/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "scikit_build_core.build" [project] name = "leann-backend-diskann" -version = "0.1.0" -dependencies = ["leann-core==0.1.0", "numpy"] +version = "0.1.2" +dependencies = ["leann-core==0.1.2", "numpy"] [tool.scikit-build] # Key: simplified CMake path diff --git a/packages/leann-backend-diskann/third_party/DiskANN b/packages/leann-backend-diskann/third_party/DiskANN index af2a264..25339b0 160000 --- a/packages/leann-backend-diskann/third_party/DiskANN +++ b/packages/leann-backend-diskann/third_party/DiskANN @@ -1 +1 @@ -Subproject commit af2a26481e65232b57b82d96e68833cdee9f7635 +Subproject commit 25339b03413b5067c25b6092ea3e0f77ef8515c8 diff --git a/packages/leann-backend-hnsw/pyproject.toml b/packages/leann-backend-hnsw/pyproject.toml index 3aabd89..14b0055 100644 --- a/packages/leann-backend-hnsw/pyproject.toml +++ b/packages/leann-backend-hnsw/pyproject.toml @@ -6,10 +6,10 @@ build-backend = "scikit_build_core.build" [project] name = "leann-backend-hnsw" -version = "0.1.0" +version = "0.1.2" description = "Custom-built HNSW (Faiss) backend for the Leann toolkit." dependencies = [ - "leann-core==0.1.0", + "leann-core==0.1.2", "numpy", "pyzmq>=23.0.0", "msgpack>=1.0.0", diff --git a/packages/leann-core/pyproject.toml b/packages/leann-core/pyproject.toml index 6a3606a..b0c06f5 100644 --- a/packages/leann-core/pyproject.toml +++ b/packages/leann-core/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "leann-core" -version = "0.1.0" +version = "0.1.2" description = "Core API and plugin system for LEANN" readme = "README.md" requires-python = ">=3.9" diff --git a/packages/leann/pyproject.toml b/packages/leann/pyproject.toml index 54a470c..31844c4 100644 --- a/packages/leann/pyproject.toml +++ b/packages/leann/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "leann" -version = "0.1.0" +version = "0.1.2" description = "LEANN - The smallest vector index in the world. RAG Everything with LEANN!" readme = "README.md" requires-python = ">=3.9" diff --git a/scripts/bump_version.sh b/scripts/bump_version.sh index 20b7e56..71043bb 100755 --- a/scripts/bump_version.sh +++ b/scripts/bump_version.sh @@ -7,12 +7,25 @@ fi NEW_VERSION=$1 +# Get the directory where the script is located +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +PROJECT_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )" + # Update all pyproject.toml files +echo "Updating versions in $PROJECT_ROOT/packages/" + # Use different sed syntax for macOS vs Linux if [[ "$OSTYPE" == "darwin"* ]]; then - sed -i '' "s/version = \".*\"/version = \"$NEW_VERSION\"/" packages/*/pyproject.toml + # Update version fields + find "$PROJECT_ROOT/packages" -name "pyproject.toml" -exec sed -i '' "s/version = \".*\"/version = \"$NEW_VERSION\"/" {} \; + # Update leann-core dependencies + find "$PROJECT_ROOT/packages" -name "pyproject.toml" -exec sed -i '' "s/leann-core==[0-9.]*/leann-core==$NEW_VERSION/" {} \; else - sed -i "s/version = \".*\"/version = \"$NEW_VERSION\"/" packages/*/pyproject.toml + # Update version fields + find "$PROJECT_ROOT/packages" -name "pyproject.toml" -exec sed -i "s/version = \".*\"/version = \"$NEW_VERSION\"/" {} \; + # Update leann-core dependencies + find "$PROJECT_ROOT/packages" -name "pyproject.toml" -exec sed -i "s/leann-core==[0-9.]*/leann-core==$NEW_VERSION/" {} \; fi -echo "Version updated to $NEW_VERSION" \ No newline at end of file +echo "✅ Version updated to $NEW_VERSION" +echo "✅ Dependencies updated to use leann-core==$NEW_VERSION" \ No newline at end of file