Merge branch 'main' of https://github.com/yichuan-w/LEANN

2025-07-24 19:03:52 -07:00
parent 5461b71d8c 374db0ebb8
commit 43eb4f9a1d
10 changed files with 137 additions and 179 deletions
--- a/.github/workflows/build-and-publish.yml
+++ b/.github/workflows/build-and-publish.yml
@@ -1,15 +1,14 @@
-name: Build and Publish to PyPI
+name: CI - Build Multi-Platform Packages

 on:
-  release:
-    types: [published]
  push:
-    tags:
-      - 'v*'
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
  workflow_dispatch:
    inputs:
      publish:
-        description: 'Publish to PyPI'
+        description: 'Publish to PyPI (only use for emergency fixes)'
        required: true
        default: 'false'
        type: choice
@@ -221,12 +220,12 @@ jobs:
          name: leann-meta-dist
          path: packages/leann/dist/

-  # Publish to PyPI
+  # Publish to PyPI (only for emergency fixes or manual triggers)
  publish:
-    name: Publish to PyPI
+    name: Publish to PyPI (Emergency)
    needs: [build-core, build-hnsw, build-diskann, build-meta]
    runs-on: ubuntu-latest
-    if: github.event_name == 'release' || github.event.inputs.publish != 'false'
+    if: github.event_name == 'workflow_dispatch' && github.event.inputs.publish != 'false'
    
    steps:
      - name: Download all artifacts
@@ -240,17 +239,24 @@ jobs:
          find dist -name "*.whl" -exec cp {} all_wheels/ \;
          find dist -name "*.tar.gz" -exec cp {} all_wheels/ \;
          
+      - name: Show what will be published
+        run: |
+          echo "📦 Packages to be published:"
+          ls -la all_wheels/
+          
      - name: Publish to Test PyPI
-        if: github.event.inputs.publish == 'test' || github.event_name == 'workflow_dispatch'
+        if: github.event.inputs.publish == 'test'
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          password: ${{ secrets.TEST_PYPI_API_TOKEN }}
          repository-url: https://test.pypi.org/legacy/
          packages-dir: all_wheels/
+          skip-existing: true
          
      - name: Publish to PyPI
-        if: github.event_name == 'release' || github.event.inputs.publish == 'prod'
+        if: github.event.inputs.publish == 'prod'
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          password: ${{ secrets.PYPI_API_TOKEN }}
-          packages-dir: all_wheels/ 
+          packages-dir: all_wheels/
+          skip-existing: true 
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1,110 +0,0 @@
-name: CI - Build and Test
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  build-test:
-    strategy:
-      matrix:
-        os: [ubuntu-latest, macos-latest]
-        python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
-    
-    runs-on: ${{ matrix.os }}
-    
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: recursive
-      
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-      
-      - name: Install uv
-        run: |
-          curl -LsSf https://astral.sh/uv/install.sh | sh
-          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
-      
-      - name: Install system dependencies (Ubuntu)
-        if: runner.os == 'Linux'
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y libomp-dev libboost-all-dev libzmq3-dev \
-            pkg-config libopenblas-dev patchelf \
-            libaio-dev protobuf-compiler libprotobuf-dev libabsl-dev
-          
-          # Install Intel MKL for DiskANN
-          wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/79153e0f-74d7-45af-b8c2-258941adf58a/intel-onemkl-2025.0.0.940.sh
-          sudo sh intel-onemkl-2025.0.0.940.sh -a --components intel.oneapi.lin.mkl.devel --action install --eula accept -s
-          source /opt/intel/oneapi/setvars.sh
-          echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV
-          echo "LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH" >> $GITHUB_ENV
-      
-      - name: Install system dependencies (macOS)
-        if: runner.os == 'macOS'
-        run: |
-          brew install libomp boost zeromq protobuf
-      
-      - name: Build all packages
-        run: |
-          echo "🔨 Building on ${{ matrix.os }} with Python ${{ matrix.python-version }}..."
-          export UV_SYSTEM_PYTHON=1
-          
-          # Verify Python version
-          python --version
-          which python
-          
-          # Build each package
-          for pkg in leann-core leann-backend-hnsw leann-backend-diskann leann; do
-            echo "Building $pkg..."
-            cd packages/$pkg
-            rm -rf dist/ build/ _skbuild/
-            # Use explicit python interpreter
-            uv build --wheel --python python
-            if [ ! -f dist/*.whl ]; then
-              echo "❌ Failed to build $pkg!"
-              exit 1
-            fi
-            echo "✅ $pkg built successfully"
-            cd ../..
-          done
-      
-      - name: Install and test packages
-        run: |
-          # Create clean test environment
-          python -m venv test_env
-          if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "win32" ]]; then
-            source test_env/Scripts/activate
-          else
-            source test_env/bin/activate
-          fi
-          
-          # Install built packages
-          pip install packages/*/dist/*.whl
-          
-          # Basic import test
-          python -c "import leann; print('✅ LEANN imported successfully')"
-          python -c "import leann_backend_hnsw; print('✅ HNSW backend imported')"
-          python -c "import leann_backend_diskann; print('✅ DiskANN backend imported')"
-      
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: wheels-${{ matrix.os }}-py${{ matrix.python-version }}
-          path: packages/*/dist/*.whl
-          retention-days: 7
-  
-  # Summary job to ensure all builds pass
-  ci-success:
-    needs: build-test
-    runs-on: ubuntu-latest
-    steps:
-      - name: CI Success
-        run: |
-          echo "✅ All CI builds passed!"
-          echo "Ready for manual release when needed." 
--- a/.github/workflows/release-manual.yml
+++ b/.github/workflows/release-manual.yml
@@ -65,41 +65,49 @@ jobs:
          git add packages/*/pyproject.toml
          git commit -m "chore: release v${{ inputs.version }}"
      
-      - name: Get CI run ID
-        id: get-ci-run
+      - name: Push version update
        run: |
-          # Get the latest successful CI run on the previous commit (before version bump)
-          COMMIT_SHA=$(git rev-parse HEAD~1)
-          RUN_ID=$(gh run list \
-            --workflow="CI - Build and Test" \
-            --status=success \
-            --commit=$COMMIT_SHA \
-            --json databaseId \
-            --jq '.[0].databaseId')
+          git push origin HEAD:main
+          echo "✅ Pushed version update to main branch"
+          COMMIT_SHA=$(git rev-parse HEAD)
+          echo "commit-sha=$COMMIT_SHA" >> $GITHUB_OUTPUT
+        id: push-version
          
-          if [ -z "$RUN_ID" ]; then
-            echo "❌ No successful CI run found for commit $COMMIT_SHA"
-            echo ""
-            echo "This usually means:"
-            echo "1. CI hasn't run on the latest commit yet"
-            echo "2. CI failed on the latest commit"
-            echo ""
-            echo "Please ensure CI passes on main branch before releasing."
-            exit 1
-          fi
+      - name: Wait for CI to complete
+        id: wait-for-ci
+        run: |
+          echo "⏳ Waiting for CI to build new version..."
+          COMMIT_SHA="${{ steps.push-version.outputs.commit-sha }}"
          
-          echo "✅ Found CI run: $RUN_ID"
-          echo "run-id=$RUN_ID" >> $GITHUB_OUTPUT
+          # Wait up to 20 minutes for CI to complete
+          for i in {1..40}; do
+            RUN_ID=$(gh run list \
+              --workflow="CI - Build Multi-Platform Packages" \
+              --commit=$COMMIT_SHA \
+              --json databaseId,status \
+              --jq '.[] | select(.status == "completed") | .databaseId' | head -1)
+            
+            if [ ! -z "$RUN_ID" ]; then
+              echo "✅ Found completed CI run: $RUN_ID"
+              echo "run-id=$RUN_ID" >> $GITHUB_OUTPUT
+              exit 0
+            fi
+            
+            echo "⏳ Waiting for CI... (attempt $i/40)"
+            sleep 30
+          done
+          
+          echo "❌ CI did not complete within 20 minutes"
+          exit 1
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      
-      - name: Download artifacts from CI
+      - name: Download artifacts from CI run
        run: |
-          echo "📦 Downloading artifacts from CI run ${{ steps.get-ci-run.outputs.run-id }}..."
+          echo "📦 Downloading artifacts from CI run ${{ steps.wait-for-ci.outputs.run-id }}..."
          
-          # Download all wheel artifacts
-          gh run download ${{ steps.get-ci-run.outputs.run-id }} \
-            --pattern "wheels-*" \
+          # Download all artifacts (not just wheels-*)
+          gh run download ${{ steps.wait-for-ci.outputs.run-id }} \
            --dir ./dist-downloads
          
          # Consolidate all wheels into packages/*/dist/
@@ -131,19 +139,58 @@ jobs:
      
      - name: Test on TestPyPI (optional)
        if: inputs.test_pypi
+        continue-on-error: true
        env:
          TWINE_USERNAME: __token__
          TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }}
        run: |
+          if [ -z "$TWINE_PASSWORD" ]; then
+            echo "⚠️  TEST_PYPI_API_TOKEN not configured, skipping TestPyPI upload"
+            echo "   To enable TestPyPI testing, add TEST_PYPI_API_TOKEN to repository secrets"
+            exit 0
+          fi
+          
          pip install twine
          echo "📦 Uploading to TestPyPI..."
-          twine upload --repository testpypi packages/*/dist/* --verbose
+          twine upload --repository testpypi packages/*/dist/* --verbose || {
+            echo "⚠️  TestPyPI upload failed, but continuing with release"
+            echo "   This is optional and won't block the release"
+            exit 0
+          }
          echo "✅ Test upload successful!"
          echo "📋 Check packages at: https://test.pypi.org/user/your-username/"
          echo ""
          echo "To test installation:"
          echo "pip install -i https://test.pypi.org/simple/ leann"
      
+      - name: Publish to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+        run: |
+          if [ -z "$TWINE_PASSWORD" ]; then
+            echo "❌ PYPI_API_TOKEN not configured!"
+            echo "   Please add PYPI_API_TOKEN to repository secrets"
+            exit 1
+          fi
+          
+          pip install twine
+          echo "📦 Publishing to PyPI..."
+          
+          # Collect all wheels in one place
+          mkdir -p all_wheels
+          find packages/*/dist -name "*.whl" -exec cp {} all_wheels/ \;
+          find packages/*/dist -name "*.tar.gz" -exec cp {} all_wheels/ \;
+          
+          echo "📋 Packages to publish:"
+          ls -la all_wheels/
+          
+          # Upload to PyPI
+          twine upload all_wheels/* --skip-existing --verbose
+          
+          echo "✅ Published to PyPI!"
+          echo "🎉 Check packages at: https://pypi.org/project/leann/"
+      
      - name: Create and push tag
        run: |
          git tag "v${{ inputs.version }}"
@@ -165,19 +212,4 @@ jobs:
            ### Installation
            ```bash
            pip install leann==${{ inputs.version }}
-            ```
-            
-            ### Test Installation (if using TestPyPI)
-            ```bash
-            pip install -i https://test.pypi.org/simple/ leann==${{ inputs.version }}
-            ```
-          draft: false
-          prerelease: false
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      
-      - name: Trigger PyPI publish
-        run: |
-          echo "🚀 Triggering PyPI publish workflow..."
-          # The existing build-and-publish.yml will be triggered by the tag push
-          echo "✅ Release process completed! The publish workflow will run automatically." 
+            ```
--- a/docs/RELEASE.md
+++ b/docs/RELEASE.md
@@ -7,6 +7,24 @@ Before releasing, ensure:
 2. ✅ CI has passed on the latest commit (check [Actions](https://github.com/yichuan-w/LEANN/actions/workflows/ci.yml))
 3. ✅ You have determined the new version number

+### Required: PyPI Configuration
+
+To enable PyPI publishing:
+1. Get a PyPI API token from https://pypi.org/manage/account/token/
+2. Add it to repository secrets: Settings → Secrets → Actions → New repository secret
+   - Name: `PYPI_API_TOKEN`
+   - Value: Your PyPI token (starts with `pypi-`)
+
+### Optional: TestPyPI Configuration
+
+To enable TestPyPI testing (recommended but not required):
+1. Get a TestPyPI API token from https://test.pypi.org/manage/account/token/
+2. Add it to repository secrets: Settings → Secrets → Actions → New repository secret
+   - Name: `TEST_PYPI_API_TOKEN`
+   - Value: Your TestPyPI token (starts with `pypi-`)
+
+**Note**: TestPyPI testing is optional. If not configured, the release will skip TestPyPI and proceed.
+
 ## 🚀 Recommended: Manual Release Workflow

 ### Via GitHub UI (Most Reliable)
@@ -19,12 +37,11 @@ Before releasing, ensure:
 6. Click "Run workflow"

 **What happens:**
- ✅ Validates version format
 - ✅ Downloads pre-built packages from CI (no rebuild needed!)
 - ✅ Updates all package versions
 - ✅ Optionally tests on TestPyPI
+- ✅ **Publishes directly to PyPI**
 - ✅ Creates tag and GitHub release
- ✅ Automatically triggers PyPI publish

 ### Via Command Line

@@ -44,7 +61,7 @@ This script will:
 1. Update all package versions
 2. Commit and push changes
 3. Create GitHub release
-4. CI automatically builds and publishes to PyPI
+4. **Manual Release workflow will automatically publish to PyPI**

 ⚠️ **Note**: If CI fails, you'll need to manually fix and re-tag

--- a/packages/leann-backend-diskann/pyproject.toml
+++ b/packages/leann-backend-diskann/pyproject.toml
@@ -4,8 +4,8 @@ build-backend = "scikit_build_core.build"

 [project]
 name = "leann-backend-diskann"
-version = "0.1.0"
-dependencies = ["leann-core==0.1.0", "numpy"]
+version = "0.1.2"
+dependencies = ["leann-core==0.1.2", "numpy"]

 [tool.scikit-build]
 # Key: simplified CMake path
--- a/packages/leann-backend-diskann/third_party/DiskANN
+++ b/packages/leann-backend-diskann/third_party/DiskANN
--- a/packages/leann-backend-hnsw/pyproject.toml
+++ b/packages/leann-backend-hnsw/pyproject.toml
@@ -6,10 +6,10 @@ build-backend = "scikit_build_core.build"

 [project]
 name = "leann-backend-hnsw"
-version = "0.1.0"
+version = "0.1.2"
 description = "Custom-built HNSW (Faiss) backend for the Leann toolkit."
 dependencies = [
-    "leann-core==0.1.0", 
+    "leann-core==0.1.2", 
    "numpy",
    "pyzmq>=23.0.0",
    "msgpack>=1.0.0",
--- a/packages/leann-core/pyproject.toml
+++ b/packages/leann-core/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"

 [project]
 name = "leann-core"
-version = "0.1.0"
+version = "0.1.2"
 description = "Core API and plugin system for LEANN"
 readme = "README.md"
 requires-python = ">=3.9"
--- a/packages/leann/pyproject.toml
+++ b/packages/leann/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"

 [project]
 name = "leann"
-version = "0.1.0"
+version = "0.1.2"
 description = "LEANN - The smallest vector index in the world. RAG Everything with LEANN!"
 readme = "README.md"
 requires-python = ">=3.9"
--- a/scripts/bump_version.sh
+++ b/scripts/bump_version.sh
@@ -7,12 +7,25 @@ fi

 NEW_VERSION=$1

+# Get the directory where the script is located
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+PROJECT_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )"
+
 # Update all pyproject.toml files
+echo "Updating versions in $PROJECT_ROOT/packages/"
+
 # Use different sed syntax for macOS vs Linux
 if [[ "$OSTYPE" == "darwin"* ]]; then
-    sed -i '' "s/version = \".*\"/version = \"$NEW_VERSION\"/" packages/*/pyproject.toml
+    # Update version fields
+    find "$PROJECT_ROOT/packages" -name "pyproject.toml" -exec sed -i '' "s/version = \".*\"/version = \"$NEW_VERSION\"/" {} \;
+    # Update leann-core dependencies
+    find "$PROJECT_ROOT/packages" -name "pyproject.toml" -exec sed -i '' "s/leann-core==[0-9.]*/leann-core==$NEW_VERSION/" {} \;
 else
-    sed -i "s/version = \".*\"/version = \"$NEW_VERSION\"/" packages/*/pyproject.toml
+    # Update version fields
+    find "$PROJECT_ROOT/packages" -name "pyproject.toml" -exec sed -i "s/version = \".*\"/version = \"$NEW_VERSION\"/" {} \;
+    # Update leann-core dependencies
+    find "$PROJECT_ROOT/packages" -name "pyproject.toml" -exec sed -i "s/leann-core==[0-9.]*/leann-core==$NEW_VERSION/" {} \;
 fi

-echo "Version updated to $NEW_VERSION" 
+echo "✅ Version updated to $NEW_VERSION"
+echo "✅ Dependencies updated to use leann-core==$NEW_VERSION"