Compare commits

...

11 Commits

Author SHA1 Message Date
GitHub Actions
52584a171f chore: release v0.1.10 2025-07-25 23:12:16 +00:00
Andy Lee
efd6b5324b fix: add protobuf as a dependency for DiskANN backend
- Fixes 'No module named google' error when starting DiskANN embedding server
- Prevents users from having to manually install protobuf
2025-07-25 16:10:25 -07:00
Andy Lee
2baaa4549b fix: handle relative paths in HNSW embedding server metadata
- Convert relative paths to absolute paths based on metadata file location
- Fixes FileNotFoundError when starting embedding server
- Resolves issue with passages file not found in different working directories
2025-07-25 16:09:53 -07:00
Andy Lee
35310ddd52 fix: pure Python packages not building due to ubuntu-latest check
The build workflow was checking for matrix.os == 'ubuntu-latest',
but we changed the matrix to use 'ubuntu-22.04', causing the
pure Python packages (leann-core and leann) to never be built.

Changed to use pattern matching [[ == ubuntu-* ]] to match any
Ubuntu version.

This explains why v0.1.9 only published the C++ backend packages
but not the pure Python packages.
2025-07-25 15:14:21 -07:00
Andy Lee
fc9c5cb39d fix: make release workflow idempotent
- Check if version is already updated before trying to update
- Check if tag already exists before creating
- Check if GitHub release already exists before creating
- This allows re-running the workflow after partial failures

Previously, if the workflow failed after updating version but before
completing the release, it couldn't be re-run with the same version.
2025-07-25 14:47:35 -07:00
Andy Lee
8f2a1e87ea Merge pull request #7 from yichuan-w/fix/simple-ubuntu22-build
fix: simplify build system for Colab compatibility
2025-07-25 14:08:37 -07:00
Andy Lee
50caf65f28 fix: change ubuntu-latest to ubuntu-22.04 and add Python 3.13
- Explicitly use ubuntu-22.04 instead of ubuntu-latest
- Add Python 3.13 to the build matrix
- This ensures we build on the same OS version as Google Colab
2025-07-25 13:48:59 -07:00
Andy Lee
1b48794ca8 cleanup: remove cibuildwheel workflow files
- Remove ci-cibuildwheel.yml and build-cibuildwheel.yml
- These files were not present in v0.1.5
- Keep only the simple build system
2025-07-25 13:48:08 -07:00
Andy Lee
4aef1d814e revert: simplify build system by removing manylinux/cibuildwheel
- Revert to simple Ubuntu 22.04 builds that should work with Colab
- Remove all manylinux container complexity
- Colab runs on Ubuntu 22.04, so direct builds should be compatible
- Restore build-reusable.yml to v0.1.5 version
- Remove cibuildwheel option from release workflow

This should fix the overcomplicated build issues while maintaining
Colab compatibility through direct Ubuntu 22.04 builds.
2025-07-25 13:46:51 -07:00
GitHub Actions
75ddcd6158 chore: release v0.1.9 2025-07-25 20:04:42 +00:00
Andy Lee
2a4df11f5c fix: absolute path for passages 2025-07-25 11:59:30 -07:00
10 changed files with 89 additions and 367 deletions

View File

@@ -1,144 +0,0 @@
name: Build with cibuildwheel
on:
workflow_call:
inputs:
ref:
description: 'Git ref to build'
required: false
type: string
default: ''
jobs:
build_wheels:
name: Build wheels on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest]
steps:
- uses: actions/checkout@v4
with:
ref: ${{ inputs.ref }}
submodules: recursive
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: '3.11' # Version for building pure Python packages
# Build each package separately in our monorepo
- name: Build pure Python packages (leann-core, leann)
if: matrix.os == 'ubuntu-latest' # Only build once, they're platform-independent
run: |
# Install build tools
python -m pip install --upgrade pip build
# Build pure Python packages
python -m build packages/leann-core --outdir wheelhouse/
python -m build packages/leann --outdir wheelhouse/
- name: Build leann-backend-hnsw wheels
uses: pypa/cibuildwheel@v2.16.2
with:
package-dir: packages/leann-backend-hnsw
output-dir: wheelhouse
env:
CIBW_BUILD: cp39-* cp310-* cp311-* cp312-*
CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
CIBW_SKIP: "*-win32 *-manylinux_i686 pp*"
CIBW_BEFORE_ALL_LINUX: |
yum clean all && yum makecache
yum install -y epel-release || true
yum makecache || true
# Install system dependencies
yum install -y \
gcc-c++ \
boost-devel \
protobuf-compiler \
protobuf-devel \
zeromq-devel \
pkgconfig \
openblas-devel \
cmake || echo "Some packages failed, continuing..."
# Verify zmq installation and create pkg-config file if needed
if [ ! -f /usr/lib64/pkgconfig/libzmq.pc ] && [ ! -f /usr/lib/pkgconfig/libzmq.pc ]; then
echo "Creating libzmq.pc file..."
mkdir -p /usr/lib64/pkgconfig
cat > /usr/lib64/pkgconfig/libzmq.pc << 'EOF'
prefix=/usr
exec_prefix=${prefix}
libdir=${exec_prefix}/lib64
includedir=${prefix}/include
Name: libzmq
Description: ZeroMQ library
Version: 4.1.4
Libs: -L${libdir} -lzmq
Cflags: -I${includedir}
EOF
fi
CIBW_BEFORE_ALL_MACOS: |
brew install llvm libomp boost protobuf zeromq
CIBW_ENVIRONMENT_LINUX: |
PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH
CIBW_ENVIRONMENT_MACOS: |
CC=$(brew --prefix llvm)/bin/clang
CXX=$(brew --prefix llvm)/bin/clang++
CIBW_TEST_REQUIRES: leann-core numpy pyzmq msgpack
CIBW_TEST_COMMAND: python -c "import leann_backend_hnsw"
- name: Build leann-backend-diskann wheels
uses: pypa/cibuildwheel@v2.16.2
with:
package-dir: packages/leann-backend-diskann
output-dir: wheelhouse
env:
CIBW_BUILD: cp39-* cp310-* cp311-* cp312-*
CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
CIBW_SKIP: "*-win32 *-manylinux_i686 pp*"
CIBW_BEFORE_ALL_LINUX: |
yum clean all && yum makecache
yum install -y epel-release || true
yum makecache || true
# Install system dependencies for DiskANN
yum install -y \
gcc-c++ \
protobuf-compiler \
protobuf-devel \
openblas-devel \
pkgconfig \
cmake || echo "Some packages failed, continuing..."
yum install -y libaio-devel || echo "libaio-devel not available, continuing..."
CIBW_BEFORE_ALL_MACOS: |
brew install llvm libomp protobuf
CIBW_ENVIRONMENT_LINUX: |
PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH
CIBW_ENVIRONMENT_MACOS: |
CC=$(brew --prefix llvm)/bin/clang
CXX=$(brew --prefix llvm)/bin/clang++
CIBW_TEST_REQUIRES: leann-core numpy
CIBW_TEST_COMMAND: python -c "import leann_backend_diskann"
- name: List built packages
run: |
echo "📦 Built packages:"
ls -la wheelhouse/
- uses: actions/upload-artifact@v4
with:
name: cibw-wheels-${{ matrix.os }}
path: ./wheelhouse/*.whl

View File

@@ -13,107 +13,46 @@ jobs:
build:
name: Build ${{ matrix.os }} Python ${{ matrix.python }}
strategy:
fail-fast: false
matrix:
include:
- os: ubuntu-latest
- os: ubuntu-22.04
python: '3.9'
container: 'quay.io/pypa/manylinux2014_x86_64'
- os: ubuntu-latest
- os: ubuntu-22.04
python: '3.10'
container: 'quay.io/pypa/manylinux2014_x86_64'
- os: ubuntu-latest
- os: ubuntu-22.04
python: '3.11'
container: 'quay.io/pypa/manylinux2014_x86_64'
- os: ubuntu-latest
- os: ubuntu-22.04
python: '3.12'
container: 'quay.io/pypa/manylinux2014_x86_64'
- os: ubuntu-latest
- os: ubuntu-22.04
python: '3.13'
container: 'quay.io/pypa/manylinux2014_x86_64'
- os: macos-latest
python: '3.9'
container: ''
- os: macos-latest
python: '3.10'
container: ''
- os: macos-latest
python: '3.11'
container: ''
- os: macos-latest
python: '3.12'
container: ''
- os: macos-latest
python: '3.13'
container: ''
runs-on: ${{ matrix.os }}
container: ${{ matrix.container }}
steps:
# For manylinux2014 compatibility, we'll handle checkout differently
- uses: actions/checkout@v4
if: matrix.container == ''
with:
ref: ${{ inputs.ref }}
submodules: recursive
# Manual checkout for containers to avoid Node.js compatibility issues
- name: Manual checkout in container
if: matrix.container != ''
run: |
# Install git if not available
yum install -y git || true
# Configure git to handle the directory ownership issue
git config --global --add safe.directory ${GITHUB_WORKSPACE}
git config --global --add safe.directory /__w/LEANN/LEANN
git config --global --add safe.directory /github/workspace
git config --global --add safe.directory $(pwd)
# Clone the repository manually in the container
git init
git remote add origin https://github.com/${GITHUB_REPOSITORY}.git
# Fetch the appropriate ref
if [ -n "${{ inputs.ref }}" ]; then
git fetch --depth=1 origin ${{ inputs.ref }}
else
git fetch --depth=1 origin ${GITHUB_SHA}
fi
git checkout FETCH_HEAD
# Initialize submodules
git submodule update --init --recursive
- name: Setup Python (macOS and regular Ubuntu)
if: matrix.container == ''
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: Setup Python (manylinux container)
if: matrix.container != ''
run: |
# Use the pre-installed Python version in manylinux container
# Convert Python version format (3.9 -> 39, 3.10 -> 310, etc.)
PY_VER=$(echo "${{ matrix.python }}" | sed 's/\.//g')
/opt/python/cp${PY_VER}-*/bin/python -m pip install --upgrade pip
# Create symlinks for convenience
ln -sf /opt/python/cp${PY_VER}-*/bin/python /usr/local/bin/python
ln -sf /opt/python/cp${PY_VER}-*/bin/pip /usr/local/bin/pip
- name: Install uv (macOS and regular Ubuntu)
if: matrix.container == ''
- name: Install uv
uses: astral-sh/setup-uv@v4
- name: Install uv (manylinux container)
if: matrix.container != ''
run: |
curl -LsSf https://astral.sh/uv/install.sh | sh
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
- name: Install system dependencies (Ubuntu - regular)
if: runner.os == 'Linux' && matrix.container == ''
- name: Install system dependencies (Ubuntu)
if: runner.os == 'Linux'
run: |
sudo apt-get update
sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \
@@ -126,64 +65,6 @@ jobs:
echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH" >> $GITHUB_ENV
- name: Install system dependencies (manylinux container)
if: runner.os == 'Linux' && matrix.container != ''
run: |
# manylinux2014 uses yum instead of apt
# Update yum cache first
yum clean all
yum makecache
# Install EPEL repository
yum install -y epel-release || true
# Update cache again after EPEL
yum makecache || true
# Install development packages
# Note: Some packages might have different names in CentOS 7
yum install -y \
gcc-c++ \
boost-devel \
protobuf-compiler \
protobuf-devel \
zeromq-devel \
pkgconfig \
openblas-devel \
cmake || {
echo "Some packages failed to install, trying alternatives..."
# Try alternative package names
yum install -y libzmq3-devel || true
yum install -y libzmq-devel || true
}
# Install optional packages that might not be available
yum install -y libaio-devel || echo "libaio-devel not available, continuing..."
# Verify zmq installation and create pkg-config file if needed
if [ ! -f /usr/lib64/pkgconfig/libzmq.pc ] && [ ! -f /usr/lib/pkgconfig/libzmq.pc ]; then
echo "Creating libzmq.pc file..."
mkdir -p /usr/lib64/pkgconfig
cat > /usr/lib64/pkgconfig/libzmq.pc << 'EOF'
prefix=/usr
exec_prefix=${prefix}
libdir=${exec_prefix}/lib64
includedir=${prefix}/include
Name: libzmq
Description: ZeroMQ library
Version: 4.1.4
Libs: -L${libdir} -lzmq
Cflags: -I${includedir}
EOF
fi
# Update PKG_CONFIG_PATH
echo "PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH" >> $GITHUB_ENV
# Build tools are pre-installed in manylinux
# MKL is more complex in container, skip for now and use OpenBLAS
- name: Install system dependencies (macOS)
if: runner.os == 'macOS'
run: |
@@ -191,65 +72,44 @@ jobs:
- name: Install build dependencies
run: |
if [[ -n "${{ matrix.container }}" ]]; then
# In manylinux container, use regular pip
pip install scikit-build-core numpy swig Cython pybind11 auditwheel
uv pip install --system scikit-build-core numpy swig Cython pybind11
if [[ "$RUNNER_OS" == "Linux" ]]; then
uv pip install --system auditwheel
else
# Regular environment, use uv
uv pip install --system scikit-build-core numpy swig Cython pybind11
if [[ "$RUNNER_OS" == "Linux" ]]; then
uv pip install --system auditwheel
else
uv pip install --system delocate
fi
uv pip install --system delocate
fi
- name: Build packages
run: |
# Choose build command based on environment
if [[ -n "${{ matrix.container }}" ]]; then
BUILD_CMD="pip wheel . --no-deps -w dist"
else
BUILD_CMD="uv build --wheel --python python"
fi
# Build core (platform independent)
if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
if [[ "${{ matrix.os }}" == ubuntu-* ]]; then
cd packages/leann-core
if [[ -n "${{ matrix.container }}" ]]; then
pip wheel . --no-deps -w dist
else
uv build
fi
uv build
cd ../..
fi
# Build HNSW backend
cd packages/leann-backend-hnsw
if [ "${{ matrix.os }}" == "macos-latest" ]; then
CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ $BUILD_CMD
CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build --wheel --python python
else
eval $BUILD_CMD
uv build --wheel --python python
fi
cd ../..
# Build DiskANN backend
cd packages/leann-backend-diskann
if [ "${{ matrix.os }}" == "macos-latest" ]; then
CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ $BUILD_CMD
CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build --wheel --python python
else
eval $BUILD_CMD
uv build --wheel --python python
fi
cd ../..
# Build meta package (platform independent)
if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
if [[ "${{ matrix.os }}" == ubuntu-* ]]; then
cd packages/leann
if [[ -n "${{ matrix.container }}" ]]; then
pip wheel . --no-deps -w dist
else
uv build
fi
uv build
cd ../..
fi
@@ -259,9 +119,6 @@ jobs:
# Repair HNSW wheel
cd packages/leann-backend-hnsw
if [ -d dist ]; then
# Show what platform auditwheel will use
auditwheel show dist/*.whl || true
# Let auditwheel auto-detect the appropriate manylinux tag
auditwheel repair dist/*.whl -w dist_repaired
rm -rf dist
mv dist_repaired dist
@@ -271,9 +128,6 @@ jobs:
# Repair DiskANN wheel
cd packages/leann-backend-diskann
if [ -d dist ]; then
# Show what platform auditwheel will use
auditwheel show dist/*.whl || true
# Let auditwheel auto-detect the appropriate manylinux tag
auditwheel repair dist/*.whl -w dist_repaired
rm -rf dist
mv dist_repaired dist
@@ -309,5 +163,5 @@ jobs:
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: packages-${{ matrix.os }}-py${{ matrix.python }}${{ matrix.container && '-manylinux' || '' }}
name: packages-${{ matrix.os }}-py${{ matrix.python }}
path: packages/*/dist/

View File

@@ -1,12 +0,0 @@
name: CI - cibuildwheel (Test)
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
workflow_dispatch: # Allow manual triggering
jobs:
build:
uses: ./.github/workflows/build-cibuildwheel.yml

View File

@@ -7,11 +7,6 @@ on:
description: 'Version to release (e.g., 0.1.2)'
required: true
type: string
use_cibuildwheel:
description: 'Use cibuildwheel for better compatibility (recommended for Colab)'
required: false
type: boolean
default: false
jobs:
update-version:
@@ -36,37 +31,38 @@ jobs:
- name: Update versions and push
id: push
run: |
./scripts/bump_version.sh ${{ inputs.version }}
git config user.name "GitHub Actions"
git config user.email "actions@github.com"
git add packages/*/pyproject.toml
git commit -m "chore: release v${{ inputs.version }}"
git push origin main
# Check current version
CURRENT_VERSION=$(grep "^version" packages/leann-core/pyproject.toml | cut -d'"' -f2)
echo "Current version: $CURRENT_VERSION"
echo "Target version: ${{ inputs.version }}"
if [ "$CURRENT_VERSION" = "${{ inputs.version }}" ]; then
echo "⚠️ Version is already ${{ inputs.version }}, skipping update"
COMMIT_SHA=$(git rev-parse HEAD)
else
./scripts/bump_version.sh ${{ inputs.version }}
git config user.name "GitHub Actions"
git config user.email "actions@github.com"
git add packages/*/pyproject.toml
git commit -m "chore: release v${{ inputs.version }}"
git push origin main
COMMIT_SHA=$(git rev-parse HEAD)
echo "✅ Pushed version update: $COMMIT_SHA"
fi
COMMIT_SHA=$(git rev-parse HEAD)
echo "commit-sha=$COMMIT_SHA" >> $GITHUB_OUTPUT
echo "✅ Pushed version update: $COMMIT_SHA"
build-packages-reusable:
name: Build packages (Standard)
build-packages:
name: Build packages
needs: update-version
if: ${{ !inputs.use_cibuildwheel }}
uses: ./.github/workflows/build-reusable.yml
with:
ref: ${{ needs.update-version.outputs.commit-sha }}
build-packages-cibuildwheel:
name: Build packages (cibuildwheel)
needs: update-version
if: ${{ inputs.use_cibuildwheel }}
uses: ./.github/workflows/build-cibuildwheel.yml
with:
ref: ${{ needs.update-version.outputs.commit-sha }}
publish:
name: Publish and Release
needs: [update-version, build-packages-reusable, build-packages-cibuildwheel]
if: always() && needs.update-version.result == 'success' && (needs.build-packages-reusable.result == 'success' || needs.build-packages-cibuildwheel.result == 'success')
needs: [update-version, build-packages]
if: always() && needs.update-version.result == 'success' && needs.build-packages.result == 'success'
runs-on: ubuntu-latest
permissions:
contents: write
@@ -107,12 +103,24 @@ jobs:
- name: Create release
run: |
git tag "v${{ inputs.version }}"
git push origin "v${{ inputs.version }}"
# Check if tag already exists
if git rev-parse "v${{ inputs.version }}" >/dev/null 2>&1; then
echo "⚠️ Tag v${{ inputs.version }} already exists, skipping tag creation"
else
git tag "v${{ inputs.version }}"
git push origin "v${{ inputs.version }}"
echo "✅ Created and pushed tag v${{ inputs.version }}"
fi
gh release create "v${{ inputs.version }}" \
--title "Release v${{ inputs.version }}" \
--notes "🚀 Released to PyPI: https://pypi.org/project/leann/${{ inputs.version }}/" \
--latest
# Check if release already exists
if gh release view "v${{ inputs.version }}" >/dev/null 2>&1; then
echo "⚠️ Release v${{ inputs.version }} already exists, skipping release creation"
else
gh release create "v${{ inputs.version }}" \
--title "Release v${{ inputs.version }}" \
--notes "🚀 Released to PyPI: https://pypi.org/project/leann/${{ inputs.version }}/" \
--latest
echo "✅ Created GitHub release v${{ inputs.version }}"
fi
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

View File

@@ -4,8 +4,8 @@ build-backend = "scikit_build_core.build"
[project]
name = "leann-backend-diskann"
version = "0.1.8"
dependencies = ["leann-core==0.1.8", "numpy"]
version = "0.1.10"
dependencies = ["leann-core==0.1.10", "numpy", "protobuf>=3.19.0"]
[tool.scikit-build]
# Key: simplified CMake path

View File

@@ -81,7 +81,21 @@ def create_hnsw_embedding_server(
with open(passages_file, "r") as f:
meta = json.load(f)
passages = PassageManager(meta["passage_sources"])
# Convert relative paths to absolute paths based on metadata file location
metadata_dir = Path(
passages_file
).parent.parent # Go up one level from the metadata file
passage_sources = []
for source in meta["passage_sources"]:
source_copy = source.copy()
# Convert relative paths to absolute paths
if not Path(source_copy["path"]).is_absolute():
source_copy["path"] = str(metadata_dir / source_copy["path"])
if not Path(source_copy["index_path"]).is_absolute():
source_copy["index_path"] = str(metadata_dir / source_copy["index_path"])
passage_sources.append(source_copy)
passages = PassageManager(passage_sources)
logger.info(
f"Loaded PassageManager with {len(passages.global_offset_map)} passages from metadata"
)
@@ -270,15 +284,15 @@ def create_hnsw_embedding_server(
if __name__ == "__main__":
import signal
import sys
def signal_handler(sig, frame):
logger.info(f"Received signal {sig}, shutting down gracefully...")
sys.exit(0)
# Register signal handlers for graceful shutdown
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGINT, signal_handler)
parser = argparse.ArgumentParser(description="HNSW Embedding service")
parser.add_argument("--zmq-port", type=int, default=5555, help="ZMQ port to run on")
parser.add_argument(

View File

@@ -6,10 +6,10 @@ build-backend = "scikit_build_core.build"
[project]
name = "leann-backend-hnsw"
version = "0.1.8"
version = "0.1.10"
description = "Custom-built HNSW (Faiss) backend for the Leann toolkit."
dependencies = [
"leann-core==0.1.8",
"leann-core==0.1.10",
"numpy",
"pyzmq>=23.0.0",
"msgpack>=1.0.0",

View File

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "leann-core"
version = "0.1.8"
version = "0.1.10"
description = "Core API and plugin system for LEANN"
readme = "README.md"
requires-python = ">=3.9"

View File

@@ -269,7 +269,9 @@ class EmbeddingServerManager:
]
if kwargs.get("passages_file"):
command.extend(["--passages-file", str(kwargs["passages_file"])])
# Convert to absolute path to ensure subprocess can find the file
passages_file = Path(kwargs["passages_file"]).resolve()
command.extend(["--passages-file", str(passages_file)])
if embedding_mode != "sentence-transformers":
command.extend(["--embedding-mode", embedding_mode])

View File

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "leann"
version = "0.1.8"
version = "0.1.10"
description = "LEANN - The smallest vector index in the world. RAG Everything with LEANN!"
readme = "README.md"
requires-python = ">=3.9"