fix: clean build system and Python 3.9 compatibility
Build system improvements: - Simplify macOS environment detection using brew --prefix - Remove complex hardcoded paths and CMAKE_ARGS - Let CMake automatically find Homebrew packages via CMAKE_PREFIX_PATH - Clean separation between Intel (/usr/local) and Apple Silicon (/opt/homebrew) Python 3.9 compatibility: - Set ruff target-version to py39 to match project requirements - Replace str | None with Union[str, None] in type annotations - Add Union imports where needed - Fix core interface, CLI, chat, and embedding server files 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
34
.github/workflows/build-reusable.yml
vendored
34
.github/workflows/build-reusable.yml
vendored
@@ -108,7 +108,6 @@ jobs:
|
|||||||
if: runner.os == 'macOS'
|
if: runner.os == 'macOS'
|
||||||
run: |
|
run: |
|
||||||
# Don't install LLVM, use system clang for better compatibility
|
# Don't install LLVM, use system clang for better compatibility
|
||||||
# abseil is automatically installed as a dependency of protobuf
|
|
||||||
brew install libomp boost protobuf zeromq
|
brew install libomp boost protobuf zeromq
|
||||||
|
|
||||||
- name: Install build dependencies
|
- name: Install build dependencies
|
||||||
@@ -123,20 +122,17 @@ jobs:
|
|||||||
- name: Set macOS environment variables
|
- name: Set macOS environment variables
|
||||||
if: runner.os == 'macOS'
|
if: runner.os == 'macOS'
|
||||||
run: |
|
run: |
|
||||||
# Detect Homebrew installation path and set environment variables
|
# Use brew --prefix to automatically detect Homebrew installation path
|
||||||
if [ -d "/opt/homebrew/opt/libomp" ]; then
|
HOMEBREW_PREFIX=$(brew --prefix)
|
||||||
echo "HOMEBREW_PREFIX=/opt/homebrew" >> $GITHUB_ENV
|
echo "HOMEBREW_PREFIX=${HOMEBREW_PREFIX}" >> $GITHUB_ENV
|
||||||
echo "OpenMP_ROOT=/opt/homebrew/opt/libomp" >> $GITHUB_ENV
|
echo "OpenMP_ROOT=${HOMEBREW_PREFIX}/opt/libomp" >> $GITHUB_ENV
|
||||||
echo "CMAKE_PREFIX_PATH=/opt/homebrew/opt/libomp:/opt/homebrew/opt/boost:/opt/homebrew/opt/protobuf:/opt/homebrew/opt/zeromq:/opt/homebrew/opt/abseil" >> $GITHUB_ENV
|
|
||||||
echo "LDFLAGS=-L/opt/homebrew/opt/libomp/lib" >> $GITHUB_ENV
|
# Set CMAKE_PREFIX_PATH to let CMake find all packages automatically
|
||||||
echo "CPPFLAGS=-I/opt/homebrew/opt/libomp/include -I/opt/homebrew/opt/abseil/include" >> $GITHUB_ENV
|
echo "CMAKE_PREFIX_PATH=${HOMEBREW_PREFIX}" >> $GITHUB_ENV
|
||||||
elif [ -d "/usr/local/opt/libomp" ]; then
|
|
||||||
echo "HOMEBREW_PREFIX=/usr/local" >> $GITHUB_ENV
|
# Set compiler flags for OpenMP (required for both backends)
|
||||||
echo "OpenMP_ROOT=/usr/local/opt/libomp" >> $GITHUB_ENV
|
echo "LDFLAGS=-L${HOMEBREW_PREFIX}/opt/libomp/lib" >> $GITHUB_ENV
|
||||||
echo "CMAKE_PREFIX_PATH=/usr/local/opt/libomp:/usr/local/opt/boost:/usr/local/opt/protobuf:/usr/local/opt/zeromq:/usr/local/opt/abseil" >> $GITHUB_ENV
|
echo "CPPFLAGS=-I${HOMEBREW_PREFIX}/opt/libomp/include" >> $GITHUB_ENV
|
||||||
echo "LDFLAGS=-L/usr/local/opt/libomp/lib" >> $GITHUB_ENV
|
|
||||||
echo "CPPFLAGS=-I/usr/local/opt/libomp/include -I/usr/local/opt/abseil/include" >> $GITHUB_ENV
|
|
||||||
fi
|
|
||||||
|
|
||||||
- name: Build packages
|
- name: Build packages
|
||||||
run: |
|
run: |
|
||||||
@@ -150,12 +146,10 @@ jobs:
|
|||||||
# Build HNSW backend
|
# Build HNSW backend
|
||||||
cd packages/leann-backend-hnsw
|
cd packages/leann-backend-hnsw
|
||||||
if [[ "${{ matrix.os }}" == macos-* ]]; then
|
if [[ "${{ matrix.os }}" == macos-* ]]; then
|
||||||
# Use system clang instead of homebrew LLVM for better compatibility
|
# Use system clang for better compatibility
|
||||||
export CC=clang
|
export CC=clang
|
||||||
export CXX=clang++
|
export CXX=clang++
|
||||||
export MACOSX_DEPLOYMENT_TARGET=11.0
|
export MACOSX_DEPLOYMENT_TARGET=11.0
|
||||||
# Ensure CMake can find all Homebrew packages including abseil for protobuf
|
|
||||||
export CMAKE_ARGS="-DOpenMP_ROOT=${OpenMP_ROOT} -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH} -Dabsl_DIR=${HOMEBREW_PREFIX}/opt/abseil/lib/cmake/absl"
|
|
||||||
uv build --wheel --python python
|
uv build --wheel --python python
|
||||||
else
|
else
|
||||||
uv build --wheel --python python
|
uv build --wheel --python python
|
||||||
@@ -165,13 +159,11 @@ jobs:
|
|||||||
# Build DiskANN backend
|
# Build DiskANN backend
|
||||||
cd packages/leann-backend-diskann
|
cd packages/leann-backend-diskann
|
||||||
if [[ "${{ matrix.os }}" == macos-* ]]; then
|
if [[ "${{ matrix.os }}" == macos-* ]]; then
|
||||||
# Use system clang instead of homebrew LLVM for better compatibility
|
# Use system clang for better compatibility
|
||||||
export CC=clang
|
export CC=clang
|
||||||
export CXX=clang++
|
export CXX=clang++
|
||||||
# DiskANN requires macOS 13.3+ for sgesdd_ LAPACK function
|
# DiskANN requires macOS 13.3+ for sgesdd_ LAPACK function
|
||||||
export MACOSX_DEPLOYMENT_TARGET=13.3
|
export MACOSX_DEPLOYMENT_TARGET=13.3
|
||||||
# Ensure CMake can find all Homebrew packages including abseil for protobuf
|
|
||||||
export CMAKE_ARGS="-DOpenMP_ROOT=${OpenMP_ROOT} -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH} -Dabsl_DIR=${HOMEBREW_PREFIX}/opt/abseil/lib/cmake/absl"
|
|
||||||
uv build --wheel --python python
|
uv build --wheel --python python
|
||||||
else
|
else
|
||||||
uv build --wheel --python python
|
uv build --wheel --python python
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import sys
|
|||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import Union
|
||||||
|
|
||||||
import msgpack
|
import msgpack
|
||||||
import numpy as np
|
import numpy as np
|
||||||
@@ -33,7 +34,7 @@ if not logger.handlers:
|
|||||||
|
|
||||||
|
|
||||||
def create_hnsw_embedding_server(
|
def create_hnsw_embedding_server(
|
||||||
passages_file: str | None = None,
|
passages_file: Union[str, None] = None,
|
||||||
zmq_port: int = 5555,
|
zmq_port: int = 5555,
|
||||||
model_name: str = "sentence-transformers/all-mpnet-base-v2",
|
model_name: str = "sentence-transformers/all-mpnet-base-v2",
|
||||||
distance_metric: str = "mips",
|
distance_metric: str = "mips",
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import difflib
|
|||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from typing import Any
|
from typing import Any, Union
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
@@ -309,7 +309,7 @@ def search_hf_models(query: str, limit: int = 10) -> list[str]:
|
|||||||
return search_hf_models_fuzzy(query, limit)
|
return search_hf_models_fuzzy(query, limit)
|
||||||
|
|
||||||
|
|
||||||
def validate_model_and_suggest(model_name: str, llm_type: str) -> str | None:
|
def validate_model_and_suggest(model_name: str, llm_type: str) -> Union[str, None]:
|
||||||
"""Validate model name and provide suggestions if invalid"""
|
"""Validate model name and provide suggestions if invalid"""
|
||||||
if llm_type == "ollama":
|
if llm_type == "ollama":
|
||||||
available_models = check_ollama_models()
|
available_models = check_ollama_models()
|
||||||
@@ -683,7 +683,7 @@ class HFChat(LLMInterface):
|
|||||||
class OpenAIChat(LLMInterface):
|
class OpenAIChat(LLMInterface):
|
||||||
"""LLM interface for OpenAI models."""
|
"""LLM interface for OpenAI models."""
|
||||||
|
|
||||||
def __init__(self, model: str = "gpt-4o", api_key: str | None = None):
|
def __init__(self, model: str = "gpt-4o", api_key: Union[str, None] = None):
|
||||||
self.model = model
|
self.model = model
|
||||||
self.api_key = api_key or os.getenv("OPENAI_API_KEY")
|
self.api_key = api_key or os.getenv("OPENAI_API_KEY")
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import argparse
|
import argparse
|
||||||
import asyncio
|
import asyncio
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import Union
|
||||||
|
|
||||||
from llama_index.core import SimpleDirectoryReader
|
from llama_index.core import SimpleDirectoryReader
|
||||||
from llama_index.core.node_parser import SentenceSplitter
|
from llama_index.core.node_parser import SentenceSplitter
|
||||||
@@ -270,7 +271,7 @@ Examples:
|
|||||||
print(f' leann search {example_name} "your query"')
|
print(f' leann search {example_name} "your query"')
|
||||||
print(f" leann ask {example_name} --interactive")
|
print(f" leann ask {example_name} --interactive")
|
||||||
|
|
||||||
def load_documents(self, docs_dir: str, custom_file_types: str | None = None):
|
def load_documents(self, docs_dir: str, custom_file_types: Union[str, None] = None):
|
||||||
print(f"Loading documents from {docs_dir}...")
|
print(f"Loading documents from {docs_dir}...")
|
||||||
if custom_file_types:
|
if custom_file_types:
|
||||||
print(f"Using custom file types: {custom_file_types}")
|
print(f"Using custom file types: {custom_file_types}")
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from typing import Any, Literal
|
from typing import Any, Literal, Union
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
|
||||||
@@ -34,7 +34,9 @@ class LeannBackendSearcherInterface(ABC):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def _ensure_server_running(self, passages_source_file: str, port: int | None, **kwargs) -> int:
|
def _ensure_server_running(
|
||||||
|
self, passages_source_file: str, port: Union[int, None], **kwargs
|
||||||
|
) -> int:
|
||||||
"""Ensure server is running"""
|
"""Ensure server is running"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@@ -48,7 +50,7 @@ class LeannBackendSearcherInterface(ABC):
|
|||||||
prune_ratio: float = 0.0,
|
prune_ratio: float = 0.0,
|
||||||
recompute_embeddings: bool = False,
|
recompute_embeddings: bool = False,
|
||||||
pruning_strategy: Literal["global", "local", "proportional"] = "global",
|
pruning_strategy: Literal["global", "local", "proportional"] = "global",
|
||||||
zmq_port: int | None = None,
|
zmq_port: Union[int, None] = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
"""Search for nearest neighbors
|
"""Search for nearest neighbors
|
||||||
@@ -74,7 +76,7 @@ class LeannBackendSearcherInterface(ABC):
|
|||||||
self,
|
self,
|
||||||
query: str,
|
query: str,
|
||||||
use_server_if_available: bool = True,
|
use_server_if_available: bool = True,
|
||||||
zmq_port: int | None = None,
|
zmq_port: Union[int, None] = None,
|
||||||
) -> np.ndarray:
|
) -> np.ndarray:
|
||||||
"""Compute embedding for a query string
|
"""Compute embedding for a query string
|
||||||
|
|
||||||
|
|||||||
@@ -88,7 +88,7 @@ leann-backend-diskann = { path = "packages/leann-backend-diskann", editable = tr
|
|||||||
leann-backend-hnsw = { path = "packages/leann-backend-hnsw", editable = true }
|
leann-backend-hnsw = { path = "packages/leann-backend-hnsw", editable = true }
|
||||||
|
|
||||||
[tool.ruff]
|
[tool.ruff]
|
||||||
target-version = "py310"
|
target-version = "py39"
|
||||||
line-length = 100
|
line-length = 100
|
||||||
extend-exclude = [
|
extend-exclude = [
|
||||||
"third_party",
|
"third_party",
|
||||||
|
|||||||
Reference in New Issue
Block a user