Compare commits
22 Commits
v0.3.3
...
fix-arm64-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2159a29073 | ||
|
|
185bd38112 | ||
|
|
936fa525de | ||
|
|
f6c83898b8 | ||
|
|
40cb39ed8a | ||
|
|
b74718332e | ||
|
|
2c0e4ec58d | ||
|
|
9836ce049d | ||
|
|
478b10c7c1 | ||
|
|
1fce9ad445 | ||
|
|
d452b1ffa3 | ||
|
|
e0085da8ba | ||
|
|
377c952134 | ||
|
|
0ff18a7d79 | ||
|
|
08f9757c45 | ||
|
|
c5c8a57441 | ||
|
|
a5ef3e66d0 | ||
|
|
5079a8b799 | ||
|
|
07f8129f65 | ||
|
|
45ef563bda | ||
|
|
e9d447ac2a | ||
|
|
141e498329 |
@@ -4,8 +4,8 @@ build-backend = "scikit_build_core.build"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "leann-backend-diskann"
|
name = "leann-backend-diskann"
|
||||||
version = "0.3.3"
|
version = "0.3.2"
|
||||||
dependencies = ["leann-core==0.3.3", "numpy", "protobuf>=3.19.0"]
|
dependencies = ["leann-core==0.3.2", "numpy", "protobuf>=3.19.0"]
|
||||||
|
|
||||||
[tool.scikit-build]
|
[tool.scikit-build]
|
||||||
# Key: simplified CMake path
|
# Key: simplified CMake path
|
||||||
|
|||||||
@@ -6,10 +6,10 @@ build-backend = "scikit_build_core.build"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "leann-backend-hnsw"
|
name = "leann-backend-hnsw"
|
||||||
version = "0.3.3"
|
version = "0.3.2"
|
||||||
description = "Custom-built HNSW (Faiss) backend for the Leann toolkit."
|
description = "Custom-built HNSW (Faiss) backend for the Leann toolkit."
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"leann-core==0.3.3",
|
"leann-core==0.3.2",
|
||||||
"numpy",
|
"numpy",
|
||||||
"pyzmq>=23.0.0",
|
"pyzmq>=23.0.0",
|
||||||
"msgpack>=1.0.0",
|
"msgpack>=1.0.0",
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "leann-core"
|
name = "leann-core"
|
||||||
version = "0.3.3"
|
version = "0.3.2"
|
||||||
description = "Core API and plugin system for LEANN"
|
description = "Core API and plugin system for LEANN"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.9"
|
requires-python = ">=3.9"
|
||||||
|
|||||||
@@ -322,17 +322,9 @@ Examples:
|
|||||||
|
|
||||||
return basic_matches
|
return basic_matches
|
||||||
|
|
||||||
def _should_exclude_file(self, file_path: Path, gitignore_matches) -> bool:
|
def _should_exclude_file(self, relative_path: Path, gitignore_matches) -> bool:
|
||||||
"""Check if a file should be excluded using gitignore parser.
|
"""Check if a file should be excluded using gitignore parser."""
|
||||||
|
return gitignore_matches(str(relative_path))
|
||||||
Always match against absolute, posix-style paths for consistency with
|
|
||||||
gitignore_parser expectations.
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
absolute_path = file_path.resolve()
|
|
||||||
except Exception:
|
|
||||||
absolute_path = Path(str(file_path))
|
|
||||||
return gitignore_matches(absolute_path.as_posix())
|
|
||||||
|
|
||||||
def _is_git_submodule(self, path: Path) -> bool:
|
def _is_git_submodule(self, path: Path) -> bool:
|
||||||
"""Check if a path is a git submodule."""
|
"""Check if a path is a git submodule."""
|
||||||
@@ -404,9 +396,7 @@ Examples:
|
|||||||
print(f" {current_path}")
|
print(f" {current_path}")
|
||||||
print(" " + "─" * 45)
|
print(" " + "─" * 45)
|
||||||
|
|
||||||
current_indexes = self._discover_indexes_in_project(
|
current_indexes = self._discover_indexes_in_project(current_path)
|
||||||
current_path, exclude_dirs=other_projects
|
|
||||||
)
|
|
||||||
if current_indexes:
|
if current_indexes:
|
||||||
for idx in current_indexes:
|
for idx in current_indexes:
|
||||||
total_indexes += 1
|
total_indexes += 1
|
||||||
@@ -445,14 +435,9 @@ Examples:
|
|||||||
print(" leann build my-docs --docs ./documents")
|
print(" leann build my-docs --docs ./documents")
|
||||||
else:
|
else:
|
||||||
# Count only projects that have at least one discoverable index
|
# Count only projects that have at least one discoverable index
|
||||||
projects_count = 0
|
projects_count = sum(
|
||||||
for p in valid_projects:
|
1 for p in valid_projects if len(self._discover_indexes_in_project(p)) > 0
|
||||||
if p == current_path:
|
)
|
||||||
discovered = self._discover_indexes_in_project(p, exclude_dirs=other_projects)
|
|
||||||
else:
|
|
||||||
discovered = self._discover_indexes_in_project(p)
|
|
||||||
if len(discovered) > 0:
|
|
||||||
projects_count += 1
|
|
||||||
print(f"📊 Total: {total_indexes} indexes across {projects_count} projects")
|
print(f"📊 Total: {total_indexes} indexes across {projects_count} projects")
|
||||||
|
|
||||||
if current_indexes_count > 0:
|
if current_indexes_count > 0:
|
||||||
@@ -469,22 +454,9 @@ Examples:
|
|||||||
print("\n💡 Create your first index:")
|
print("\n💡 Create your first index:")
|
||||||
print(" leann build my-docs --docs ./documents")
|
print(" leann build my-docs --docs ./documents")
|
||||||
|
|
||||||
def _discover_indexes_in_project(
|
def _discover_indexes_in_project(self, project_path: Path):
|
||||||
self, project_path: Path, exclude_dirs: Optional[list[Path]] = None
|
"""Discover all indexes in a project directory (both CLI and apps formats)"""
|
||||||
):
|
|
||||||
"""Discover all indexes in a project directory (both CLI and apps formats)
|
|
||||||
|
|
||||||
exclude_dirs: when provided, skip any APP-format index files that are
|
|
||||||
located under these directories. This prevents duplicates when the
|
|
||||||
current project is a parent directory of other registered projects.
|
|
||||||
"""
|
|
||||||
indexes = []
|
indexes = []
|
||||||
exclude_dirs = exclude_dirs or []
|
|
||||||
# normalize to resolved paths once for comparison
|
|
||||||
try:
|
|
||||||
exclude_dirs_resolved = [p.resolve() for p in exclude_dirs]
|
|
||||||
except Exception:
|
|
||||||
exclude_dirs_resolved = exclude_dirs
|
|
||||||
|
|
||||||
# 1. CLI format: .leann/indexes/index_name/
|
# 1. CLI format: .leann/indexes/index_name/
|
||||||
cli_indexes_dir = project_path / ".leann" / "indexes"
|
cli_indexes_dir = project_path / ".leann" / "indexes"
|
||||||
@@ -523,17 +495,6 @@ Examples:
|
|||||||
continue
|
continue
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
# Skip meta files that live under excluded directories
|
|
||||||
try:
|
|
||||||
meta_parent_resolved = meta_file.parent.resolve()
|
|
||||||
if any(
|
|
||||||
meta_parent_resolved.is_relative_to(ex_dir)
|
|
||||||
for ex_dir in exclude_dirs_resolved
|
|
||||||
):
|
|
||||||
continue
|
|
||||||
except Exception:
|
|
||||||
# best effort; if resolve or comparison fails, do not exclude
|
|
||||||
pass
|
|
||||||
# Use the parent directory name as the app index display name
|
# Use the parent directory name as the app index display name
|
||||||
display_name = meta_file.parent.name
|
display_name = meta_file.parent.name
|
||||||
# Extract file base used to store files
|
# Extract file base used to store files
|
||||||
@@ -1061,8 +1022,7 @@ Examples:
|
|||||||
|
|
||||||
# Try to use better PDF parsers first, but only if PDFs are requested
|
# Try to use better PDF parsers first, but only if PDFs are requested
|
||||||
documents = []
|
documents = []
|
||||||
# Use resolved absolute paths to avoid mismatches (symlinks, relative vs absolute)
|
docs_path = Path(docs_dir)
|
||||||
docs_path = Path(docs_dir).resolve()
|
|
||||||
|
|
||||||
# Check if we should process PDFs
|
# Check if we should process PDFs
|
||||||
should_process_pdfs = custom_file_types is None or ".pdf" in custom_file_types
|
should_process_pdfs = custom_file_types is None or ".pdf" in custom_file_types
|
||||||
@@ -1071,15 +1031,10 @@ Examples:
|
|||||||
for file_path in docs_path.rglob("*.pdf"):
|
for file_path in docs_path.rglob("*.pdf"):
|
||||||
# Check if file matches any exclude pattern
|
# Check if file matches any exclude pattern
|
||||||
try:
|
try:
|
||||||
# Ensure both paths are resolved before computing relativity
|
|
||||||
file_path_resolved = file_path.resolve()
|
|
||||||
# Determine directory scope using the non-resolved path to avoid
|
|
||||||
# misclassifying symlinked entries as outside the docs directory
|
|
||||||
relative_path = file_path.relative_to(docs_path)
|
relative_path = file_path.relative_to(docs_path)
|
||||||
if not include_hidden and _path_has_hidden_segment(relative_path):
|
if not include_hidden and _path_has_hidden_segment(relative_path):
|
||||||
continue
|
continue
|
||||||
# Use absolute path for gitignore matching
|
if self._should_exclude_file(relative_path, gitignore_matches):
|
||||||
if self._should_exclude_file(file_path_resolved, gitignore_matches):
|
|
||||||
continue
|
continue
|
||||||
except ValueError:
|
except ValueError:
|
||||||
# Skip files that can't be made relative to docs_path
|
# Skip files that can't be made relative to docs_path
|
||||||
@@ -1122,11 +1077,10 @@ Examples:
|
|||||||
) -> bool:
|
) -> bool:
|
||||||
"""Return True if file should be included (not excluded)"""
|
"""Return True if file should be included (not excluded)"""
|
||||||
try:
|
try:
|
||||||
docs_path_obj = Path(docs_dir).resolve()
|
docs_path_obj = Path(docs_dir)
|
||||||
file_path_obj = Path(file_path).resolve()
|
file_path_obj = Path(file_path)
|
||||||
# Use absolute path for gitignore matching
|
relative_path = file_path_obj.relative_to(docs_path_obj)
|
||||||
_ = file_path_obj.relative_to(docs_path_obj) # validate scope
|
return not self._should_exclude_file(relative_path, gitignore_matches)
|
||||||
return not self._should_exclude_file(file_path_obj, gitignore_matches)
|
|
||||||
except (ValueError, OSError):
|
except (ValueError, OSError):
|
||||||
return True # Include files that can't be processed
|
return True # Include files that can't be processed
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "leann"
|
name = "leann"
|
||||||
version = "0.3.3"
|
version = "0.3.2"
|
||||||
description = "LEANN - The smallest vector index in the world. RAG Everything with LEANN!"
|
description = "LEANN - The smallest vector index in the world. RAG Everything with LEANN!"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.9"
|
requires-python = ">=3.9"
|
||||||
|
|||||||
Reference in New Issue
Block a user