diff --git a/docs/ast_chunking_guide.md b/docs/ast_chunking_guide.md index dd5be37..34d7ccb 100644 --- a/docs/ast_chunking_guide.md +++ b/docs/ast_chunking_guide.md @@ -26,6 +26,21 @@ leann build my-code-index --docs ./src --use-ast-chunking uv pip install -e "." ``` +#### For normal users (PyPI install) +- Use `pip install leann` or `uv pip install leann`. +- `astchunk` is pulled automatically from PyPI as a dependency; no extra steps. + +#### For developers (from source, editable) +```bash +git clone https://github.com/yichuan-w/LEANN.git leann +cd leann +git submodule update --init --recursive +uv sync +``` +- This repo vendors `astchunk` as a git submodule at `packages/astchunk-leann` (our fork). +- `[tool.uv.sources]` maps the `astchunk` package to that path in editable mode. +- You can edit code under `packages/astchunk-leann` and Python will use your changes immediately (no separate `pip install astchunk` needed). + ## Best Practices ### When to Use AST Chunking diff --git a/thirdparty/astchunk-leann b/packages/astchunk-leann similarity index 100% rename from thirdparty/astchunk-leann rename to packages/astchunk-leann diff --git a/pyproject.toml b/pyproject.toml index d738017..7265c66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -99,6 +99,7 @@ wechat-exporter = "wechat_exporter.main:main" leann-core = { path = "packages/leann-core", editable = true } leann-backend-diskann = { path = "packages/leann-backend-diskann", editable = true } leann-backend-hnsw = { path = "packages/leann-backend-hnsw", editable = true } +astchunk = { path = "packages/astchunk-leann", editable = true } [tool.ruff] target-version = "py39" diff --git a/uv.lock b/uv.lock index 9bb8f17..1a8c0bf 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.9" resolution-markers = [ "python_full_version >= '3.12'", @@ -201,7 +201,7 @@ wheels = [ [[package]] name = "astchunk" version = "0.1.0" -source = { registry = "https://pypi.org/simple" } +source = { editable = "packages/astchunk-leann" } dependencies = [ { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, @@ -214,10 +214,31 @@ dependencies = [ { name = "tree-sitter-python" }, { name = "tree-sitter-typescript" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/db/2a/7a35e2fac7d550265ae2ee40651425083b37555f921d1a1b77c3f525e0df/astchunk-0.1.0.tar.gz", hash = "sha256:f4dff0ef8b3b3bcfeac363384db1e153f74d4c825dc2e35864abfab027713be4", size = 18093, upload-time = "2025-06-19T04:37:25.34Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/be/84/5433ab0e933b572750cb16fd7edf3d6c7902b069461a22ec670042752a4d/astchunk-0.1.0-py3-none-any.whl", hash = "sha256:33ada9fc3620807fdda5846fa1948af463f281a60e0d43d4f3782b6dbb416d24", size = 15396, upload-time = "2025-06-19T04:37:23.87Z" }, + +[package.metadata] +requires-dist = [ + { name = "black", marker = "extra == 'dev'", specifier = ">=22.0.0" }, + { name = "flake8", marker = "extra == 'dev'", specifier = ">=5.0.0" }, + { name = "isort", marker = "extra == 'dev'", specifier = ">=5.10.0" }, + { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.0.0" }, + { name = "myst-parser", marker = "extra == 'docs'", specifier = ">=0.18.0" }, + { name = "numpy", specifier = ">=1.20.0" }, + { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=2.20.0" }, + { name = "pyrsistent", specifier = ">=0.18.0" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=7.0.0" }, + { name = "pytest", marker = "extra == 'test'", specifier = ">=7.0.0" }, + { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=4.0.0" }, + { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=4.0.0" }, + { name = "pytest-xdist", marker = "extra == 'test'", specifier = ">=2.5.0" }, + { name = "sphinx", marker = "extra == 'docs'", specifier = ">=5.0.0" }, + { name = "sphinx-rtd-theme", marker = "extra == 'docs'", specifier = ">=1.0.0" }, + { name = "tree-sitter", specifier = ">=0.20.0" }, + { name = "tree-sitter-c-sharp", specifier = ">=0.20.0" }, + { name = "tree-sitter-java", specifier = ">=0.20.0" }, + { name = "tree-sitter-python", specifier = ">=0.20.0" }, + { name = "tree-sitter-typescript", specifier = ">=0.20.0" }, ] +provides-extras = ["dev", "docs", "test"] [[package]] name = "asttokens" @@ -2297,7 +2318,7 @@ test = [ [package.metadata] requires-dist = [ - { name = "astchunk", specifier = ">=0.1.0" }, + { name = "astchunk", editable = "packages/astchunk-leann" }, { name = "beautifulsoup4", marker = "extra == 'documents'", specifier = ">=4.13.0" }, { name = "black", marker = "extra == 'dev'", specifier = ">=23.0" }, { name = "boto3" },