Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
72c22e93e1 | ||
|
|
b2eb34dc17 | ||
|
|
2d58650b04 | ||
|
|
950728bd25 |
4
.gitmodules
vendored
4
.gitmodules
vendored
@@ -14,3 +14,7 @@
|
|||||||
[submodule "packages/leann-backend-hnsw/third_party/libzmq"]
|
[submodule "packages/leann-backend-hnsw/third_party/libzmq"]
|
||||||
path = packages/leann-backend-hnsw/third_party/libzmq
|
path = packages/leann-backend-hnsw/third_party/libzmq
|
||||||
url = https://github.com/zeromq/libzmq.git
|
url = https://github.com/zeromq/libzmq.git
|
||||||
|
[submodule "packages/astchunk-leann"]
|
||||||
|
path = packages/astchunk-leann
|
||||||
|
url = git@github.com:yichuan-w/astchunk-leann.git
|
||||||
|
branch = main
|
||||||
|
|||||||
@@ -26,6 +26,21 @@ leann build my-code-index --docs ./src --use-ast-chunking
|
|||||||
uv pip install -e "."
|
uv pip install -e "."
|
||||||
```
|
```
|
||||||
|
|
||||||
|
#### For normal users (PyPI install)
|
||||||
|
- Use `pip install leann` or `uv pip install leann`.
|
||||||
|
- `astchunk` is pulled automatically from PyPI as a dependency; no extra steps.
|
||||||
|
|
||||||
|
#### For developers (from source, editable)
|
||||||
|
```bash
|
||||||
|
git clone https://github.com/yichuan-w/LEANN.git leann
|
||||||
|
cd leann
|
||||||
|
git submodule update --init --recursive
|
||||||
|
uv sync
|
||||||
|
```
|
||||||
|
- This repo vendors `astchunk` as a git submodule at `packages/astchunk-leann` (our fork).
|
||||||
|
- `[tool.uv.sources]` maps the `astchunk` package to that path in editable mode.
|
||||||
|
- You can edit code under `packages/astchunk-leann` and Python will use your changes immediately (no separate `pip install astchunk` needed).
|
||||||
|
|
||||||
## Best Practices
|
## Best Practices
|
||||||
|
|
||||||
### When to Use AST Chunking
|
### When to Use AST Chunking
|
||||||
|
|||||||
@@ -71,7 +71,7 @@ configs = searcher.search("server_port=", use_grep=True)
|
|||||||
### Search Process
|
### Search Process
|
||||||
|
|
||||||
```
|
```
|
||||||
Query: "def train_model"
|
Query: "def train_model"
|
||||||
↓
|
↓
|
||||||
grep -i -n "def train_model" documents.leann.passages.jsonl
|
grep -i -n "def train_model" documents.leann.passages.jsonl
|
||||||
↓
|
↓
|
||||||
@@ -130,14 +130,14 @@ from leann.api import LeannSearcher
|
|||||||
def demonstrate_grep_search():
|
def demonstrate_grep_search():
|
||||||
# Initialize searcher
|
# Initialize searcher
|
||||||
searcher = LeannSearcher("my_index")
|
searcher = LeannSearcher("my_index")
|
||||||
|
|
||||||
print("=== Function Search ===")
|
print("=== Function Search ===")
|
||||||
functions = searcher.search("def __init__", use_grep=True, top_k=5)
|
functions = searcher.search("def __init__", use_grep=True, top_k=5)
|
||||||
for i, result in enumerate(functions, 1):
|
for i, result in enumerate(functions, 1):
|
||||||
print(f"{i}. Score: {result.score}")
|
print(f"{i}. Score: {result.score}")
|
||||||
print(f" Preview: {result.text[:60]}...")
|
print(f" Preview: {result.text[:60]}...")
|
||||||
print()
|
print()
|
||||||
|
|
||||||
print("=== Error Search ===")
|
print("=== Error Search ===")
|
||||||
errors = searcher.search("FileNotFoundError", use_grep=True, top_k=3)
|
errors = searcher.search("FileNotFoundError", use_grep=True, top_k=3)
|
||||||
for result in errors:
|
for result in errors:
|
||||||
|
|||||||
1
packages/astchunk-leann
Submodule
1
packages/astchunk-leann
Submodule
Submodule packages/astchunk-leann added at a4537018a3
@@ -99,6 +99,7 @@ wechat-exporter = "wechat_exporter.main:main"
|
|||||||
leann-core = { path = "packages/leann-core", editable = true }
|
leann-core = { path = "packages/leann-core", editable = true }
|
||||||
leann-backend-diskann = { path = "packages/leann-backend-diskann", editable = true }
|
leann-backend-diskann = { path = "packages/leann-backend-diskann", editable = true }
|
||||||
leann-backend-hnsw = { path = "packages/leann-backend-hnsw", editable = true }
|
leann-backend-hnsw = { path = "packages/leann-backend-hnsw", editable = true }
|
||||||
|
astchunk = { path = "packages/astchunk-leann", editable = true }
|
||||||
|
|
||||||
[tool.ruff]
|
[tool.ruff]
|
||||||
target-version = "py39"
|
target-version = "py39"
|
||||||
|
|||||||
33
uv.lock
generated
33
uv.lock
generated
@@ -1,5 +1,5 @@
|
|||||||
version = 1
|
version = 1
|
||||||
revision = 3
|
revision = 2
|
||||||
requires-python = ">=3.9"
|
requires-python = ">=3.9"
|
||||||
resolution-markers = [
|
resolution-markers = [
|
||||||
"python_full_version >= '3.12'",
|
"python_full_version >= '3.12'",
|
||||||
@@ -201,7 +201,7 @@ wheels = [
|
|||||||
[[package]]
|
[[package]]
|
||||||
name = "astchunk"
|
name = "astchunk"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
source = { registry = "https://pypi.org/simple" }
|
source = { editable = "packages/astchunk-leann" }
|
||||||
dependencies = [
|
dependencies = [
|
||||||
{ name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
|
{ name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
|
||||||
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
|
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
|
||||||
@@ -214,10 +214,31 @@ dependencies = [
|
|||||||
{ name = "tree-sitter-python" },
|
{ name = "tree-sitter-python" },
|
||||||
{ name = "tree-sitter-typescript" },
|
{ name = "tree-sitter-typescript" },
|
||||||
]
|
]
|
||||||
sdist = { url = "https://files.pythonhosted.org/packages/db/2a/7a35e2fac7d550265ae2ee40651425083b37555f921d1a1b77c3f525e0df/astchunk-0.1.0.tar.gz", hash = "sha256:f4dff0ef8b3b3bcfeac363384db1e153f74d4c825dc2e35864abfab027713be4", size = 18093, upload-time = "2025-06-19T04:37:25.34Z" }
|
|
||||||
wheels = [
|
[package.metadata]
|
||||||
{ url = "https://files.pythonhosted.org/packages/be/84/5433ab0e933b572750cb16fd7edf3d6c7902b069461a22ec670042752a4d/astchunk-0.1.0-py3-none-any.whl", hash = "sha256:33ada9fc3620807fdda5846fa1948af463f281a60e0d43d4f3782b6dbb416d24", size = 15396, upload-time = "2025-06-19T04:37:23.87Z" },
|
requires-dist = [
|
||||||
|
{ name = "black", marker = "extra == 'dev'", specifier = ">=22.0.0" },
|
||||||
|
{ name = "flake8", marker = "extra == 'dev'", specifier = ">=5.0.0" },
|
||||||
|
{ name = "isort", marker = "extra == 'dev'", specifier = ">=5.10.0" },
|
||||||
|
{ name = "mypy", marker = "extra == 'dev'", specifier = ">=1.0.0" },
|
||||||
|
{ name = "myst-parser", marker = "extra == 'docs'", specifier = ">=0.18.0" },
|
||||||
|
{ name = "numpy", specifier = ">=1.20.0" },
|
||||||
|
{ name = "pre-commit", marker = "extra == 'dev'", specifier = ">=2.20.0" },
|
||||||
|
{ name = "pyrsistent", specifier = ">=0.18.0" },
|
||||||
|
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=7.0.0" },
|
||||||
|
{ name = "pytest", marker = "extra == 'test'", specifier = ">=7.0.0" },
|
||||||
|
{ name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=4.0.0" },
|
||||||
|
{ name = "pytest-cov", marker = "extra == 'test'", specifier = ">=4.0.0" },
|
||||||
|
{ name = "pytest-xdist", marker = "extra == 'test'", specifier = ">=2.5.0" },
|
||||||
|
{ name = "sphinx", marker = "extra == 'docs'", specifier = ">=5.0.0" },
|
||||||
|
{ name = "sphinx-rtd-theme", marker = "extra == 'docs'", specifier = ">=1.0.0" },
|
||||||
|
{ name = "tree-sitter", specifier = ">=0.20.0" },
|
||||||
|
{ name = "tree-sitter-c-sharp", specifier = ">=0.20.0" },
|
||||||
|
{ name = "tree-sitter-java", specifier = ">=0.20.0" },
|
||||||
|
{ name = "tree-sitter-python", specifier = ">=0.20.0" },
|
||||||
|
{ name = "tree-sitter-typescript", specifier = ">=0.20.0" },
|
||||||
]
|
]
|
||||||
|
provides-extras = ["dev", "docs", "test"]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "asttokens"
|
name = "asttokens"
|
||||||
@@ -2297,7 +2318,7 @@ test = [
|
|||||||
|
|
||||||
[package.metadata]
|
[package.metadata]
|
||||||
requires-dist = [
|
requires-dist = [
|
||||||
{ name = "astchunk", specifier = ">=0.1.0" },
|
{ name = "astchunk", editable = "packages/astchunk-leann" },
|
||||||
{ name = "beautifulsoup4", marker = "extra == 'documents'", specifier = ">=4.13.0" },
|
{ name = "beautifulsoup4", marker = "extra == 'documents'", specifier = ">=4.13.0" },
|
||||||
{ name = "black", marker = "extra == 'dev'", specifier = ">=23.0" },
|
{ name = "black", marker = "extra == 'dev'", specifier = ">=23.0" },
|
||||||
{ name = "boto3" },
|
{ name = "boto3" },
|
||||||
|
|||||||
Reference in New Issue
Block a user