TBNilles/LEANN
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style

Browse Source
This commit is contained in:
Andy Lee
2025-09-17 16:42:19 -07:00
parent bc4d02c2d3
commit 2e7b2d29dc
1 changed files with 13 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
packages/leann-core/src/leann/chunking_utils.py
View File

@@ -179,7 +179,13 @@ def create_text_chunks(
local_code_extensions = CODE_EXTENSIONS.copy() local_code_extensions = CODE_EXTENSIONS.copy()
if code_file_extensions: if code_file_extensions:
ext_mapping = {".py": "python", ".java": "java", ".cs": "c_sharp", ".ts": "typescript", ".tsx": "typescript"} ext_mapping = {
".py": "python",
".java": "java",
".cs": "c_sharp",
".ts": "typescript",
".tsx": "typescript",
}
for ext in code_file_extensions: for ext in code_file_extensions:
if ext.lower() not in local_code_extensions: if ext.lower() not in local_code_extensions:
if ext.lower() in ext_mapping: if ext.lower() in ext_mapping:
@@ -193,12 +199,16 @@ def create_text_chunks(
if code_docs: if code_docs:
try: try:
all_chunks.extend( all_chunks.extend(
create_ast_chunks(code_docs, max_chunk_size=ast_chunk_size, chunk_overlap=ast_chunk_overlap) create_ast_chunks(
code_docs, max_chunk_size=ast_chunk_size, chunk_overlap=ast_chunk_overlap
)
) )
except Exception as e: except Exception as e:
logger.error(f"AST chunking failed: {e}") logger.error(f"AST chunking failed: {e}")
if ast_fallback_traditional: if ast_fallback_traditional:
all_chunks.extend(create_traditional_chunks(code_docs, chunk_size, chunk_overlap)) all_chunks.extend(
create_traditional_chunks(code_docs, chunk_size, chunk_overlap)
)
else: else:
raise raise
if text_docs: if text_docs: