diff --git a/README.md b/README.md index 7538bc0..132ea6a 100755 --- a/README.md +++ b/README.md @@ -176,6 +176,9 @@ response = chat.ask("How much storage does LEANN save?", top_k=1) LEANN supports RAG on various data sources including documents (`.pdf`, `.txt`, `.md`), Apple Mail, Google Search History, WeChat, and more. +**AST-Aware Code Chunking** - LEANN also features intelligent code chunking that preserves semantic boundaries (functions, classes, methods) for Python, Java, C#, and TypeScript files, providing improved code understanding compared to traditional text-based approaches. +๐Ÿ“– Read the [AST Chunking Guide โ†’](docs/ast_chunking_guide.md) to learn more. + ### Generation Model Setup LEANN supports multiple LLM providers for text generation (OpenAI API, HuggingFace, Ollama). @@ -294,6 +297,12 @@ python -m apps.document_rag --data-dir "~/Documents/Papers" --chunk-size 1024 # Filter only markdown and Python files with smaller chunks python -m apps.document_rag --data-dir "./docs" --chunk-size 256 --file-types .md .py + +# Enable AST-aware chunking for code files +python -m apps.document_rag --enable-code-chunking --data-dir "./my_project" + +# Or use the specialized code RAG for better code understanding +python -m apps.code_rag --repo-dir "./my_codebase" --query "How does authentication work?" ``` @@ -472,6 +481,7 @@ Once the index is built, you can ask questions like: **Key features:** - ๐Ÿ” **Semantic code search** across your entire project, fully local index and lightweight +- ๐Ÿง  **AST-aware chunking** preserves code structure (functions, classes) - ๐Ÿ“š **Context-aware assistance** for debugging and development - ๐Ÿš€ **Zero-config setup** with automatic language detection @@ -534,7 +544,8 @@ leann remove my-docs **Key CLI features:** - Auto-detects document formats (PDF, TXT, MD, DOCX, PPTX + code files) -- Smart text chunking with overlap +- **๐Ÿง  AST-aware chunking** for Python, Java, C#, TypeScript files +- Smart text chunking with overlap for all other content - Multiple LLM providers (Ollama, OpenAI, HuggingFace) - Organized index storage in `.leann/indexes/` (project-local) - Support for advanced search parameters diff --git a/apps/base_rag_example.py b/apps/base_rag_example.py index 8d539d3..46ff187 100644 --- a/apps/base_rag_example.py +++ b/apps/base_rag_example.py @@ -11,7 +11,6 @@ from typing import Any import dotenv from leann.api import LeannBuilder, LeannChat from leann.registry import register_project_directory -from llama_index.core.node_parser import SentenceSplitter dotenv.load_dotenv() @@ -109,6 +108,38 @@ class BaseRAGExample(ABC): help="Thinking budget for reasoning models (low/medium/high). Supported by GPT-Oss:20b and other reasoning models.", ) + # AST Chunking parameters + ast_group = parser.add_argument_group("AST Chunking Parameters") + ast_group.add_argument( + "--use-ast-chunking", + action="store_true", + help="Enable AST-aware chunking for code files (requires astchunk)", + ) + ast_group.add_argument( + "--ast-chunk-size", + type=int, + default=512, + help="Maximum characters per AST chunk (default: 512)", + ) + ast_group.add_argument( + "--ast-chunk-overlap", + type=int, + default=64, + help="Overlap between AST chunks (default: 64)", + ) + ast_group.add_argument( + "--code-file-extensions", + nargs="+", + default=None, + help="Additional code file extensions to process with AST chunking (e.g., .py .java .cs .ts)", + ) + ast_group.add_argument( + "--ast-fallback-traditional", + action="store_true", + default=True, + help="Fall back to traditional chunking if AST chunking fails (default: True)", + ) + # Search parameters search_group = parser.add_argument_group("Search Parameters") search_group.add_argument( @@ -310,21 +341,3 @@ class BaseRAGExample(ABC): await self.run_single_query(args, index_path, args.query) else: await self.run_interactive_chat(args, index_path) - - -def create_text_chunks(documents, chunk_size=256, chunk_overlap=25) -> list[str]: - """Helper function to create text chunks from documents.""" - node_parser = SentenceSplitter( - chunk_size=chunk_size, - chunk_overlap=chunk_overlap, - separator=" ", - paragraph_separator="\n\n", - ) - - all_texts = [] - for doc in documents: - nodes = node_parser.get_nodes_from_documents([doc]) - if nodes: - all_texts.extend(node.get_content() for node in nodes) - - return all_texts diff --git a/apps/chunking/__init__.py b/apps/chunking/__init__.py new file mode 100644 index 0000000..3cd5c0d --- /dev/null +++ b/apps/chunking/__init__.py @@ -0,0 +1,22 @@ +""" +Chunking utilities for LEANN RAG applications. +Provides AST-aware and traditional text chunking functionality. +""" + +from .utils import ( + CODE_EXTENSIONS, + create_ast_chunks, + create_text_chunks, + create_traditional_chunks, + detect_code_files, + get_language_from_extension, +) + +__all__ = [ + "CODE_EXTENSIONS", + "create_ast_chunks", + "create_text_chunks", + "create_traditional_chunks", + "detect_code_files", + "get_language_from_extension", +] diff --git a/apps/chunking/utils.py b/apps/chunking/utils.py new file mode 100644 index 0000000..9a19c63 --- /dev/null +++ b/apps/chunking/utils.py @@ -0,0 +1,320 @@ +""" +Enhanced chunking utilities with AST-aware code chunking support. +Provides unified interface for both traditional and AST-based text chunking. +""" + +import logging +from pathlib import Path +from typing import Optional + +from llama_index.core.node_parser import SentenceSplitter + +logger = logging.getLogger(__name__) + +# Code file extensions supported by astchunk +CODE_EXTENSIONS = { + ".py": "python", + ".java": "java", + ".cs": "csharp", + ".ts": "typescript", + ".tsx": "typescript", + ".js": "typescript", + ".jsx": "typescript", +} + +# Default chunk parameters for different content types +DEFAULT_CHUNK_PARAMS = { + "code": { + "max_chunk_size": 512, + "chunk_overlap": 64, + }, + "text": { + "chunk_size": 256, + "chunk_overlap": 128, + }, +} + + +def detect_code_files(documents, code_extensions=None) -> tuple[list, list]: + """ + Separate documents into code files and regular text files. + + Args: + documents: List of LlamaIndex Document objects + code_extensions: Dict mapping file extensions to languages (defaults to CODE_EXTENSIONS) + + Returns: + Tuple of (code_documents, text_documents) + """ + if code_extensions is None: + code_extensions = CODE_EXTENSIONS + + code_docs = [] + text_docs = [] + + for doc in documents: + # Get file path from metadata + file_path = doc.metadata.get("file_path", "") + if not file_path: + # Fallback to file_name + file_path = doc.metadata.get("file_name", "") + + if file_path: + file_ext = Path(file_path).suffix.lower() + if file_ext in code_extensions: + # Add language info to metadata + doc.metadata["language"] = code_extensions[file_ext] + doc.metadata["is_code"] = True + code_docs.append(doc) + else: + doc.metadata["is_code"] = False + text_docs.append(doc) + else: + # If no file path, treat as text + doc.metadata["is_code"] = False + text_docs.append(doc) + + logger.info(f"Detected {len(code_docs)} code files and {len(text_docs)} text files") + return code_docs, text_docs + + +def get_language_from_extension(file_path: str) -> Optional[str]: + """Get the programming language from file extension.""" + ext = Path(file_path).suffix.lower() + return CODE_EXTENSIONS.get(ext) + + +def create_ast_chunks( + documents, + max_chunk_size: int = 512, + chunk_overlap: int = 64, + metadata_template: str = "default", +) -> list[str]: + """ + Create AST-aware chunks from code documents using astchunk. + + Args: + documents: List of code documents + max_chunk_size: Maximum characters per chunk + chunk_overlap: Number of AST nodes to overlap between chunks + metadata_template: Template for chunk metadata + + Returns: + List of text chunks with preserved code structure + """ + try: + from astchunk import ASTChunkBuilder + except ImportError as e: + logger.error(f"astchunk not available: {e}") + logger.info("Falling back to traditional chunking for code files") + return create_traditional_chunks(documents, max_chunk_size, chunk_overlap) + + all_chunks = [] + + for doc in documents: + # Get language from metadata (set by detect_code_files) + language = doc.metadata.get("language") + if not language: + logger.warning( + "No language detected for document, falling back to traditional chunking" + ) + traditional_chunks = create_traditional_chunks([doc], max_chunk_size, chunk_overlap) + all_chunks.extend(traditional_chunks) + continue + + try: + # Configure astchunk + configs = { + "max_chunk_size": max_chunk_size, + "language": language, + "metadata_template": metadata_template, + "chunk_overlap": chunk_overlap if chunk_overlap > 0 else 0, + } + + # Add repository-level metadata if available + repo_metadata = { + "file_path": doc.metadata.get("file_path", ""), + "file_name": doc.metadata.get("file_name", ""), + "creation_date": doc.metadata.get("creation_date", ""), + "last_modified_date": doc.metadata.get("last_modified_date", ""), + } + configs["repo_level_metadata"] = repo_metadata + + # Create chunk builder and process + chunk_builder = ASTChunkBuilder(**configs) + code_content = doc.get_content() + + if not code_content or not code_content.strip(): + logger.warning("Empty code content, skipping") + continue + + chunks = chunk_builder.chunkify(code_content) + + # Extract text content from chunks + for chunk in chunks: + if hasattr(chunk, "text"): + chunk_text = chunk.text + elif isinstance(chunk, dict) and "text" in chunk: + chunk_text = chunk["text"] + elif isinstance(chunk, str): + chunk_text = chunk + else: + # Try to convert to string + chunk_text = str(chunk) + + if chunk_text and chunk_text.strip(): + all_chunks.append(chunk_text.strip()) + + logger.info( + f"Created {len(chunks)} AST chunks from {language} file: {doc.metadata.get('file_name', 'unknown')}" + ) + + except Exception as e: + logger.warning(f"AST chunking failed for {language} file: {e}") + logger.info("Falling back to traditional chunking") + traditional_chunks = create_traditional_chunks([doc], max_chunk_size, chunk_overlap) + all_chunks.extend(traditional_chunks) + + return all_chunks + + +def create_traditional_chunks( + documents, chunk_size: int = 256, chunk_overlap: int = 128 +) -> list[str]: + """ + Create traditional text chunks using LlamaIndex SentenceSplitter. + + Args: + documents: List of documents to chunk + chunk_size: Size of each chunk in characters + chunk_overlap: Overlap between chunks + + Returns: + List of text chunks + """ + # Handle invalid chunk_size values + if chunk_size <= 0: + logger.warning(f"Invalid chunk_size={chunk_size}, using default value of 256") + chunk_size = 256 + + # Ensure chunk_overlap is not negative and not larger than chunk_size + if chunk_overlap < 0: + chunk_overlap = 0 + if chunk_overlap >= chunk_size: + chunk_overlap = chunk_size // 2 + + node_parser = SentenceSplitter( + chunk_size=chunk_size, + chunk_overlap=chunk_overlap, + separator=" ", + paragraph_separator="\n\n", + ) + + all_texts = [] + for doc in documents: + try: + nodes = node_parser.get_nodes_from_documents([doc]) + if nodes: + chunk_texts = [node.get_content() for node in nodes] + all_texts.extend(chunk_texts) + logger.debug(f"Created {len(chunk_texts)} traditional chunks from document") + except Exception as e: + logger.error(f"Traditional chunking failed for document: {e}") + # As last resort, add the raw content + content = doc.get_content() + if content and content.strip(): + all_texts.append(content.strip()) + + return all_texts + + +def create_text_chunks( + documents, + chunk_size: int = 256, + chunk_overlap: int = 128, + use_ast_chunking: bool = False, + ast_chunk_size: int = 512, + ast_chunk_overlap: int = 64, + code_file_extensions: Optional[list[str]] = None, + ast_fallback_traditional: bool = True, +) -> list[str]: + """ + Create text chunks from documents with optional AST support for code files. + + Args: + documents: List of LlamaIndex Document objects + chunk_size: Size for traditional text chunks + chunk_overlap: Overlap for traditional text chunks + use_ast_chunking: Whether to use AST chunking for code files + ast_chunk_size: Size for AST chunks + ast_chunk_overlap: Overlap for AST chunks + code_file_extensions: Custom list of code file extensions + ast_fallback_traditional: Fall back to traditional chunking on AST errors + + Returns: + List of text chunks + """ + if not documents: + logger.warning("No documents provided for chunking") + return [] + + # Create a local copy of supported extensions for this function call + local_code_extensions = CODE_EXTENSIONS.copy() + + # Update supported extensions if provided + if code_file_extensions: + # Map extensions to languages (simplified mapping) + ext_mapping = { + ".py": "python", + ".java": "java", + ".cs": "c_sharp", + ".ts": "typescript", + ".tsx": "typescript", + } + for ext in code_file_extensions: + if ext.lower() not in local_code_extensions: + # Try to guess language from extension + if ext.lower() in ext_mapping: + local_code_extensions[ext.lower()] = ext_mapping[ext.lower()] + else: + logger.warning(f"Unsupported extension {ext}, will use traditional chunking") + + all_chunks = [] + + if use_ast_chunking: + # Separate code and text documents using local extensions + code_docs, text_docs = detect_code_files(documents, local_code_extensions) + + # Process code files with AST chunking + if code_docs: + logger.info(f"Processing {len(code_docs)} code files with AST chunking") + try: + ast_chunks = create_ast_chunks( + code_docs, max_chunk_size=ast_chunk_size, chunk_overlap=ast_chunk_overlap + ) + all_chunks.extend(ast_chunks) + logger.info(f"Created {len(ast_chunks)} AST chunks from code files") + except Exception as e: + logger.error(f"AST chunking failed: {e}") + if ast_fallback_traditional: + logger.info("Falling back to traditional chunking for code files") + traditional_code_chunks = create_traditional_chunks( + code_docs, chunk_size, chunk_overlap + ) + all_chunks.extend(traditional_code_chunks) + else: + raise + + # Process text files with traditional chunking + if text_docs: + logger.info(f"Processing {len(text_docs)} text files with traditional chunking") + text_chunks = create_traditional_chunks(text_docs, chunk_size, chunk_overlap) + all_chunks.extend(text_chunks) + logger.info(f"Created {len(text_chunks)} traditional chunks from text files") + else: + # Use traditional chunking for all files + logger.info(f"Processing {len(documents)} documents with traditional chunking") + all_chunks = create_traditional_chunks(documents, chunk_size, chunk_overlap) + + logger.info(f"Total chunks created: {len(all_chunks)}") + return all_chunks diff --git a/apps/code_rag.py b/apps/code_rag.py new file mode 100644 index 0000000..7518bb9 --- /dev/null +++ b/apps/code_rag.py @@ -0,0 +1,211 @@ +""" +Code RAG example using AST-aware chunking for optimal code understanding. +Specialized for code repositories with automatic language detection and +optimized chunking parameters. +""" + +import sys +from pathlib import Path + +# Add parent directory to path for imports +sys.path.insert(0, str(Path(__file__).parent)) + +from base_rag_example import BaseRAGExample +from chunking import CODE_EXTENSIONS, create_text_chunks +from llama_index.core import SimpleDirectoryReader + + +class CodeRAG(BaseRAGExample): + """Specialized RAG example for code repositories with AST-aware chunking.""" + + def __init__(self): + super().__init__( + name="Code", + description="Process and query code repositories with AST-aware chunking", + default_index_name="code_index", + ) + # Override defaults for code-specific usage + self.embedding_model_default = "facebook/contriever" # Good for code + self.max_items_default = -1 # Process all code files by default + + def _add_specific_arguments(self, parser): + """Add code-specific arguments.""" + code_group = parser.add_argument_group("Code Repository Parameters") + + code_group.add_argument( + "--repo-dir", + type=str, + default=".", + help="Code repository directory to index (default: current directory)", + ) + code_group.add_argument( + "--include-extensions", + nargs="+", + default=list(CODE_EXTENSIONS.keys()), + help="File extensions to include (default: supported code extensions)", + ) + code_group.add_argument( + "--exclude-dirs", + nargs="+", + default=[ + ".git", + "__pycache__", + "node_modules", + "venv", + ".venv", + "build", + "dist", + "target", + ], + help="Directories to exclude from indexing", + ) + code_group.add_argument( + "--max-file-size", + type=int, + default=1000000, # 1MB + help="Maximum file size in bytes to process (default: 1MB)", + ) + code_group.add_argument( + "--include-comments", + action="store_true", + help="Include comments in chunking (useful for documentation)", + ) + code_group.add_argument( + "--preserve-imports", + action="store_true", + default=True, + help="Try to preserve import statements in chunks (default: True)", + ) + + async def load_data(self, args) -> list[str]: + """Load code files and convert to AST-aware chunks.""" + print(f"๐Ÿ” Scanning code repository: {args.repo_dir}") + print(f"๐Ÿ“ Including extensions: {args.include_extensions}") + print(f"๐Ÿšซ Excluding directories: {args.exclude_dirs}") + + # Check if repository directory exists + repo_path = Path(args.repo_dir) + if not repo_path.exists(): + raise ValueError(f"Repository directory not found: {args.repo_dir}") + + # Load code files with filtering + reader_kwargs = { + "recursive": True, + "encoding": "utf-8", + "required_exts": args.include_extensions, + "exclude_hidden": True, + } + + # Create exclusion filter + def file_filter(file_path: str) -> bool: + """Filter out unwanted files and directories.""" + path = Path(file_path) + + # Check file size + try: + if path.stat().st_size > args.max_file_size: + print(f"โš ๏ธ Skipping large file: {path.name} ({path.stat().st_size} bytes)") + return False + except Exception: + return False + + # Check if in excluded directory + for exclude_dir in args.exclude_dirs: + if exclude_dir in path.parts: + return False + + return True + + try: + # Load documents with file filtering + documents = SimpleDirectoryReader( + args.repo_dir, + file_extractor=None, # Use default extractors + **reader_kwargs, + ).load_data(show_progress=True) + + # Apply custom filtering + filtered_docs = [] + for doc in documents: + file_path = doc.metadata.get("file_path", "") + if file_filter(file_path): + filtered_docs.append(doc) + + documents = filtered_docs + + except Exception as e: + print(f"โŒ Error loading code files: {e}") + return [] + + if not documents: + print( + f"โŒ No code files found in {args.repo_dir} with extensions {args.include_extensions}" + ) + return [] + + print(f"โœ… Loaded {len(documents)} code files") + + # Show breakdown by language/extension + ext_counts = {} + for doc in documents: + file_path = doc.metadata.get("file_path", "") + if file_path: + ext = Path(file_path).suffix.lower() + ext_counts[ext] = ext_counts.get(ext, 0) + 1 + + print("๐Ÿ“Š Files by extension:") + for ext, count in sorted(ext_counts.items()): + print(f" {ext}: {count} files") + + # Use AST-aware chunking by default for code + print( + f"๐Ÿง  Using AST-aware chunking (chunk_size: {args.ast_chunk_size}, overlap: {args.ast_chunk_overlap})" + ) + + all_texts = create_text_chunks( + documents, + chunk_size=256, # Fallback for non-code files + chunk_overlap=64, + use_ast_chunking=True, # Always use AST for code RAG + ast_chunk_size=args.ast_chunk_size, + ast_chunk_overlap=args.ast_chunk_overlap, + code_file_extensions=args.include_extensions, + ast_fallback_traditional=True, + ) + + # Apply max_items limit if specified + if args.max_items > 0 and len(all_texts) > args.max_items: + print(f"โณ Limiting to {args.max_items} chunks (from {len(all_texts)})") + all_texts = all_texts[: args.max_items] + + print(f"โœ… Generated {len(all_texts)} code chunks") + return all_texts + + +if __name__ == "__main__": + import asyncio + + # Example queries for code RAG + print("\n๐Ÿ’ป Code RAG Example") + print("=" * 50) + print("\nExample queries you can try:") + print("- 'How does the embedding computation work?'") + print("- 'What are the main classes in this codebase?'") + print("- 'Show me the search implementation'") + print("- 'How is error handling implemented?'") + print("- 'What design patterns are used?'") + print("- 'Explain the chunking logic'") + print("\n๐Ÿš€ Features:") + print("- โœ… AST-aware chunking preserves code structure") + print("- โœ… Automatic language detection") + print("- โœ… Smart filtering of large files and common excludes") + print("- โœ… Optimized for code understanding") + print("\nUsage examples:") + print(" python -m apps.code_rag --repo-dir ./my_project") + print( + " python -m apps.code_rag --include-extensions .py .js --query 'How does authentication work?'" + ) + print("\nOr run without --query for interactive mode\n") + + rag = CodeRAG() + asyncio.run(rag.run()) diff --git a/apps/document_rag.py b/apps/document_rag.py index 1ac15c7..8472f6f 100644 --- a/apps/document_rag.py +++ b/apps/document_rag.py @@ -9,7 +9,8 @@ from pathlib import Path # Add parent directory to path for imports sys.path.insert(0, str(Path(__file__).parent)) -from base_rag_example import BaseRAGExample, create_text_chunks +from base_rag_example import BaseRAGExample +from chunking import create_text_chunks from llama_index.core import SimpleDirectoryReader @@ -44,6 +45,11 @@ class DocumentRAG(BaseRAGExample): doc_group.add_argument( "--chunk-overlap", type=int, default=128, help="Text chunk overlap (default: 128)" ) + doc_group.add_argument( + "--enable-code-chunking", + action="store_true", + help="Enable AST-aware chunking for code files in the data directory", + ) async def load_data(self, args) -> list[str]: """Load documents and convert to text chunks.""" @@ -76,9 +82,22 @@ class DocumentRAG(BaseRAGExample): print(f"Loaded {len(documents)} documents") - # Convert to text chunks + # Determine chunking strategy + use_ast = args.enable_code_chunking or getattr(args, "use_ast_chunking", False) + + if use_ast: + print("Using AST-aware chunking for code files") + + # Convert to text chunks with optional AST support all_texts = create_text_chunks( - documents, chunk_size=args.chunk_size, chunk_overlap=args.chunk_overlap + documents, + chunk_size=args.chunk_size, + chunk_overlap=args.chunk_overlap, + use_ast_chunking=use_ast, + ast_chunk_size=getattr(args, "ast_chunk_size", 512), + ast_chunk_overlap=getattr(args, "ast_chunk_overlap", 64), + code_file_extensions=getattr(args, "code_file_extensions", None), + ast_fallback_traditional=getattr(args, "ast_fallback_traditional", True), ) # Apply max_items limit if specified @@ -102,6 +121,10 @@ if __name__ == "__main__": print( "- 'What is the problem of developing pan gu model Huawei meets? (็›˜ๅคๅคงๆจกๅž‹ๅผ€ๅ‘ไธญ้‡ๅˆฐไป€ไนˆ้—ฎ้ข˜?)'" ) + print("\n๐Ÿš€ NEW: Code-aware chunking available!") + print("- Use --enable-code-chunking to enable AST-aware chunking for code files") + print("- Supports Python, Java, C#, TypeScript files") + print("- Better semantic understanding of code structure") print("\nOr run without --query for interactive mode\n") rag = DocumentRAG() diff --git a/docs/ast_chunking_guide.md b/docs/ast_chunking_guide.md new file mode 100644 index 0000000..dd5be37 --- /dev/null +++ b/docs/ast_chunking_guide.md @@ -0,0 +1,128 @@ +# AST-Aware Code chunking guide + +## Overview + +This guide covers best practices for using AST-aware code chunking in LEANN. AST chunking provides better semantic understanding of code structure compared to traditional text-based chunking. + +## Quick Start + +### Basic Usage + +```bash +# Enable AST chunking for mixed content (code + docs) +python -m apps.document_rag --enable-code-chunking --data-dir ./my_project + +# Specialized code repository indexing +python -m apps.code_rag --repo-dir ./my_codebase + +# Global CLI with AST support +leann build my-code-index --docs ./src --use-ast-chunking +``` + +### Installation + +```bash +# Install LEANN with AST chunking support +uv pip install -e "." +``` + +## Best Practices + +### When to Use AST Chunking + +โœ… **Recommended for:** +- Code repositories with multiple languages +- Mixed documentation and code content +- Complex codebases with deep function/class hierarchies +- When working with Claude Code for code assistance + +โŒ **Not recommended for:** +- Pure text documents +- Very large files (>1MB) +- Languages not supported by tree-sitter + +### Optimal Configuration + +```bash +# Recommended settings for most codebases +python -m apps.code_rag \ + --repo-dir ./src \ + --ast-chunk-size 768 \ + --ast-chunk-overlap 96 \ + --exclude-dirs .git __pycache__ node_modules build dist +``` + +### Supported Languages + +| Extension | Language | Status | +|-----------|----------|--------| +| `.py` | Python | โœ… Full support | +| `.java` | Java | โœ… Full support | +| `.cs` | C# | โœ… Full support | +| `.ts`, `.tsx` | TypeScript | โœ… Full support | +| `.js`, `.jsx` | JavaScript | โœ… Via TypeScript parser | + +## Integration Examples + +### Document RAG with Code Support + +```python +# Enable code chunking in document RAG +python -m apps.document_rag \ + --enable-code-chunking \ + --data-dir ./project \ + --query "How does authentication work in the codebase?" +``` + +### Claude Code Integration + +When using with Claude Code MCP server, AST chunking provides better context for: +- Code completion and suggestions +- Bug analysis and debugging +- Architecture understanding +- Refactoring assistance + +## Troubleshooting + +### Common Issues + +1. **Fallback to Traditional Chunking** + - Normal behavior for unsupported languages + - Check logs for specific language support + +2. **Performance with Large Files** + - Adjust `--max-file-size` parameter + - Use `--exclude-dirs` to skip unnecessary directories + +3. **Quality Issues** + - Try different `--ast-chunk-size` values (512, 768, 1024) + - Adjust overlap for better context preservation + +### Debug Mode + +```bash +export LEANN_LOG_LEVEL=DEBUG +python -m apps.code_rag --repo-dir ./my_code +``` + +## Migration from Traditional Chunking + +Existing workflows continue to work without changes. To enable AST chunking: + +```bash +# Before +python -m apps.document_rag --chunk-size 256 + +# After (maintains traditional chunking for non-code files) +python -m apps.document_rag --enable-code-chunking --chunk-size 256 --ast-chunk-size 768 +``` + +## References + +- [astchunk GitHub Repository](https://github.com/yilinjz/astchunk) +- [LEANN MCP Integration](../packages/leann-mcp/README.md) +- [Research Paper](https://arxiv.org/html/2506.15655v1) + +--- + +**Note**: AST chunking maintains full backward compatibility while enhancing code understanding capabilities. diff --git a/docs/features.md b/docs/features.md index da4e495..0a7f9dc 100644 --- a/docs/features.md +++ b/docs/features.md @@ -3,6 +3,7 @@ ## ๐Ÿ”ฅ Core Features - **๐Ÿ”„ Real-time Embeddings** - Eliminate heavy embedding storage with dynamic computation using optimized ZMQ servers and highly optimized search paradigm (overlapping and batching) with highly optimized embedding engine +- **๐Ÿง  AST-Aware Code Chunking** - Intelligent code chunking that preserves semantic boundaries (functions, classes, methods) for Python, Java, C#, and TypeScript files - **๐Ÿ“ˆ Scalable Architecture** - Handles millions of documents on consumer hardware; the larger your dataset, the more LEANN can save - **๐ŸŽฏ Graph Pruning** - Advanced techniques to minimize the storage overhead of vector search to a limited footprint - **๐Ÿ—๏ธ Pluggable Backends** - HNSW/FAISS (default), with optional DiskANN for large-scale deployments diff --git a/packages/leann-core/src/leann/cli.py b/packages/leann-core/src/leann/cli.py index 9dcb6c7..5a2611a 100644 --- a/packages/leann-core/src/leann/cli.py +++ b/packages/leann-core/src/leann/cli.py @@ -1,7 +1,8 @@ import argparse import asyncio +import sys from pathlib import Path -from typing import Optional, Union +from typing import Any, Optional, Union from llama_index.core import SimpleDirectoryReader from llama_index.core.node_parser import SentenceSplitter @@ -180,6 +181,29 @@ Examples: default=50, help="Code chunk overlap (default: 50)", ) + build_parser.add_argument( + "--use-ast-chunking", + action="store_true", + help="Enable AST-aware chunking for code files (requires astchunk)", + ) + build_parser.add_argument( + "--ast-chunk-size", + type=int, + default=768, + help="AST chunk size in characters (default: 768)", + ) + build_parser.add_argument( + "--ast-chunk-overlap", + type=int, + default=96, + help="AST chunk overlap in characters (default: 96)", + ) + build_parser.add_argument( + "--ast-fallback-traditional", + action="store_true", + default=True, + help="Fall back to traditional chunking if AST chunking fails (default: True)", + ) # Search command search_parser = subparsers.add_parser("search", help="Search documents") @@ -833,6 +857,7 @@ Examples: docs_paths: Union[str, list], custom_file_types: Union[str, None] = None, include_hidden: bool = False, + args: Optional[dict[str, Any]] = None, ): # Handle both single path (string) and multiple paths (list) for backward compatibility if isinstance(docs_paths, str): @@ -1138,18 +1163,50 @@ Examples: } print("start chunking documents") - # Add progress bar for document chunking - for doc in tqdm(documents, desc="Chunking documents", unit="doc"): - # Check if this is a code file based on source path - source_path = doc.metadata.get("source", "") - is_code_file = any(source_path.endswith(ext) for ext in code_file_exts) - # Use appropriate parser based on file type - parser = self.code_parser if is_code_file else self.node_parser - nodes = parser.get_nodes_from_documents([doc]) + # Check if AST chunking is requested + use_ast = getattr(args, "use_ast_chunking", False) - for node in nodes: - all_texts.append(node.get_content()) + if use_ast: + print("๐Ÿง  Using AST-aware chunking for code files") + try: + # Import enhanced chunking utilities + # Add apps directory to path to import chunking utilities + apps_dir = Path(__file__).parent.parent.parent.parent.parent / "apps" + if apps_dir.exists(): + sys.path.insert(0, str(apps_dir)) + + from chunking import create_text_chunks + + # Use enhanced chunking with AST support + all_texts = create_text_chunks( + documents, + chunk_size=self.node_parser.chunk_size, + chunk_overlap=self.node_parser.chunk_overlap, + use_ast_chunking=True, + ast_chunk_size=getattr(args, "ast_chunk_size", 768), + ast_chunk_overlap=getattr(args, "ast_chunk_overlap", 96), + code_file_extensions=None, # Use defaults + ast_fallback_traditional=getattr(args, "ast_fallback_traditional", True), + ) + + except ImportError as e: + print(f"โš ๏ธ AST chunking not available ({e}), falling back to traditional chunking") + use_ast = False + + if not use_ast: + # Use traditional chunking logic + for doc in tqdm(documents, desc="Chunking documents", unit="doc"): + # Check if this is a code file based on source path + source_path = doc.metadata.get("source", "") + is_code_file = any(source_path.endswith(ext) for ext in code_file_exts) + + # Use appropriate parser based on file type + parser = self.code_parser if is_code_file else self.node_parser + nodes = parser.get_nodes_from_documents([doc]) + + for node in nodes: + all_texts.append(node.get_content()) print(f"Loaded {len(documents)} documents, {len(all_texts)} chunks") return all_texts @@ -1216,7 +1273,7 @@ Examples: ) all_texts = self.load_documents( - docs_paths, args.file_types, include_hidden=args.include_hidden + docs_paths, args.file_types, include_hidden=args.include_hidden, args=args ) if not all_texts: print("No documents found") diff --git a/pyproject.toml b/pyproject.toml index 3267332..d738017 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,13 @@ dependencies = [ "pathspec>=0.12.1", "nbconvert>=7.16.6", "gitignore-parser>=0.1.12", + # AST-aware code chunking dependencies + "astchunk>=0.1.0", + "tree-sitter>=0.20.0", + "tree-sitter-python>=0.20.0", + "tree-sitter-java>=0.20.0", + "tree-sitter-c-sharp>=0.20.0", + "tree-sitter-typescript>=0.20.0", ] [project.optional-dependencies] diff --git a/tests/test_astchunk_integration.py b/tests/test_astchunk_integration.py new file mode 100644 index 0000000..df34521 --- /dev/null +++ b/tests/test_astchunk_integration.py @@ -0,0 +1,397 @@ +""" +Test suite for astchunk integration with LEANN. +Tests AST-aware chunking functionality, language detection, and fallback mechanisms. +""" + +import os +import subprocess +import sys +import tempfile +from pathlib import Path +from unittest.mock import patch + +import pytest + +# Add apps directory to path for imports +sys.path.insert(0, str(Path(__file__).parent.parent / "apps")) + +from typing import Optional + +from chunking import ( + create_ast_chunks, + create_text_chunks, + create_traditional_chunks, + detect_code_files, + get_language_from_extension, +) + + +class MockDocument: + """Mock LlamaIndex Document for testing.""" + + def __init__(self, content: str, file_path: str = "", metadata: Optional[dict] = None): + self.content = content + self.metadata = metadata or {} + if file_path: + self.metadata["file_path"] = file_path + + def get_content(self) -> str: + return self.content + + +class TestCodeFileDetection: + """Test code file detection and language mapping.""" + + def test_detect_code_files_python(self): + """Test detection of Python files.""" + docs = [ + MockDocument("print('hello')", "/path/to/file.py"), + MockDocument("This is text", "/path/to/file.txt"), + ] + + code_docs, text_docs = detect_code_files(docs) + + assert len(code_docs) == 1 + assert len(text_docs) == 1 + assert code_docs[0].metadata["language"] == "python" + assert code_docs[0].metadata["is_code"] is True + assert text_docs[0].metadata["is_code"] is False + + def test_detect_code_files_multiple_languages(self): + """Test detection of multiple programming languages.""" + docs = [ + MockDocument("def func():", "/path/to/script.py"), + MockDocument("public class Test {}", "/path/to/Test.java"), + MockDocument("interface ITest {}", "/path/to/test.ts"), + MockDocument("using System;", "/path/to/Program.cs"), + MockDocument("Regular text content", "/path/to/document.txt"), + ] + + code_docs, text_docs = detect_code_files(docs) + + assert len(code_docs) == 4 + assert len(text_docs) == 1 + + languages = [doc.metadata["language"] for doc in code_docs] + assert "python" in languages + assert "java" in languages + assert "typescript" in languages + assert "csharp" in languages + + def test_detect_code_files_no_file_path(self): + """Test handling of documents without file paths.""" + docs = [ + MockDocument("some content"), + MockDocument("other content", metadata={"some_key": "value"}), + ] + + code_docs, text_docs = detect_code_files(docs) + + assert len(code_docs) == 0 + assert len(text_docs) == 2 + for doc in text_docs: + assert doc.metadata["is_code"] is False + + def test_get_language_from_extension(self): + """Test language detection from file extensions.""" + assert get_language_from_extension("test.py") == "python" + assert get_language_from_extension("Test.java") == "java" + assert get_language_from_extension("component.tsx") == "typescript" + assert get_language_from_extension("Program.cs") == "csharp" + assert get_language_from_extension("document.txt") is None + assert get_language_from_extension("") is None + + +class TestChunkingFunctions: + """Test various chunking functionality.""" + + def test_create_traditional_chunks(self): + """Test traditional text chunking.""" + docs = [ + MockDocument( + "This is a test document. It has multiple sentences. We want to test chunking." + ) + ] + + chunks = create_traditional_chunks(docs, chunk_size=50, chunk_overlap=10) + + assert len(chunks) > 0 + assert all(isinstance(chunk, str) for chunk in chunks) + assert all(len(chunk.strip()) > 0 for chunk in chunks) + + def test_create_traditional_chunks_empty_docs(self): + """Test traditional chunking with empty documents.""" + chunks = create_traditional_chunks([], chunk_size=50, chunk_overlap=10) + assert chunks == [] + + @pytest.mark.skipif( + os.environ.get("CI") == "true", + reason="Skip astchunk tests in CI - dependency may not be available", + ) + def test_create_ast_chunks_with_astchunk_available(self): + """Test AST chunking when astchunk is available.""" + python_code = ''' +def hello_world(): + """Print hello world message.""" + print("Hello, World!") + +def add_numbers(a, b): + """Add two numbers and return the result.""" + return a + b + +class Calculator: + """A simple calculator class.""" + + def __init__(self): + self.history = [] + + def add(self, a, b): + result = a + b + self.history.append(f"{a} + {b} = {result}") + return result +''' + + docs = [MockDocument(python_code, "/test/calculator.py", {"language": "python"})] + + try: + chunks = create_ast_chunks(docs, max_chunk_size=200, chunk_overlap=50) + + # Should have multiple chunks due to different functions/classes + assert len(chunks) > 0 + assert all(isinstance(chunk, str) for chunk in chunks) + assert all(len(chunk.strip()) > 0 for chunk in chunks) + + # Check that code structure is somewhat preserved + combined_content = " ".join(chunks) + assert "def hello_world" in combined_content + assert "class Calculator" in combined_content + + except ImportError: + # astchunk not available, should fall back to traditional chunking + chunks = create_ast_chunks(docs, max_chunk_size=200, chunk_overlap=50) + assert len(chunks) > 0 # Should still get chunks from fallback + + def test_create_ast_chunks_fallback_to_traditional(self): + """Test AST chunking falls back to traditional when astchunk is not available.""" + docs = [MockDocument("def test(): pass", "/test/script.py", {"language": "python"})] + + # Mock astchunk import to fail + with patch("chunking.create_ast_chunks"): + # First call (actual test) should import astchunk and potentially fail + # Let's call the actual function to test the import error handling + chunks = create_ast_chunks(docs) + + # Should return some chunks (either from astchunk or fallback) + assert isinstance(chunks, list) + + def test_create_text_chunks_traditional_mode(self): + """Test text chunking in traditional mode.""" + docs = [ + MockDocument("def test(): pass", "/test/script.py"), + MockDocument("This is regular text.", "/test/doc.txt"), + ] + + chunks = create_text_chunks(docs, use_ast_chunking=False, chunk_size=50, chunk_overlap=10) + + assert len(chunks) > 0 + assert all(isinstance(chunk, str) for chunk in chunks) + + def test_create_text_chunks_ast_mode(self): + """Test text chunking in AST mode.""" + docs = [ + MockDocument("def test(): pass", "/test/script.py"), + MockDocument("This is regular text.", "/test/doc.txt"), + ] + + chunks = create_text_chunks( + docs, + use_ast_chunking=True, + ast_chunk_size=100, + ast_chunk_overlap=20, + chunk_size=50, + chunk_overlap=10, + ) + + assert len(chunks) > 0 + assert all(isinstance(chunk, str) for chunk in chunks) + + def test_create_text_chunks_custom_extensions(self): + """Test text chunking with custom code file extensions.""" + docs = [ + MockDocument("function test() {}", "/test/script.js"), # Not in default extensions + MockDocument("Regular text", "/test/doc.txt"), + ] + + # First without custom extensions - should treat .js as text + chunks_without = create_text_chunks(docs, use_ast_chunking=True, code_file_extensions=None) + + # Then with custom extensions - should treat .js as code + chunks_with = create_text_chunks( + docs, use_ast_chunking=True, code_file_extensions=[".js", ".jsx"] + ) + + # Both should return chunks + assert len(chunks_without) > 0 + assert len(chunks_with) > 0 + + +class TestIntegrationWithDocumentRAG: + """Integration tests with the document RAG system.""" + + @pytest.fixture + def temp_code_dir(self): + """Create a temporary directory with sample code files.""" + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Create sample Python file + python_file = temp_path / "example.py" + python_file.write_text(''' +def fibonacci(n): + """Calculate fibonacci number.""" + if n <= 1: + return n + return fibonacci(n-1) + fibonacci(n-2) + +class MathUtils: + @staticmethod + def factorial(n): + if n <= 1: + return 1 + return n * MathUtils.factorial(n-1) +''') + + # Create sample text file + text_file = temp_path / "readme.txt" + text_file.write_text("This is a sample text file for testing purposes.") + + yield temp_path + + @pytest.mark.skipif( + os.environ.get("CI") == "true", + reason="Skip integration tests in CI to avoid dependency issues", + ) + def test_document_rag_with_ast_chunking(self, temp_code_dir): + """Test document RAG with AST chunking enabled.""" + with tempfile.TemporaryDirectory() as index_dir: + cmd = [ + sys.executable, + "apps/document_rag.py", + "--llm", + "simulated", + "--embedding-model", + "facebook/contriever", + "--embedding-mode", + "sentence-transformers", + "--index-dir", + index_dir, + "--data-dir", + str(temp_code_dir), + "--enable-code-chunking", + "--query", + "How does the fibonacci function work?", + ] + + env = os.environ.copy() + env["HF_HUB_DISABLE_SYMLINKS"] = "1" + env["TOKENIZERS_PARALLELISM"] = "false" + + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=300, # 5 minutes + env=env, + ) + + # Should succeed even if astchunk is not available (fallback) + assert result.returncode == 0, f"Command failed: {result.stderr}" + + output = result.stdout + result.stderr + assert "Index saved to" in output or "Using existing index" in output + + except subprocess.TimeoutExpired: + pytest.skip("Test timed out - likely due to model download in CI") + + @pytest.mark.skipif( + os.environ.get("CI") == "true", + reason="Skip integration tests in CI to avoid dependency issues", + ) + def test_code_rag_application(self, temp_code_dir): + """Test the specialized code RAG application.""" + with tempfile.TemporaryDirectory() as index_dir: + cmd = [ + sys.executable, + "apps/code_rag.py", + "--llm", + "simulated", + "--embedding-model", + "facebook/contriever", + "--index-dir", + index_dir, + "--repo-dir", + str(temp_code_dir), + "--query", + "What classes are defined in this code?", + ] + + env = os.environ.copy() + env["HF_HUB_DISABLE_SYMLINKS"] = "1" + env["TOKENIZERS_PARALLELISM"] = "false" + + try: + result = subprocess.run(cmd, capture_output=True, text=True, timeout=300, env=env) + + # Should succeed + assert result.returncode == 0, f"Command failed: {result.stderr}" + + output = result.stdout + result.stderr + assert "Using AST-aware chunking" in output or "traditional chunking" in output + + except subprocess.TimeoutExpired: + pytest.skip("Test timed out - likely due to model download in CI") + + +class TestErrorHandling: + """Test error handling and edge cases.""" + + def test_text_chunking_empty_documents(self): + """Test text chunking with empty document list.""" + chunks = create_text_chunks([]) + assert chunks == [] + + def test_text_chunking_invalid_parameters(self): + """Test text chunking with invalid parameters.""" + docs = [MockDocument("test content")] + + # Should handle negative chunk sizes gracefully + chunks = create_text_chunks( + docs, chunk_size=0, chunk_overlap=0, ast_chunk_size=0, ast_chunk_overlap=0 + ) + + # Should still return some result + assert isinstance(chunks, list) + + def test_create_ast_chunks_no_language(self): + """Test AST chunking with documents missing language metadata.""" + docs = [MockDocument("def test(): pass", "/test/script.py")] # No language set + + chunks = create_ast_chunks(docs) + + # Should fall back to traditional chunking + assert isinstance(chunks, list) + assert len(chunks) >= 0 # May be empty if fallback also fails + + def test_create_ast_chunks_empty_content(self): + """Test AST chunking with empty content.""" + docs = [MockDocument("", "/test/script.py", {"language": "python"})] + + chunks = create_ast_chunks(docs) + + # Should handle empty content gracefully + assert isinstance(chunks, list) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/test_document_rag.py b/tests/test_document_rag.py index ddd0bfd..091767a 100644 --- a/tests/test_document_rag.py +++ b/tests/test_document_rag.py @@ -57,6 +57,51 @@ def test_document_rag_simulated(test_data_dir): assert "This is a simulated answer" in output +@pytest.mark.skipif( + os.environ.get("CI") == "true", + reason="Skip AST chunking tests in CI to avoid dependency issues", +) +def test_document_rag_with_ast_chunking(test_data_dir): + """Test document_rag with AST-aware chunking enabled.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Use a subdirectory that doesn't exist yet to force index creation + index_dir = Path(temp_dir) / "test_ast_index" + cmd = [ + sys.executable, + "apps/document_rag.py", + "--llm", + "simulated", + "--embedding-model", + "facebook/contriever", + "--embedding-mode", + "sentence-transformers", + "--index-dir", + str(index_dir), + "--data-dir", + str(test_data_dir), + "--enable-code-chunking", # Enable AST chunking + "--query", + "What is Pride and Prejudice about?", + ] + + env = os.environ.copy() + env["HF_HUB_DISABLE_SYMLINKS"] = "1" + env["TOKENIZERS_PARALLELISM"] = "false" + + result = subprocess.run(cmd, capture_output=True, text=True, timeout=600, env=env) + + # Check return code + assert result.returncode == 0, f"Command failed: {result.stderr}" + + # Verify output + output = result.stdout + result.stderr + assert "Index saved to" in output or "Using existing index" in output + assert "This is a simulated answer" in output + + # Should mention AST chunking if code files are present + # (might not be relevant for the test data, but command should succeed) + + @pytest.mark.skipif(not os.environ.get("OPENAI_API_KEY"), reason="OpenAI API key not available") @pytest.mark.skipif( os.environ.get("CI") == "true", reason="Skip OpenAI tests in CI to avoid API costs" diff --git a/uv.lock b/uv.lock index 830bba5..00c8263 100644 --- a/uv.lock +++ b/uv.lock @@ -198,6 +198,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/29/5ecc3a15d5a33e31b26c11426c45c501e439cb865d0bff96315d86443b78/appnope-0.1.4-py2.py3-none-any.whl", hash = "sha256:502575ee11cd7a28c0205f379b525beefebab9d161b7c964670864014ed7213c", size = 4321, upload-time = "2024-02-06T09:43:09.663Z" }, ] +[[package]] +name = "astchunk" +version = "0.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, + { name = "numpy", version = "2.3.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "pyrsistent" }, + { name = "tree-sitter", version = "0.23.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "tree-sitter", version = "0.25.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "tree-sitter-c-sharp" }, + { name = "tree-sitter-java" }, + { name = "tree-sitter-python" }, + { name = "tree-sitter-typescript" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/db/2a/7a35e2fac7d550265ae2ee40651425083b37555f921d1a1b77c3f525e0df/astchunk-0.1.0.tar.gz", hash = "sha256:f4dff0ef8b3b3bcfeac363384db1e153f74d4c825dc2e35864abfab027713be4", size = 18093, upload-time = "2025-06-19T04:37:25.34Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/84/5433ab0e933b572750cb16fd7edf3d6c7902b069461a22ec670042752a4d/astchunk-0.1.0-py3-none-any.whl", hash = "sha256:33ada9fc3620807fdda5846fa1948af463f281a60e0d43d4f3782b6dbb416d24", size = 15396, upload-time = "2025-06-19T04:37:23.87Z" }, +] + [[package]] name = "asttokens" version = "3.0.0" @@ -2164,6 +2185,7 @@ name = "leann-workspace" version = "0.1.0" source = { editable = "." } dependencies = [ + { name = "astchunk" }, { name = "boto3" }, { name = "colorama" }, { name = "datasets" }, @@ -2198,6 +2220,12 @@ dependencies = [ { name = "sglang" }, { name = "torch" }, { name = "tqdm" }, + { name = "tree-sitter", version = "0.23.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "tree-sitter", version = "0.25.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "tree-sitter-c-sharp" }, + { name = "tree-sitter-java" }, + { name = "tree-sitter-python" }, + { name = "tree-sitter-typescript" }, { name = "typer" }, ] @@ -2231,6 +2259,7 @@ test = [ [package.metadata] requires-dist = [ + { name = "astchunk", specifier = ">=0.1.0" }, { name = "beautifulsoup4", marker = "extra == 'documents'", specifier = ">=4.13.0" }, { name = "black", marker = "extra == 'dev'", specifier = ">=23.0" }, { name = "boto3" }, @@ -2280,6 +2309,11 @@ requires-dist = [ { name = "sglang" }, { name = "torch" }, { name = "tqdm" }, + { name = "tree-sitter", specifier = ">=0.20.0" }, + { name = "tree-sitter-c-sharp", specifier = ">=0.20.0" }, + { name = "tree-sitter-java", specifier = ">=0.20.0" }, + { name = "tree-sitter-python", specifier = ">=0.20.0" }, + { name = "tree-sitter-typescript", specifier = ">=0.20.0" }, { name = "typer", specifier = ">=0.12.3" }, ] provides-extras = ["dev", "test", "diskann", "documents"] @@ -4441,6 +4475,39 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/7a/097801205b991bc3115e8af1edb850d30aeaf0118520b016354cf5ccd3f6/pypdfium2-4.30.0-py3-none-win_arm64.whl", hash = "sha256:119b2969a6d6b1e8d55e99caaf05290294f2d0fe49c12a3f17102d01c441bd29", size = 2752118, upload-time = "2024-05-09T18:33:15.489Z" }, ] +[[package]] +name = "pyrsistent" +version = "0.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/3a/5031723c09068e9c8c2f0bc25c3a9245f2b1d1aea8396c787a408f2b95ca/pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4", size = 103642, upload-time = "2023-10-25T21:06:56.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/19/c343b14061907b629b765444b6436b160e2bd4184d17d4804bbe6381f6be/pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce", size = 83416, upload-time = "2023-10-25T21:06:04.579Z" }, + { url = "https://files.pythonhosted.org/packages/9f/4f/8342079ea331031ef9ed57edd312a9ad283bcc8adfaf268931ae356a09a6/pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f", size = 118021, upload-time = "2023-10-25T21:06:06.953Z" }, + { url = "https://files.pythonhosted.org/packages/d7/b7/64a125c488243965b7c5118352e47c6f89df95b4ac306d31cee409153d57/pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34", size = 117747, upload-time = "2023-10-25T21:06:08.5Z" }, + { url = "https://files.pythonhosted.org/packages/fe/a5/43c67bd5f80df9e7583042398d12113263ec57f27c0607abe9d78395d18f/pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b", size = 114524, upload-time = "2023-10-25T21:06:10.728Z" }, + { url = "https://files.pythonhosted.org/packages/8a/98/b382a87e89ca839106d874f7bf78d226b3eedb26735eb6f751f1a3375f21/pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f", size = 60780, upload-time = "2023-10-25T21:06:12.14Z" }, + { url = "https://files.pythonhosted.org/packages/37/8a/23e2193f7adea6901262e3cf39c7fe18ac0c446176c0ff0e19aeb2e9681e/pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7", size = 63310, upload-time = "2023-10-25T21:06:13.598Z" }, + { url = "https://files.pythonhosted.org/packages/df/63/7544dc7d0953294882a5c587fb1b10a26e0c23d9b92281a14c2514bac1f7/pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958", size = 83481, upload-time = "2023-10-25T21:06:15.238Z" }, + { url = "https://files.pythonhosted.org/packages/ae/a0/49249bc14d71b1bf2ffe89703acfa86f2017c25cfdabcaea532b8c8a5810/pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8", size = 120222, upload-time = "2023-10-25T21:06:17.144Z" }, + { url = "https://files.pythonhosted.org/packages/a1/94/9808e8c9271424120289b9028a657da336ad7e43da0647f62e4f6011d19b/pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a", size = 120002, upload-time = "2023-10-25T21:06:18.727Z" }, + { url = "https://files.pythonhosted.org/packages/3f/f6/9ecfb78b2fc8e2540546db0fe19df1fae0f56664a5958c21ff8861b0f8da/pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224", size = 116850, upload-time = "2023-10-25T21:06:20.424Z" }, + { url = "https://files.pythonhosted.org/packages/83/c8/e6d28bc27a0719f8eaae660357df9757d6e9ca9be2691595721de9e8adfc/pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656", size = 60775, upload-time = "2023-10-25T21:06:21.815Z" }, + { url = "https://files.pythonhosted.org/packages/98/87/c6ef52ff30388f357922d08de012abdd3dc61e09311d88967bdae23ab657/pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee", size = 63306, upload-time = "2023-10-25T21:06:22.874Z" }, + { url = "https://files.pythonhosted.org/packages/15/ee/ff2ed52032ac1ce2e7ba19e79bd5b05d152ebfb77956cf08fcd6e8d760ea/pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e", size = 83537, upload-time = "2023-10-25T21:06:24.17Z" }, + { url = "https://files.pythonhosted.org/packages/80/f1/338d0050b24c3132bcfc79b68c3a5f54bce3d213ecef74d37e988b971d8a/pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e", size = 122615, upload-time = "2023-10-25T21:06:25.815Z" }, + { url = "https://files.pythonhosted.org/packages/07/3a/e56d6431b713518094fae6ff833a04a6f49ad0fbe25fb7c0dc7408e19d20/pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3", size = 122335, upload-time = "2023-10-25T21:06:28.631Z" }, + { url = "https://files.pythonhosted.org/packages/4a/bb/5f40a4d5e985a43b43f607250e766cdec28904682c3505eb0bd343a4b7db/pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d", size = 118510, upload-time = "2023-10-25T21:06:30.718Z" }, + { url = "https://files.pythonhosted.org/packages/1c/13/e6a22f40f5800af116c02c28e29f15c06aa41cb2036f6a64ab124647f28b/pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174", size = 60865, upload-time = "2023-10-25T21:06:32.742Z" }, + { url = "https://files.pythonhosted.org/packages/75/ef/2fa3b55023ec07c22682c957808f9a41836da4cd006b5f55ec76bf0fbfa6/pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d", size = 63239, upload-time = "2023-10-25T21:06:34.035Z" }, + { url = "https://files.pythonhosted.org/packages/18/0c/289126299fcebf54fd01d385fb5176c328fef2c4233139c23dd48346e992/pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce", size = 83379, upload-time = "2023-10-25T21:06:45.585Z" }, + { url = "https://files.pythonhosted.org/packages/4e/45/62639d53ac09eaafc00f2e5845565e70d3eddb2d296337a77637186ca03e/pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0", size = 117740, upload-time = "2023-10-25T21:06:46.918Z" }, + { url = "https://files.pythonhosted.org/packages/ab/12/24b9a6ef7b991b6722756e0aa169a39463af2b8ed0fb526f0a00aae34ea4/pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022", size = 117457, upload-time = "2023-10-25T21:06:48.911Z" }, + { url = "https://files.pythonhosted.org/packages/19/3c/ab06510f86bc0934b77ade41948924ff1f33dcd3433f32feca2028218837/pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca", size = 114280, upload-time = "2023-10-25T21:06:50.503Z" }, + { url = "https://files.pythonhosted.org/packages/ee/b1/1275bbfb929854d20e72aa2bbfb50ea3b1d7d41a95848b353691875e2817/pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f", size = 60764, upload-time = "2023-10-25T21:06:52.093Z" }, + { url = "https://files.pythonhosted.org/packages/28/77/0d7af973c0e3b1b83d8b45943601f77f85b943007e3a4d8744f7102c652b/pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf", size = 63289, upload-time = "2023-10-25T21:06:53.221Z" }, + { url = "https://files.pythonhosted.org/packages/23/88/0acd180010aaed4987c85700b7cc17f9505f3edb4e5873e4dc67f613e338/pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b", size = 58106, upload-time = "2023-10-25T21:06:54.387Z" }, +] + [[package]] name = "pytest" version = "8.4.1" @@ -5827,6 +5894,165 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cc/34/4d82dc596764de9d14285f8ed53b50896bf05fbbcd71a82c6d174b3ab8c7/transformers-4.54.0-py3-none-any.whl", hash = "sha256:c96e607f848625965b76c677b2c2576f2c7b7097c1c5292b281919d90675a25e", size = 11176597, upload-time = "2025-07-25T18:58:17.677Z" }, ] +[[package]] +name = "tree-sitter" +version = "0.23.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +sdist = { url = "https://files.pythonhosted.org/packages/0f/50/fd5fafa42b884f741b28d9e6fd366c3f34e15d2ed3aa9633b34e388379e2/tree-sitter-0.23.2.tar.gz", hash = "sha256:66bae8dd47f1fed7bdef816115146d3a41c39b5c482d7bad36d9ba1def088450", size = 166800, upload-time = "2024-10-24T15:31:02.238Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/04/2068a7b725265ecfcbf63ecdae038f1d4124ebccd55b8a7ce145b70e2b6a/tree_sitter-0.23.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3a937f5d8727bc1c74c4bf2a9d1c25ace049e8628273016ad0d45914ae904e10", size = 139289, upload-time = "2024-10-24T15:29:59.27Z" }, + { url = "https://files.pythonhosted.org/packages/a8/07/a5b943121f674fe1ac77694a698e71ce95353830c1f3f4ce45da7ef3e406/tree_sitter-0.23.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2c7eae7fe2af215645a38660d2d57d257a4c461fe3ec827cca99a79478284e80", size = 132379, upload-time = "2024-10-24T15:30:01.437Z" }, + { url = "https://files.pythonhosted.org/packages/d4/96/fcc72c33d464a2d722db1e95b74a53ced771a47b3cfde60aced29764a783/tree_sitter-0.23.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a71d607595270b6870eaf778a1032d146b2aa79bfcfa60f57a82a7b7584a4c7", size = 552884, upload-time = "2024-10-24T15:30:02.672Z" }, + { url = "https://files.pythonhosted.org/packages/d0/af/b0e787a52767155b4643a55d6de03c1e4ae77abb61e1dc1629ad983e0a40/tree_sitter-0.23.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fe9b9ea7a0aa23b52fd97354da95d1b2580065bc12a4ac868f9164a127211d6", size = 566561, upload-time = "2024-10-24T15:30:04.073Z" }, + { url = "https://files.pythonhosted.org/packages/65/fd/05e966b5317b1c6679c071c5b0203f28af9d26c9363700cb9682e1bcf343/tree_sitter-0.23.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d74d00a8021719eae14d10d1b1e28649e15d8b958c01c2b2c3dad7a2ebc4dbae", size = 558273, upload-time = "2024-10-24T15:30:06.177Z" }, + { url = "https://files.pythonhosted.org/packages/60/bc/19145efdf3f47711aa3f1bf06f0b50593f97f1108550d38694841fd97b7c/tree_sitter-0.23.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:6de18d8d8a7f67ab71f472d1fcb01cc506e080cbb5e13d52929e4b6fdce6bbee", size = 569176, upload-time = "2024-10-24T15:30:07.902Z" }, + { url = "https://files.pythonhosted.org/packages/32/08/3553d8e488ae9284a0762effafb7d2639a306e184963b7f99853923084d6/tree_sitter-0.23.2-cp310-cp310-win_amd64.whl", hash = "sha256:12b60dca70d2282af942b650a6d781be487485454668c7c956338a367b98cdee", size = 117902, upload-time = "2024-10-24T15:30:09.675Z" }, + { url = "https://files.pythonhosted.org/packages/1d/39/836fa485e985c33e8aa1cc3abbf7a84be1c2c382e69547a765631fdd7ce3/tree_sitter-0.23.2-cp310-cp310-win_arm64.whl", hash = "sha256:3346a4dd0447a42aabb863443b0fd8c92b909baf40ed2344fae4b94b625d5955", size = 102644, upload-time = "2024-10-24T15:30:11.484Z" }, + { url = "https://files.pythonhosted.org/packages/55/8d/2d4fb04408772be0919441d66f700673ce7cb76b9ab6682e226d740fb88d/tree_sitter-0.23.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:91fda41d4f8824335cc43c64e2c37d8089c8c563bd3900a512d2852d075af719", size = 139142, upload-time = "2024-10-24T15:30:12.627Z" }, + { url = "https://files.pythonhosted.org/packages/32/52/b8a44bfff7b0203256e5dbc8d3a372ee8896128b8ed7d3a89e1ef17b2065/tree_sitter-0.23.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:92b2b489d5ce54b41f94c6f23fbaf592bd6e84dc2877048fd1cb060480fa53f7", size = 132198, upload-time = "2024-10-24T15:30:13.893Z" }, + { url = "https://files.pythonhosted.org/packages/5d/54/746f2ee5acf6191a4a0be7f5843329f0d713bfe5196f5fc6fe2ea69cb44c/tree_sitter-0.23.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64859bd4aa1567d0d6016a811b2b49c59d4a4427d096e3d8c84b2521455f62b7", size = 554303, upload-time = "2024-10-24T15:30:15.334Z" }, + { url = "https://files.pythonhosted.org/packages/2f/5a/3169d9933be813776a9b4b3f2e671d3d50fa27e589dee5578f6ecef7ff6d/tree_sitter-0.23.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:614590611636044e071d3a0b748046d52676dbda3bc9fa431216231e11dd98f7", size = 567626, upload-time = "2024-10-24T15:30:17.12Z" }, + { url = "https://files.pythonhosted.org/packages/32/0d/23f363b3b0bc3fa0e7a4a294bf119957ac1ab02737d57815e1e8b7b3e196/tree_sitter-0.23.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08466953c78ae57be61057188fb88c89791b0a562856010228e0ccf60e2ac453", size = 559803, upload-time = "2024-10-24T15:30:18.921Z" }, + { url = "https://files.pythonhosted.org/packages/6f/b3/1ffba0f17a7ff2c9114d91a1ecc15e0748f217817797564d31fbb61d7458/tree_sitter-0.23.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:8a33f03a562de91f7fd05eefcedd8994a06cd44c62f7aabace811ad82bc11cbd", size = 570987, upload-time = "2024-10-24T15:30:21.116Z" }, + { url = "https://files.pythonhosted.org/packages/59/4b/085bcb8a11ea18003aacc4dbc91c301d1536c5e2deedb95393e8ef26f1f7/tree_sitter-0.23.2-cp311-cp311-win_amd64.whl", hash = "sha256:03b70296b569ef64f7b92b42ca5da9bf86d81bee2afd480bea35092687f51dae", size = 117771, upload-time = "2024-10-24T15:30:22.38Z" }, + { url = "https://files.pythonhosted.org/packages/4b/e5/90adc4081f49ccb6bea89a800dc9b0dcc5b6953b0da423e8eff28f63fddf/tree_sitter-0.23.2-cp311-cp311-win_arm64.whl", hash = "sha256:7cb4bb953ea7c0b50eeafc4454783e030357179d2a93c3dd5ebed2da5588ddd0", size = 102555, upload-time = "2024-10-24T15:30:23.534Z" }, + { url = "https://files.pythonhosted.org/packages/07/a7/57e0fe87b49a78c670a7b4483f70e44c000c65c29b138001096b22e7dd87/tree_sitter-0.23.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a014498b6a9e6003fae8c6eb72f5927d62da9dcb72b28b3ce8cd15c6ff6a6572", size = 139259, upload-time = "2024-10-24T15:30:24.941Z" }, + { url = "https://files.pythonhosted.org/packages/b4/b9/bc8513d818ffb54993a017a36c8739300bc5739a13677acf90b54995e7db/tree_sitter-0.23.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:04f8699b131d4bcbe3805c37e4ef3d159ee9a82a0e700587625623999ba0ea53", size = 131951, upload-time = "2024-10-24T15:30:26.176Z" }, + { url = "https://files.pythonhosted.org/packages/d7/6a/eab01bb6b1ce3c9acf16d72922ffc29a904af485eb3e60baf3a3e04edd30/tree_sitter-0.23.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4471577df285059c71686ecb208bc50fb472099b38dcc8e849b0e86652891e87", size = 557952, upload-time = "2024-10-24T15:30:27.389Z" }, + { url = "https://files.pythonhosted.org/packages/bd/95/f2f73332623cf63200d57800f85273170bc5f99d28ea3f234afd5b0048df/tree_sitter-0.23.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f342c925290dd4e20ecd5787ef7ae8749981597ab364783a1eb73173efe65226", size = 571199, upload-time = "2024-10-24T15:30:28.879Z" }, + { url = "https://files.pythonhosted.org/packages/04/ac/bd6e6cfdd0421156e86f5c93848629af1c7323083077e1a95b27d32d5811/tree_sitter-0.23.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a4e9e53d07dd076bede72e4f7d3a0173d7b9ad6576572dd86da008a740a9bb22", size = 562129, upload-time = "2024-10-24T15:30:30.199Z" }, + { url = "https://files.pythonhosted.org/packages/7b/bd/8a9edcbcf8a76b0bf58e3b927ed291e3598e063d56667367762833cc8709/tree_sitter-0.23.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8caebe65bc358759dac2500d8f8feed3aed939c4ade9a684a1783fe07bc7d5db", size = 574307, upload-time = "2024-10-24T15:30:32.085Z" }, + { url = "https://files.pythonhosted.org/packages/0c/c2/3fb2c6c0ae2f59a7411dc6d3e7945e3cb6f34c8552688708acc8b2b13f83/tree_sitter-0.23.2-cp312-cp312-win_amd64.whl", hash = "sha256:fc5a72eb50d43485000dbbb309acb350467b7467e66dc747c6bb82ce63041582", size = 117858, upload-time = "2024-10-24T15:30:33.353Z" }, + { url = "https://files.pythonhosted.org/packages/e2/18/4ca2c0f4a0c802ebcb3a92264cc436f1d54b394fa24dfa76bf57cdeaca9e/tree_sitter-0.23.2-cp312-cp312-win_arm64.whl", hash = "sha256:a0320eb6c7993359c5f7b371d22719ccd273f440d41cf1bd65dac5e9587f2046", size = 102496, upload-time = "2024-10-24T15:30:34.782Z" }, + { url = "https://files.pythonhosted.org/packages/ba/c6/4ead9ce3113a7c27f37a2bdef163c09757efbaa85adbdfe7b3fbf0317c57/tree_sitter-0.23.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:eff630dddee7ba05accb439b17e559e15ce13f057297007c246237ceb6306332", size = 139266, upload-time = "2024-10-24T15:30:35.946Z" }, + { url = "https://files.pythonhosted.org/packages/76/c9/b4197c5b0c1d6ba648202a547846ac910a53163b69a459504b2aa6cdb76e/tree_sitter-0.23.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4780ba8f3894f2dea869fad2995c2aceab3fd5ab9e6a27c45475d2acd7f7e84e", size = 131959, upload-time = "2024-10-24T15:30:37.646Z" }, + { url = "https://files.pythonhosted.org/packages/99/94/0f7c5580d2adff3b57d36f1998725b0caf6cf1af50ceafc00c6cdbc2fef6/tree_sitter-0.23.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f0b609460b8e3e256361fb12e94fae5b728cb835b16f0f9d590b5aadbf9d109b", size = 557582, upload-time = "2024-10-24T15:30:39.019Z" }, + { url = "https://files.pythonhosted.org/packages/97/8a/f73ff06959d43fd47fc283cbcc4d8efa6550b2cc431d852b184504992447/tree_sitter-0.23.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78d070d8eaeaeb36cf535f55e5578fddbfc3bf53c1980f58bf1a99d57466b3b5", size = 570891, upload-time = "2024-10-24T15:30:40.432Z" }, + { url = "https://files.pythonhosted.org/packages/b8/86/bbda5ad09b88051ff7bf3275622a2f79bc4f728b4c283ff8b93b8fcdf36d/tree_sitter-0.23.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:878580b2ad5054c410ba3418edca4d34c81cc26706114d8f5b5541688bc2d785", size = 562343, upload-time = "2024-10-24T15:30:43.045Z" }, + { url = "https://files.pythonhosted.org/packages/ca/55/b404fa49cb5c2926ad6fe1cac033dd486ef69f1afeb7828452d21e1e05c1/tree_sitter-0.23.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:29224bdc2a3b9af535b7725e249d3ee291b2e90708e82832e73acc175e40dc48", size = 574407, upload-time = "2024-10-24T15:30:45.018Z" }, + { url = "https://files.pythonhosted.org/packages/c2/c8/eea2104443ab973091107ef3e730683bd8e6cb51dd025cef853d3fff9dae/tree_sitter-0.23.2-cp313-cp313-win_amd64.whl", hash = "sha256:c58d89348162fbc3aea1fe6511a66ee189fc0e4e4bbe937026f29e4ecef17763", size = 117854, upload-time = "2024-10-24T15:30:47.817Z" }, + { url = "https://files.pythonhosted.org/packages/89/4d/1728d9ce32a1d851081911b7e47830f5e740431f2bb920f54bb8c26175bc/tree_sitter-0.23.2-cp313-cp313-win_arm64.whl", hash = "sha256:0ff2037be5edab7801de3f6a721b9cf010853f612e2008ee454e0e0badb225a6", size = 102492, upload-time = "2024-10-24T15:30:48.892Z" }, + { url = "https://files.pythonhosted.org/packages/cb/ab/b39173a47d498cc6276e303c865f4a222134ceae890bd3c1b29427489805/tree_sitter-0.23.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a5db8e585205faef8bf219da77d8993e2ef04d08eda2e3c8ad7e4df8297ee344", size = 139550, upload-time = "2024-10-24T15:30:50.516Z" }, + { url = "https://files.pythonhosted.org/packages/4c/34/fa8f5b862dd7a6014fd5578810178e8f7601830cabb6d65d2aba050c2df1/tree_sitter-0.23.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9dbd110a30cf28be5da734ae4cd0e9031768228dbf6a79f2973962aa51de4ec7", size = 132686, upload-time = "2024-10-24T15:30:51.779Z" }, + { url = "https://files.pythonhosted.org/packages/98/b9/ccdddf35705fc23395caa71557f767e0753d38afe4b5bb99efddbf62bb22/tree_sitter-0.23.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:569514b9a996a0fd458b3a891c46ca125298be0c03cf82f2b6f0c13d5d8f25dc", size = 554958, upload-time = "2024-10-24T15:30:53.327Z" }, + { url = "https://files.pythonhosted.org/packages/8d/ba/20ae9079bdfc5cfac28b39d945a6c354c8e1385e73aec8142db6c53b635c/tree_sitter-0.23.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a357ed98a74e47787b812df99a74a2c35c0fe11e55c2095cc01d1cad144ef552", size = 568162, upload-time = "2024-10-24T15:30:54.667Z" }, + { url = "https://files.pythonhosted.org/packages/40/00/b16bf6cf88c47c1b6c8e1cce1eb9e90badb5db9e5252ae0970d858d02592/tree_sitter-0.23.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c2dfb8e8f760f4cc67888d03ef9e2dbd3353245f67f5efba375c2a14d944ac0e", size = 560278, upload-time = "2024-10-24T15:30:56.49Z" }, + { url = "https://files.pythonhosted.org/packages/7a/8f/27ab9b96cc0261af78b080ec8a9846a38e216360ec38774ea27eba35bd3c/tree_sitter-0.23.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:3ead958df87a21d706903987e665e9e0e5df7b2c5021ff69ea349826840adc6a", size = 571255, upload-time = "2024-10-24T15:30:58.254Z" }, + { url = "https://files.pythonhosted.org/packages/44/e0/95a3d66a7e5bb229574484ab10c6dc99d1c7a32972b890d194076e30dc4f/tree_sitter-0.23.2-cp39-cp39-win_amd64.whl", hash = "sha256:611cae16be332213c0e6ece72c0bfca202e30ff320a8b309b1526c6cb79ee4ba", size = 118232, upload-time = "2024-10-24T15:30:59.965Z" }, + { url = "https://files.pythonhosted.org/packages/10/b5/9eaf794fc71490573ab14a366affca415bc1ddbf86a14d78e54583db4254/tree_sitter-0.23.2-cp39-cp39-win_arm64.whl", hash = "sha256:b848e0fdd522fbb8888cdb4f4d93f8fad97ae10d70c122fb922e51363c7febcd", size = 102787, upload-time = "2024-10-24T15:31:01.084Z" }, +] + +[[package]] +name = "tree-sitter" +version = "0.25.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/89/2b/02a642e67605b9dd59986b00d13a076044dede04025a243f0592ac79d68c/tree-sitter-0.25.1.tar.gz", hash = "sha256:cd761ad0e4d1fc88a4b1b8083bae06d4f973acf6f5f29bbf13ea9609c1dec9c1", size = 177874, upload-time = "2025-08-05T17:14:34.193Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/6c/6160ca15926d11a6957d8bee887f477f3c1d9bc5272c863affc0b50b9cff/tree_sitter-0.25.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a15d62ffdb095d509bda8c140c1ddd0cc80f0c67f92b87fcc96cd242dc0c71ea", size = 146692, upload-time = "2025-08-05T17:13:54.559Z" }, + { url = "https://files.pythonhosted.org/packages/81/4a/e5eb39fe73a514a13bf94acee97925de296d673dace00557763cbbdc938f/tree_sitter-0.25.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1d938f0a1ffad1206a1a569b0501345eeca81cae0a4487bb485e53768b02f24e", size = 141015, upload-time = "2025-08-05T17:13:55.807Z" }, + { url = "https://files.pythonhosted.org/packages/63/22/c8e3ba245e5cdb8c951482028a7ee99d141302047b708dc9d670f0fafd85/tree_sitter-0.25.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba8cea296de5dcb384b9a15cf526985ac8339c81da51c7e29a251d82071f5ee9", size = 599462, upload-time = "2025-08-05T17:13:56.984Z" }, + { url = "https://files.pythonhosted.org/packages/c2/91/c866c3d278ee86354fd81fd055b5d835c510b0e9af07e1cf7e48e2f946b0/tree_sitter-0.25.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:387fd2bd8657d69e877618dc199c18e2d6fe073b8f5c59e23435f3baee4ee10a", size = 627062, upload-time = "2025-08-05T17:13:58.363Z" }, + { url = "https://files.pythonhosted.org/packages/90/96/ac010f72778dae60381ab5fcca9651ac72647d582db0b027ca6c56116920/tree_sitter-0.25.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:afa49e51f82b58ae2c1291d6b79ca31e0fb36c04bd9a20d89007472edfb70136", size = 623788, upload-time = "2025-08-05T17:13:59.431Z" }, + { url = "https://files.pythonhosted.org/packages/0e/29/190bdfd54a564a2e43a702884ad5679f4578c481a46161f9f335dd390a70/tree_sitter-0.25.1-cp310-cp310-win_amd64.whl", hash = "sha256:77be45f666adf284914510794b41100decccd71dba88010c03dc2bb0d653acec", size = 127253, upload-time = "2025-08-05T17:14:00.446Z" }, + { url = "https://files.pythonhosted.org/packages/da/60/7daca5ccf65fb204c9f2cc2907db6aeaf1cb42aa605427580c17a38a53b3/tree_sitter-0.25.1-cp310-cp310-win_arm64.whl", hash = "sha256:72badac2de4e81ae0df5efe14ec5003bd4df3e48e7cf84dbd9df3a54599ba371", size = 113930, upload-time = "2025-08-05T17:14:01.623Z" }, + { url = "https://files.pythonhosted.org/packages/17/dc/0dabb75d249108fb9062d6e9e791e4ad8e9ae5c095e06dd8af770bc07902/tree_sitter-0.25.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:33a8fbaeb2b5049cf5318306ab8b16ab365828b2b21ee13678c29e0726a1d27a", size = 146696, upload-time = "2025-08-05T17:14:02.408Z" }, + { url = "https://files.pythonhosted.org/packages/da/d0/b7305a05d65dbcfce7a97a93252bf7384f09800866e9de55a625c76e0257/tree_sitter-0.25.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:797bbbc686d8d3722d25ee0108ad979bda6ad3e1025859ce2ee290e517816bd4", size = 141014, upload-time = "2025-08-05T17:14:03.58Z" }, + { url = "https://files.pythonhosted.org/packages/84/d0/d0d8bd13c44ef6379499712a3f5e3930e7db11e5c8eb2af8655e288597a3/tree_sitter-0.25.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:629fc2ae3f5954b0f6a7b42ee3fcd8f34b68ea161e9f02fa5bf709cbbac996d3", size = 604339, upload-time = "2025-08-05T17:14:04.722Z" }, + { url = "https://files.pythonhosted.org/packages/c5/13/22869a6da25ffe2dfff922712605e72a9c3481109a93f4218bea1bc65f35/tree_sitter-0.25.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4257018c42a33a7935a5150d678aac05c6594347d6a6e6dbdf7e2ef4ae985213", size = 631593, upload-time = "2025-08-05T17:14:06.043Z" }, + { url = "https://files.pythonhosted.org/packages/ec/0c/f4590fc08422768fc57456a85c932888a02e7a13540574859308611be1cf/tree_sitter-0.25.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4027854c9feee2a3bb99642145ba04ce95d75bd17e292911c93a488cb28d0a04", size = 629265, upload-time = "2025-08-05T17:14:07.045Z" }, + { url = "https://files.pythonhosted.org/packages/a7/a8/ee9305ce9a7417715cbf038fdcc4fdb6042e30065c9837bdcf36be440388/tree_sitter-0.25.1-cp311-cp311-win_amd64.whl", hash = "sha256:183faaedcee5f0a3ba39257fa81749709d5eb7cf92c2c050b36ff38468d1774c", size = 127210, upload-time = "2025-08-05T17:14:08.331Z" }, + { url = "https://files.pythonhosted.org/packages/48/64/6a39882f534373873ef3dba8a1a8f47dc3bfb39ee63784eac2e789b404c4/tree_sitter-0.25.1-cp311-cp311-win_arm64.whl", hash = "sha256:6a3800235535a2532ce392ed0d8e6f698ee010e73805bdeac2f249da8246bab6", size = 113928, upload-time = "2025-08-05T17:14:09.376Z" }, + { url = "https://files.pythonhosted.org/packages/45/79/6dea0c098879d99f41ba919da1ea46e614fb4bf9c4d591450061aeec6fcb/tree_sitter-0.25.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:9362a202144075b54f7c9f07e0b0e44a61eed7ee19e140c506b9e64c1d21ed58", size = 146928, upload-time = "2025-08-05T17:14:10.522Z" }, + { url = "https://files.pythonhosted.org/packages/15/30/8002f4e76c7834a6101895ff7524ea29ab4f1f1da1270260ef52e2319372/tree_sitter-0.25.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:593f22529f34dd04de02f56ea6d7c2c8ec99dfab25b58be893247c1090dedd60", size = 140802, upload-time = "2025-08-05T17:14:11.38Z" }, + { url = "https://files.pythonhosted.org/packages/38/ec/d297ad9d4a4b26f551a5ca49afe48fdbcb20f058c2eff8d8463ad6c0eed1/tree_sitter-0.25.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb6849f76e1cbfa223303fa680da533d452e378d5fe372598e4752838ca7929", size = 606762, upload-time = "2025-08-05T17:14:12.264Z" }, + { url = "https://files.pythonhosted.org/packages/4a/1c/05a623cfb420b10d5f782d4ec064cf00fbfa9c21b8526ca4fd042f80acff/tree_sitter-0.25.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:034d4544bb0f82e449033d76dd083b131c3f9ecb5e37d3475f80ae55e8f382bd", size = 634632, upload-time = "2025-08-05T17:14:13.21Z" }, + { url = "https://files.pythonhosted.org/packages/c5/e0/f05fd5a2331c16d428efb8eef32dfb80dc6565438146e34e9a235ecd7925/tree_sitter-0.25.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:46a9b721560070f2f980105266e28a17d3149485582cdba14d66dca14692e932", size = 630756, upload-time = "2025-08-05T17:14:14.673Z" }, + { url = "https://files.pythonhosted.org/packages/b2/fc/79f3c5d53d1721b95ab6cda0368192a4f1d367e3a5ff7ac21d77e9841782/tree_sitter-0.25.1-cp312-cp312-win_amd64.whl", hash = "sha256:9a5c522b1350a626dc1cbc5dc203133caeaa114d3f65e400445e8b02f18b343b", size = 127157, upload-time = "2025-08-05T17:14:15.59Z" }, + { url = "https://files.pythonhosted.org/packages/24/b7/07c4e3f71af0096db6c2ecd83e7d61584e3891c79cb39b208082312d1d60/tree_sitter-0.25.1-cp312-cp312-win_arm64.whl", hash = "sha256:43e7b8e83f9fc29ca62e7d2aa8c38e3fa806ff3fc65e0d501d18588dc1509888", size = 113910, upload-time = "2025-08-05T17:14:16.385Z" }, + { url = "https://files.pythonhosted.org/packages/3f/d3/bfb08aab9c7daed2715f303cc017329e3512bb77678cc28829681decadd2/tree_sitter-0.25.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ae1eebc175e6a50b38b0e0385cdc26e92ac0bff9b32ee1c0619bbbf6829d57ea", size = 146920, upload-time = "2025-08-05T17:14:17.483Z" }, + { url = "https://files.pythonhosted.org/packages/f9/36/7f897c50489c38665255579646fca8191e1b9e5a29ac9cf11022e42e1e2b/tree_sitter-0.25.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:9e0ae03c4f132f1bffb2bc40b1bb28742785507da693ab04da8531fe534ada9c", size = 140782, upload-time = "2025-08-05T17:14:18.594Z" }, + { url = "https://files.pythonhosted.org/packages/16/e6/85012113899296b8e0789ae94f562d3971d7d3df989e8bec6128749394e1/tree_sitter-0.25.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:acf571758be0a71046a61a0936cb815f15b13e0ae7ec6d08398e4aa1560b371d", size = 607590, upload-time = "2025-08-05T17:14:19.782Z" }, + { url = "https://files.pythonhosted.org/packages/49/93/605b08dc4cf76d08cfacebc30a88467c6526ea5c94592c25240518e38b71/tree_sitter-0.25.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:632910847e3f8ae35841f92cba88a9a1b8bc56ecc1514a5affebf7951fa0fc0a", size = 635553, upload-time = "2025-08-05T17:14:21.107Z" }, + { url = "https://files.pythonhosted.org/packages/ce/27/123667f756bb32168507c940db9040104c606fbb0214397d3c20cf985073/tree_sitter-0.25.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a99ecef7771afb118b2a8435c8ba67ea7a085c60d5d33dc0a4794ed882e5f7df", size = 630844, upload-time = "2025-08-05T17:14:22.078Z" }, + { url = "https://files.pythonhosted.org/packages/2f/53/180b0ed74153a3c9a23967f54774d5930c2e0b67671ae4ca0d4d35ba18ac/tree_sitter-0.25.1-cp313-cp313-win_amd64.whl", hash = "sha256:c1d6393454d1f9d4195c74e40a487640cd4390cd4aee90837485f932a1a0f40c", size = 127159, upload-time = "2025-08-05T17:14:23.061Z" }, + { url = "https://files.pythonhosted.org/packages/32/fb/b8b7b5122ac4a80cd689a5023f2416910e10f9534ace1cdf0020a315d40d/tree_sitter-0.25.1-cp313-cp313-win_arm64.whl", hash = "sha256:c1d2dbf7d12426b71ff49739f599c355f4de338a5c0ab994de2a1d290f6e0b20", size = 113920, upload-time = "2025-08-05T17:14:23.879Z" }, + { url = "https://files.pythonhosted.org/packages/70/8c/cb851da552baf4215baf96443e5e9e39095083a95bc05c4444e640fe0fe8/tree_sitter-0.25.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:32cee52264d9ecf98885fcac0185ac63e16251b31dd8b4a3b8d8071173405f8f", size = 146775, upload-time = "2025-08-05T17:14:25.064Z" }, + { url = "https://files.pythonhosted.org/packages/f3/59/002c89df1e8f1664b82023e5d0c06de97fff5c2a2e33dce1a241c8909758/tree_sitter-0.25.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ae024d8ccfef51e61c44a81af7a48670601430701c24f450bea10f4b4effd8d1", size = 140787, upload-time = "2025-08-05T17:14:25.914Z" }, + { url = "https://files.pythonhosted.org/packages/39/48/c9e6deb88f3c7f16963ef205e5b8e3ea7f5effd048b4515d09738c7b032b/tree_sitter-0.25.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d025c56c393cea660df9ef33ca60329952a1f8ee6212d21b2b390dfec08a3874", size = 609173, upload-time = "2025-08-05T17:14:26.817Z" }, + { url = "https://files.pythonhosted.org/packages/53/a8/b782576d7ea081a87285d974005155da03b6d0c66283fe1e3a5e0dd4bd98/tree_sitter-0.25.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:044aa23ea14f337809821bea7467f33f4c6d351739dca76ba0cbe4d0154d8662", size = 635994, upload-time = "2025-08-05T17:14:28.343Z" }, + { url = "https://files.pythonhosted.org/packages/70/0a/c5b6c9cdb7bd4bf0c3d2bd494fcf356acc53f8e63007dc2a836d95bbe964/tree_sitter-0.25.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1863d96704eb002df4ad3b738294ae8bd5dcf8cefb715da18bff6cb2d33d978e", size = 630944, upload-time = "2025-08-05T17:14:31.123Z" }, + { url = "https://files.pythonhosted.org/packages/12/2a/d0b097157c2d487f5e6293dae2c106ec9ede792a6bb780249e81432e754d/tree_sitter-0.25.1-cp314-cp314-win_amd64.whl", hash = "sha256:a40a481e28e1afdbc455932d61e49ffd4163aafa83f4a3deb717524a7786197e", size = 130831, upload-time = "2025-08-05T17:14:32.458Z" }, + { url = "https://files.pythonhosted.org/packages/ce/33/3591e7b22dd49f46ae4fdee1db316ecefd0486cae880c5b497a55f0ccb24/tree_sitter-0.25.1-cp314-cp314-win_arm64.whl", hash = "sha256:f7b68f584336b39b2deab9896b629dddc3c784170733d3409f01fe825e9c04eb", size = 117376, upload-time = "2025-08-05T17:14:33.283Z" }, +] + +[[package]] +name = "tree-sitter-c-sharp" +version = "0.23.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/85/a61c782afbb706a47d990eaee6977e7c2bd013771c5bf5c81c617684f286/tree_sitter_c_sharp-0.23.1.tar.gz", hash = "sha256:322e2cfd3a547a840375276b2aea3335fa6458aeac082f6c60fec3f745c967eb", size = 1317728, upload-time = "2024-11-11T05:25:32.535Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/04/f6c2df4c53a588ccd88d50851155945cff8cd887bd70c175e00aaade7edf/tree_sitter_c_sharp-0.23.1-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:2b612a6e5bd17bb7fa2aab4bb6fc1fba45c94f09cb034ab332e45603b86e32fd", size = 372235, upload-time = "2024-11-11T05:25:19.424Z" }, + { url = "https://files.pythonhosted.org/packages/99/10/1aa9486f1e28fc22810fa92cbdc54e1051e7f5536a5e5b5e9695f609b31e/tree_sitter_c_sharp-0.23.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a8b98f62bc53efcd4d971151950c9b9cd5cbe3bacdb0cd69fdccac63350d83e", size = 419046, upload-time = "2024-11-11T05:25:20.679Z" }, + { url = "https://files.pythonhosted.org/packages/0f/21/13df29f8fcb9ba9f209b7b413a4764b673dfd58989a0dd67e9c7e19e9c2e/tree_sitter_c_sharp-0.23.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:986e93d845a438ec3c4416401aa98e6a6f6631d644bbbc2e43fcb915c51d255d", size = 415999, upload-time = "2024-11-11T05:25:22.359Z" }, + { url = "https://files.pythonhosted.org/packages/ca/72/fc6846795bcdae2f8aa94cc8b1d1af33d634e08be63e294ff0d6794b1efc/tree_sitter_c_sharp-0.23.1-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8024e466b2f5611c6dc90321f232d8584893c7fb88b75e4a831992f877616d2", size = 402830, upload-time = "2024-11-11T05:25:24.198Z" }, + { url = "https://files.pythonhosted.org/packages/fe/3a/b6028c5890ce6653807d5fa88c72232c027c6ceb480dbeb3b186d60e5971/tree_sitter_c_sharp-0.23.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7f9bf876866835492281d336b9e1f9626ab668737f74e914c31d285261507da7", size = 397880, upload-time = "2024-11-11T05:25:25.937Z" }, + { url = "https://files.pythonhosted.org/packages/47/d2/4facaa34b40f8104d8751746d0e1cd2ddf0beb9f1404b736b97f372bd1f3/tree_sitter_c_sharp-0.23.1-cp39-abi3-win_amd64.whl", hash = "sha256:ae9a9e859e8f44e2b07578d44f9a220d3fa25b688966708af6aa55d42abeebb3", size = 377562, upload-time = "2024-11-11T05:25:27.539Z" }, + { url = "https://files.pythonhosted.org/packages/d8/88/3cf6bd9959d94d1fec1e6a9c530c5f08ff4115a474f62aedb5fedb0f7241/tree_sitter_c_sharp-0.23.1-cp39-abi3-win_arm64.whl", hash = "sha256:c81548347a93347be4f48cb63ec7d60ef4b0efa91313330e69641e49aa5a08c5", size = 375157, upload-time = "2024-11-11T05:25:30.839Z" }, +] + +[[package]] +name = "tree-sitter-java" +version = "0.23.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/dc/eb9c8f96304e5d8ae1663126d89967a622a80937ad2909903569ccb7ec8f/tree_sitter_java-0.23.5.tar.gz", hash = "sha256:f5cd57b8f1270a7f0438878750d02ccc79421d45cca65ff284f1527e9ef02e38", size = 138121, upload-time = "2024-12-21T18:24:26.936Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/21/b3399780b440e1567a11d384d0ebb1aea9b642d0d98becf30fa55c0e3a3b/tree_sitter_java-0.23.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:355ce0308672d6f7013ec913dee4a0613666f4cda9044a7824240d17f38209df", size = 58926, upload-time = "2024-12-21T18:24:12.53Z" }, + { url = "https://files.pythonhosted.org/packages/57/ef/6406b444e2a93bc72a04e802f4107e9ecf04b8de4a5528830726d210599c/tree_sitter_java-0.23.5-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:24acd59c4720dedad80d548fe4237e43ef2b7a4e94c8549b0ca6e4c4d7bf6e69", size = 62288, upload-time = "2024-12-21T18:24:14.634Z" }, + { url = "https://files.pythonhosted.org/packages/4e/6c/74b1c150d4f69c291ab0b78d5dd1b59712559bbe7e7daf6d8466d483463f/tree_sitter_java-0.23.5-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9401e7271f0b333df39fc8a8336a0caf1b891d9a2b89ddee99fae66b794fc5b7", size = 85533, upload-time = "2024-12-21T18:24:16.695Z" }, + { url = "https://files.pythonhosted.org/packages/29/09/e0d08f5c212062fd046db35c1015a2621c2631bc8b4aae5740d7adb276ad/tree_sitter_java-0.23.5-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:370b204b9500b847f6d0c5ad584045831cee69e9a3e4d878535d39e4a7e4c4f1", size = 84033, upload-time = "2024-12-21T18:24:18.758Z" }, + { url = "https://files.pythonhosted.org/packages/43/56/7d06b23ddd09bde816a131aa504ee11a1bbe87c6b62ab9b2ed23849a3382/tree_sitter_java-0.23.5-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:aae84449e330363b55b14a2af0585e4e0dae75eb64ea509b7e5b0e1de536846a", size = 82564, upload-time = "2024-12-21T18:24:20.493Z" }, + { url = "https://files.pythonhosted.org/packages/da/d6/0528c7e1e88a18221dbd8ccee3825bf274b1fa300f745fd74eb343878043/tree_sitter_java-0.23.5-cp39-abi3-win_amd64.whl", hash = "sha256:1ee45e790f8d31d416bc84a09dac2e2c6bc343e89b8a2e1d550513498eedfde7", size = 60650, upload-time = "2024-12-21T18:24:22.902Z" }, + { url = "https://files.pythonhosted.org/packages/72/57/5bab54d23179350356515526fff3cc0f3ac23bfbc1a1d518a15978d4880e/tree_sitter_java-0.23.5-cp39-abi3-win_arm64.whl", hash = "sha256:402efe136104c5603b429dc26c7e75ae14faaca54cfd319ecc41c8f2534750f4", size = 59059, upload-time = "2024-12-21T18:24:24.934Z" }, +] + +[[package]] +name = "tree-sitter-python" +version = "0.23.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/30/6766433b31be476fda6569a3a374c2220e45ffee0bff75460038a57bf23b/tree_sitter_python-0.23.6.tar.gz", hash = "sha256:354bfa0a2f9217431764a631516f85173e9711af2c13dbd796a8815acfe505d9", size = 155868, upload-time = "2024-12-22T23:09:55.918Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/67/577a02acae5f776007c924ca86ef14c19c12e71de0aa9d2a036f3c248e7b/tree_sitter_python-0.23.6-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:28fbec8f74eeb2b30292d97715e60fac9ccf8a8091ce19b9d93e9b580ed280fb", size = 74361, upload-time = "2024-12-22T23:09:42.37Z" }, + { url = "https://files.pythonhosted.org/packages/d2/a6/194b3625a7245c532ad418130d63077ce6cd241152524152f533e4d6edb0/tree_sitter_python-0.23.6-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:680b710051b144fedf61c95197db0094f2245e82551bf7f0c501356333571f7a", size = 76436, upload-time = "2024-12-22T23:09:43.566Z" }, + { url = "https://files.pythonhosted.org/packages/d0/62/1da112689d6d282920e62c40e67ab39ea56463b0e7167bfc5e81818a770e/tree_sitter_python-0.23.6-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8a9dcef55507b6567207e8ee0a6b053d0688019b47ff7f26edc1764b7f4dc0a4", size = 112060, upload-time = "2024-12-22T23:09:44.721Z" }, + { url = "https://files.pythonhosted.org/packages/5d/62/c9358584c96e38318d69b6704653684fd8467601f7b74e88aa44f4e6903f/tree_sitter_python-0.23.6-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:29dacdc0cd2f64e55e61d96c6906533ebb2791972bec988450c46cce60092f5d", size = 112338, upload-time = "2024-12-22T23:09:48.323Z" }, + { url = "https://files.pythonhosted.org/packages/1a/58/c5e61add45e34fb8ecbf057c500bae9d96ed7c9ca36edb7985da8ae45526/tree_sitter_python-0.23.6-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7e048733c36f564b379831689006801feb267d8194f9e793fbb395ef1723335d", size = 109382, upload-time = "2024-12-22T23:09:49.49Z" }, + { url = "https://files.pythonhosted.org/packages/e9/f3/9b30893cae9b3811fe652dc6f90aaadfda12ae0b2757f5722fc7266f423c/tree_sitter_python-0.23.6-cp39-abi3-win_amd64.whl", hash = "sha256:a24027248399fb41594b696f929f9956828ae7cc85596d9f775e6c239cd0c2be", size = 75904, upload-time = "2024-12-22T23:09:51.597Z" }, + { url = "https://files.pythonhosted.org/packages/87/cb/ce35a65f83a47b510d8a2f1eddf3bdbb0d57aabc87351c8788caf3309f76/tree_sitter_python-0.23.6-cp39-abi3-win_arm64.whl", hash = "sha256:71334371bd73d5fe080aed39fbff49ed8efb9506edebe16795b0c7567ed6a272", size = 73649, upload-time = "2024-12-22T23:09:53.71Z" }, +] + +[[package]] +name = "tree-sitter-typescript" +version = "0.23.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1e/fc/bb52958f7e399250aee093751e9373a6311cadbe76b6e0d109b853757f35/tree_sitter_typescript-0.23.2.tar.gz", hash = "sha256:7b167b5827c882261cb7a50dfa0fb567975f9b315e87ed87ad0a0a3aedb3834d", size = 773053, upload-time = "2024-11-11T02:36:11.396Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/28/95/4c00680866280e008e81dd621fd4d3f54aa3dad1b76b857a19da1b2cc426/tree_sitter_typescript-0.23.2-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:3cd752d70d8e5371fdac6a9a4df9d8924b63b6998d268586f7d374c9fba2a478", size = 286677, upload-time = "2024-11-11T02:35:58.839Z" }, + { url = "https://files.pythonhosted.org/packages/8f/2f/1f36fda564518d84593f2740d5905ac127d590baf5c5753cef2a88a89c15/tree_sitter_typescript-0.23.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:c7cc1b0ff5d91bac863b0e38b1578d5505e718156c9db577c8baea2557f66de8", size = 302008, upload-time = "2024-11-11T02:36:00.733Z" }, + { url = "https://files.pythonhosted.org/packages/96/2d/975c2dad292aa9994f982eb0b69cc6fda0223e4b6c4ea714550477d8ec3a/tree_sitter_typescript-0.23.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b1eed5b0b3a8134e86126b00b743d667ec27c63fc9de1b7bb23168803879e31", size = 351987, upload-time = "2024-11-11T02:36:02.669Z" }, + { url = "https://files.pythonhosted.org/packages/49/d1/a71c36da6e2b8a4ed5e2970819b86ef13ba77ac40d9e333cb17df6a2c5db/tree_sitter_typescript-0.23.2-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e96d36b85bcacdeb8ff5c2618d75593ef12ebaf1b4eace3477e2bdb2abb1752c", size = 344960, upload-time = "2024-11-11T02:36:04.443Z" }, + { url = "https://files.pythonhosted.org/packages/7f/cb/f57b149d7beed1a85b8266d0c60ebe4c46e79c9ba56bc17b898e17daf88e/tree_sitter_typescript-0.23.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8d4f0f9bcb61ad7b7509d49a1565ff2cc363863644a234e1e0fe10960e55aea0", size = 340245, upload-time = "2024-11-11T02:36:06.473Z" }, + { url = "https://files.pythonhosted.org/packages/8b/ab/dd84f0e2337296a5f09749f7b5483215d75c8fa9e33738522e5ed81f7254/tree_sitter_typescript-0.23.2-cp39-abi3-win_amd64.whl", hash = "sha256:3f730b66396bc3e11811e4465c41ee45d9e9edd6de355a58bbbc49fa770da8f9", size = 278015, upload-time = "2024-11-11T02:36:07.631Z" }, + { url = "https://files.pythonhosted.org/packages/9f/e4/81f9a935789233cf412a0ed5fe04c883841d2c8fb0b7e075958a35c65032/tree_sitter_typescript-0.23.2-cp39-abi3-win_arm64.whl", hash = "sha256:05db58f70b95ef0ea126db5560f3775692f609589ed6f8dd0af84b7f19f1cbb7", size = 274052, upload-time = "2024-11-11T02:36:09.514Z" }, +] + [[package]] name = "triton" version = "3.3.1"