feat: Enhance CLI with improved list and smart remove commands

- **Better UX**: Current project shown first with clear separation - **Visual improvements**: Icons (🏠/📂), better formatting, size info - **Smart guidance**: Context-aware usage examples and getting started tips - **Safety first**: Always shows ALL matching indexes across projects - **Intelligent handling**: - Single match: Clear location display with cross-project warnings - Multiple matches: Interactive selection with final confirmation - **Prevents accidents**: No more deleting wrong indexes due to name conflicts - **User-friendly**: 'c' to cancel, clear visual hierarchy, detailed info - **Clean logging**: Hide debug messages for better CLI experience - **Comprehensive search**: Always scan all projects for transparency - **Error handling**: Graceful handling of edge cases and user input - **Safer**: Eliminates risk of accidental index deletion - **Clearer**: Users always know what they're operating on - **Smarter**: Automatic detection and handling of common scenarios
2025-08-15 23:12:21 -07:00
38 changed files with 4077 additions and 5699 deletions
--- a/.gitattributes
+++ b/.gitattributes
@@ -0,0 +1 @@
+paper_plot/data/big_graph_degree_data.npz filter=lfs diff=lfs merge=lfs -text
--- a/.github/workflows/build-reusable.yml
+++ b/.github/workflows/build-reusable.yml
@@ -87,7 +87,7 @@ jobs:
    runs-on: ${{ matrix.os }}

    steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref }}
          submodules: recursive
@@ -98,23 +98,21 @@ jobs:
          python-version: ${{ matrix.python }}

      - name: Install uv
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v4

      - name: Install system dependencies (Ubuntu)
        if: runner.os == 'Linux'
        run: |
          sudo apt-get update
          sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \
-            pkg-config libabsl-dev libaio-dev libprotobuf-dev \
-            patchelf
+            pkg-config libopenblas-dev patchelf libabsl-dev libaio-dev libprotobuf-dev

          # Install Intel MKL for DiskANN
          wget -q https://registrationcenter-download.intel.com/akdlm/IRC_NAS/79153e0f-74d7-45af-b8c2-258941adf58a/intel-onemkl-2025.0.0.940.sh
          sudo sh intel-onemkl-2025.0.0.940.sh -a --components intel.oneapi.lin.mkl.devel --action install --eula accept -s
          source /opt/intel/oneapi/setvars.sh
          echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV
-          echo "LD_LIBRARY_PATH=/opt/intel/oneapi/compiler/latest/linux/compiler/lib/intel64_lin" >> $GITHUB_ENV
-          echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/intel/oneapi/mkl/latest/lib/intel64" >> $GITHUB_ENV
+          echo "LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH" >> $GITHUB_ENV

      - name: Install system dependencies (macOS)
        if: runner.os == 'macOS'
@@ -306,53 +304,3 @@ jobs:
        with:
          name: packages-${{ matrix.os }}-py${{ matrix.python }}
          path: packages/*/dist/
-
-
-  arch-smoke:
-    name: Arch Linux smoke test (install & import)
-    needs: build
-    runs-on: ubuntu-latest
-    container:
-      image: archlinux:latest
-
-    steps:
-      - name: Prepare system
-        run: |
-          pacman -Syu --noconfirm
-          pacman -S --noconfirm python python-pip gcc git zlib openssl
-
-      - name: Download ALL wheel artifacts from this run
-        uses: actions/download-artifact@v5
-        with:
-          # Don't specify name, download all artifacts
-          path: ./wheels
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v6
-
-      - name: Create virtual environment and install wheels
-        run: |
-          uv venv
-          source .venv/bin/activate || source .venv/Scripts/activate
-          uv pip install --find-links wheels leann-core
-          uv pip install --find-links wheels leann-backend-hnsw
-          uv pip install --find-links wheels leann-backend-diskann
-          uv pip install --find-links wheels leann
-
-      - name: Import & tiny runtime check
-        env:
-          OMP_NUM_THREADS: 1
-          MKL_NUM_THREADS: 1
-        run: |
-          source .venv/bin/activate || source .venv/Scripts/activate
-          python - <<'PY'
-          import leann
-          import leann_backend_hnsw as h
-          import leann_backend_diskann as d
-          from leann import LeannBuilder, LeannSearcher
-          b = LeannBuilder(backend_name="hnsw")
-          b.add_text("hello arch")
-          b.build_index("arch_demo.leann")
-          s = LeannSearcher("arch_demo.leann")
-          print("search:", s.search("hello", top_k=1))
-          PY
--- a/.github/workflows/link-check.yml
+++ b/.github/workflows/link-check.yml
@@ -14,6 +14,6 @@ jobs:
      - uses: actions/checkout@v4
      - uses: lycheeverse/lychee-action@v2
        with:
-          args: --no-progress --insecure --user-agent 'curl/7.68.0' README.md docs/ apps/ examples/ benchmarks/
+          args: --no-progress --insecure README.md docs/ apps/ examples/ benchmarks/
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
--- a/.gitignore
+++ b/.gitignore
@@ -18,7 +18,6 @@ demo/experiment_results/**/*.json
 *.eml
 *.emlx
 *.json
-!.vscode/*.json
 *.sh
 *.txt
 !CMakeLists.txt
@@ -93,6 +92,3 @@ packages/leann-backend-diskann/third_party/DiskANN/_deps/
 batchtest.py
 tests/__pytest_cache__/
 tests/__pycache__/
-paru-bin/
-
-benchmarks/data/
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -13,5 +13,4 @@ repos:
    rev: v0.12.7  # Fixed version to match pyproject.toml
    hooks:
      - id: ruff
-        args: [--fix, --exit-non-zero-on-fix]
      - id: ruff-format
--- a/.vscode/extensions.json
+++ b/.vscode/extensions.json
@@ -1,5 +0,0 @@
-{
-    "recommendations": [
-        "charliermarsh.ruff",
-    ]
-}
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -1,22 +0,0 @@
-{
-  "python.defaultInterpreterPath": ".venv/bin/python",
-  "python.terminal.activateEnvironment": true,
-  "[python]": {
-    "editor.defaultFormatter": "charliermarsh.ruff",
-    "editor.formatOnSave": true,
-    "editor.codeActionsOnSave": {
-      "source.organizeImports": "explicit",
-      "source.fixAll": "explicit"
-    },
-    "editor.insertSpaces": true,
-    "editor.tabSize": 4
-  },
-  "ruff.enable": true,
-  "files.watcherExclude": {
-    "**/.venv/**": true,
-    "**/__pycache__/**": true,
-    "**/*.egg-info/**": true,
-    "**/build/**": true,
-    "**/dist/**": true
-  }
-}
--- a/README.md
+++ b/README.md
@@ -87,60 +87,17 @@ git submodule update --init --recursive
 ```

 **macOS:**
-
-Note: DiskANN requires MacOS 13.3 or later.
-
 ```bash
-brew install libomp boost protobuf zeromq pkgconf
-uv sync --extra diskann
+brew install llvm libomp boost protobuf zeromq pkgconf
+CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv sync
 ```

-**Linux (Ubuntu/Debian):**
-
-Note: On Ubuntu 20.04, you may need to build a newer Abseil and pin Protobuf (e.g., v3.20.x) for building DiskANN. See [Issue #30](https://github.com/yichuan-w/LEANN/issues/30) for a step-by-step note.
-
-You can manually install [Intel oneAPI MKL](https://www.intel.com/content/www/us/en/developer/tools/oneapi/onemkl.html) instead of `libmkl-full-dev` for DiskANN. You can also use `libopenblas-dev` for building HNSW only, by removing `--extra diskann` in the command below.
-
+**Linux:**
 ```bash
-sudo apt-get update && sudo apt-get install -y \
-  libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \
-  pkg-config libabsl-dev libaio-dev libprotobuf-dev \
-  libmkl-full-dev
+# Ubuntu/Debian (For Arch Linux: sudo pacman -S blas lapack openblas libaio boost protobuf abseil-cpp zeromq)
+sudo apt-get update && sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libabsl-dev libmkl-full-dev libaio-dev libzmq3-dev

-uv sync --extra diskann
-```
-
-**Linux (Arch Linux):**
-
-```bash
-sudo pacman -Syu && sudo pacman -S --needed base-devel cmake pkgconf git gcc \
-  boost boost-libs protobuf abseil-cpp libaio zeromq
-
-# For MKL in DiskANN
-sudo pacman -S --needed base-devel git
-git clone https://aur.archlinux.org/paru-bin.git
-cd paru-bin && makepkg -si
-paru -S intel-oneapi-mkl intel-oneapi-compiler
-source /opt/intel/oneapi/setvars.sh
-
-uv sync --extra diskann
-```
-
-**Linux (RHEL / CentOS Stream / Oracle / Rocky / AlmaLinux):**
-
-See [Issue #50](https://github.com/yichuan-w/LEANN/issues/50) for more details.
-
-```bash
-sudo dnf groupinstall -y "Development Tools"
-sudo dnf install -y libomp-devel boost-devel protobuf-compiler protobuf-devel \
-  abseil-cpp-devel libaio-devel zeromq-devel pkgconf-pkg-config
-
-# For MKL in DiskANN
-sudo dnf install -y intel-oneapi-mkl intel-oneapi-mkl-devel \
-  intel-oneapi-openmp || sudo dnf install -y intel-oneapi-compiler
-source /opt/intel/oneapi/setvars.sh
-
-uv sync --extra diskann
+uv sync
 ```

 </details>
@@ -176,8 +133,6 @@ response = chat.ask("How much storage does LEANN save?", top_k=1)

 LEANN supports RAG on various data sources including documents (`.pdf`, `.txt`, `.md`), Apple Mail, Google Search History, WeChat, and more.

-
-
 ### Generation Model Setup

 LEANN supports multiple LLM providers for text generation (OpenAI API, HuggingFace, Ollama).
@@ -296,12 +251,6 @@ python -m apps.document_rag --data-dir "~/Documents/Papers" --chunk-size 1024

 # Filter only markdown and Python files with smaller chunks
 python -m apps.document_rag --data-dir "./docs" --chunk-size 256 --file-types .md .py
-
-# Enable AST-aware chunking for code files
-python -m apps.document_rag --enable-code-chunking --data-dir "./my_project"
-
-# Or use the specialized code RAG for better code understanding
-python -m apps.code_rag --repo-dir "./my_codebase" --query "How does authentication work?"
 ```

 </details>
@@ -476,20 +425,10 @@ Once the index is built, you can ask questions like:

 ### 🚀 Claude Code Integration: Transform Your Development Workflow!

-<details>
-<summary><strong>NEW!! AST‑Aware Code Chunking</strong></summary>
-
-LEANN features intelligent code chunking that preserves semantic boundaries (functions, classes, methods) for Python, Java, C#, and TypeScript, improving code understanding compared to text-based chunking.
-
-📖 Read the [AST Chunking Guide →](docs/ast_chunking_guide.md)
-
-</details>
-
 **The future of code assistance is here.** Transform your development workflow with LEANN's native MCP integration for Claude Code. Index your entire codebase and get intelligent code assistance directly in your IDE.

 **Key features:**
 - 🔍 **Semantic code search** across your entire project, fully local index and lightweight
- 🧠 **AST-aware chunking** preserves code structure (functions, classes)
 - 📚 **Context-aware assistance** for debugging and development
 - 🚀 **Zero-config setup** with automatic language detection

@@ -552,8 +491,7 @@ leann remove my-docs

 **Key CLI features:**
 - Auto-detects document formats (PDF, TXT, MD, DOCX, PPTX + code files)
- **🧠 AST-aware chunking** for Python, Java, C#, TypeScript files
- Smart text chunking with overlap for all other content
+- Smart text chunking with overlap
 - Multiple LLM providers (Ollama, OpenAI, HuggingFace)
 - Organized index storage in `.leann/indexes/` (project-local)
 - Support for advanced search parameters
@@ -604,10 +542,8 @@ Options:
 leann list

 # Lists all indexes across all projects with status indicators:
-# ✅ - Index is complete and ready to use
-# ❌ - Index is incomplete or corrupted
-# 📁 - CLI-created index (in .leann/indexes/)
-# 📄 - App-created index (*.leann.meta.json files)
+# ✓ - Index is complete and ready to use
+# ✗ - Index is incomplete or corrupted
 ```

 **Remove Command:**
@@ -621,7 +557,6 @@ Options:
 # - Shows all matching indexes across projects
 # - Requires confirmation for cross-project removal
 # - Interactive selection when multiple matches found
-# - Supports both CLI and app-created indexes
 ```

 </details>
@@ -665,7 +600,6 @@ Options:
 ```bash
 uv pip install -e ".[dev]"  # Install dev dependencies
 python benchmarks/run_evaluation.py    # Will auto-download evaluation data and run benchmarks
-python benchmarks/run_evaluation.py benchmarks/data/indices/rpj_wiki/rpj_wiki --num-queries 2000    # After downloading data, you can run the benchmark with our biggest index
 ```

 The evaluation script downloads data automatically on first run. The last three results were tested with partial personal data, and you can reproduce them with your own data!
@@ -705,9 +639,6 @@ MIT License - see [LICENSE](LICENSE) for details.

 Core Contributors: [Yichuan Wang](https://yichuan-w.github.io/) & [Zhifei Li](https://github.com/andylizf).

-Active Contributors: [Gabriel Dehan](https://github.com/gabriel-dehan)
-
-
 We welcome more contributors! Feel free to open issues or submit PRs.

 This work is done at [**Berkeley Sky Computing Lab**](https://sky.cs.berkeley.edu/).
--- a/apps/base_rag_example.py
+++ b/apps/base_rag_example.py
@@ -10,7 +10,7 @@ from typing import Any

 import dotenv
 from leann.api import LeannBuilder, LeannChat
-from leann.registry import register_project_directory
+from llama_index.core.node_parser import SentenceSplitter

 dotenv.load_dotenv()

@@ -108,38 +108,6 @@ class BaseRAGExample(ABC):
            help="Thinking budget for reasoning models (low/medium/high). Supported by GPT-Oss:20b and other reasoning models.",
        )

-        # AST Chunking parameters
-        ast_group = parser.add_argument_group("AST Chunking Parameters")
-        ast_group.add_argument(
-            "--use-ast-chunking",
-            action="store_true",
-            help="Enable AST-aware chunking for code files (requires astchunk)",
-        )
-        ast_group.add_argument(
-            "--ast-chunk-size",
-            type=int,
-            default=512,
-            help="Maximum characters per AST chunk (default: 512)",
-        )
-        ast_group.add_argument(
-            "--ast-chunk-overlap",
-            type=int,
-            default=64,
-            help="Overlap between AST chunks (default: 64)",
-        )
-        ast_group.add_argument(
-            "--code-file-extensions",
-            nargs="+",
-            default=None,
-            help="Additional code file extensions to process with AST chunking (e.g., .py .java .cs .ts)",
-        )
-        ast_group.add_argument(
-            "--ast-fallback-traditional",
-            action="store_true",
-            default=True,
-            help="Fall back to traditional chunking if AST chunking fails (default: True)",
-        )
-
        # Search parameters
        search_group = parser.add_argument_group("Search Parameters")
        search_group.add_argument(
@@ -246,11 +214,6 @@ class BaseRAGExample(ABC):
        builder.build_index(index_path)
        print(f"Index saved to: {index_path}")

-        # Register project directory so leann list can discover this index
-        # The index is saved as args.index_dir/index_name.leann
-        # We want to register the current working directory where the app is run
-        register_project_directory(Path.cwd())
-
        return index_path

    async def run_interactive_chat(self, args, index_path: str):
@@ -299,6 +262,7 @@ class BaseRAGExample(ABC):
        chat = LeannChat(
            index_path,
            llm_config=self.get_llm_config(args),
+            system_prompt=f"You are a helpful assistant that answers questions about {self.name} data.",
            complexity=args.search_complexity,
        )

@@ -340,3 +304,21 @@ class BaseRAGExample(ABC):
            await self.run_single_query(args, index_path, args.query)
        else:
            await self.run_interactive_chat(args, index_path)
+
+
+def create_text_chunks(documents, chunk_size=256, chunk_overlap=25) -> list[str]:
+    """Helper function to create text chunks from documents."""
+    node_parser = SentenceSplitter(
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+        separator=" ",
+        paragraph_separator="\n\n",
+    )
+
+    all_texts = []
+    for doc in documents:
+        nodes = node_parser.get_nodes_from_documents([doc])
+        if nodes:
+            all_texts.extend(node.get_content() for node in nodes)
+
+    return all_texts
--- a/apps/chunking/init.py
+++ b/apps/chunking/init.py
@@ -1,22 +0,0 @@
-"""
-Chunking utilities for LEANN RAG applications.
-Provides AST-aware and traditional text chunking functionality.
-"""
-
-from .utils import (
-    CODE_EXTENSIONS,
-    create_ast_chunks,
-    create_text_chunks,
-    create_traditional_chunks,
-    detect_code_files,
-    get_language_from_extension,
-)
-
-__all__ = [
-    "CODE_EXTENSIONS",
-    "create_ast_chunks",
-    "create_text_chunks",
-    "create_traditional_chunks",
-    "detect_code_files",
-    "get_language_from_extension",
-]
--- a/apps/chunking/utils.py
+++ b/apps/chunking/utils.py
@@ -1,320 +0,0 @@
-"""
-Enhanced chunking utilities with AST-aware code chunking support.
-Provides unified interface for both traditional and AST-based text chunking.
-"""
-
-import logging
-from pathlib import Path
-from typing import Optional
-
-from llama_index.core.node_parser import SentenceSplitter
-
-logger = logging.getLogger(__name__)
-
-# Code file extensions supported by astchunk
-CODE_EXTENSIONS = {
-    ".py": "python",
-    ".java": "java",
-    ".cs": "csharp",
-    ".ts": "typescript",
-    ".tsx": "typescript",
-    ".js": "typescript",
-    ".jsx": "typescript",
-}
-
-# Default chunk parameters for different content types
-DEFAULT_CHUNK_PARAMS = {
-    "code": {
-        "max_chunk_size": 512,
-        "chunk_overlap": 64,
-    },
-    "text": {
-        "chunk_size": 256,
-        "chunk_overlap": 128,
-    },
-}
-
-
-def detect_code_files(documents, code_extensions=None) -> tuple[list, list]:
-    """
-    Separate documents into code files and regular text files.
-
-    Args:
-        documents: List of LlamaIndex Document objects
-        code_extensions: Dict mapping file extensions to languages (defaults to CODE_EXTENSIONS)
-
-    Returns:
-        Tuple of (code_documents, text_documents)
-    """
-    if code_extensions is None:
-        code_extensions = CODE_EXTENSIONS
-
-    code_docs = []
-    text_docs = []
-
-    for doc in documents:
-        # Get file path from metadata
-        file_path = doc.metadata.get("file_path", "")
-        if not file_path:
-            # Fallback to file_name
-            file_path = doc.metadata.get("file_name", "")
-
-        if file_path:
-            file_ext = Path(file_path).suffix.lower()
-            if file_ext in code_extensions:
-                # Add language info to metadata
-                doc.metadata["language"] = code_extensions[file_ext]
-                doc.metadata["is_code"] = True
-                code_docs.append(doc)
-            else:
-                doc.metadata["is_code"] = False
-                text_docs.append(doc)
-        else:
-            # If no file path, treat as text
-            doc.metadata["is_code"] = False
-            text_docs.append(doc)
-
-    logger.info(f"Detected {len(code_docs)} code files and {len(text_docs)} text files")
-    return code_docs, text_docs
-
-
-def get_language_from_extension(file_path: str) -> Optional[str]:
-    """Get the programming language from file extension."""
-    ext = Path(file_path).suffix.lower()
-    return CODE_EXTENSIONS.get(ext)
-
-
-def create_ast_chunks(
-    documents,
-    max_chunk_size: int = 512,
-    chunk_overlap: int = 64,
-    metadata_template: str = "default",
-) -> list[str]:
-    """
-    Create AST-aware chunks from code documents using astchunk.
-
-    Args:
-        documents: List of code documents
-        max_chunk_size: Maximum characters per chunk
-        chunk_overlap: Number of AST nodes to overlap between chunks
-        metadata_template: Template for chunk metadata
-
-    Returns:
-        List of text chunks with preserved code structure
-    """
-    try:
-        from astchunk import ASTChunkBuilder
-    except ImportError as e:
-        logger.error(f"astchunk not available: {e}")
-        logger.info("Falling back to traditional chunking for code files")
-        return create_traditional_chunks(documents, max_chunk_size, chunk_overlap)
-
-    all_chunks = []
-
-    for doc in documents:
-        # Get language from metadata (set by detect_code_files)
-        language = doc.metadata.get("language")
-        if not language:
-            logger.warning(
-                "No language detected for document, falling back to traditional chunking"
-            )
-            traditional_chunks = create_traditional_chunks([doc], max_chunk_size, chunk_overlap)
-            all_chunks.extend(traditional_chunks)
-            continue
-
-        try:
-            # Configure astchunk
-            configs = {
-                "max_chunk_size": max_chunk_size,
-                "language": language,
-                "metadata_template": metadata_template,
-                "chunk_overlap": chunk_overlap if chunk_overlap > 0 else 0,
-            }
-
-            # Add repository-level metadata if available
-            repo_metadata = {
-                "file_path": doc.metadata.get("file_path", ""),
-                "file_name": doc.metadata.get("file_name", ""),
-                "creation_date": doc.metadata.get("creation_date", ""),
-                "last_modified_date": doc.metadata.get("last_modified_date", ""),
-            }
-            configs["repo_level_metadata"] = repo_metadata
-
-            # Create chunk builder and process
-            chunk_builder = ASTChunkBuilder(**configs)
-            code_content = doc.get_content()
-
-            if not code_content or not code_content.strip():
-                logger.warning("Empty code content, skipping")
-                continue
-
-            chunks = chunk_builder.chunkify(code_content)
-
-            # Extract text content from chunks
-            for chunk in chunks:
-                if hasattr(chunk, "text"):
-                    chunk_text = chunk.text
-                elif isinstance(chunk, dict) and "text" in chunk:
-                    chunk_text = chunk["text"]
-                elif isinstance(chunk, str):
-                    chunk_text = chunk
-                else:
-                    # Try to convert to string
-                    chunk_text = str(chunk)
-
-                if chunk_text and chunk_text.strip():
-                    all_chunks.append(chunk_text.strip())
-
-            logger.info(
-                f"Created {len(chunks)} AST chunks from {language} file: {doc.metadata.get('file_name', 'unknown')}"
-            )
-
-        except Exception as e:
-            logger.warning(f"AST chunking failed for {language} file: {e}")
-            logger.info("Falling back to traditional chunking")
-            traditional_chunks = create_traditional_chunks([doc], max_chunk_size, chunk_overlap)
-            all_chunks.extend(traditional_chunks)
-
-    return all_chunks
-
-
-def create_traditional_chunks(
-    documents, chunk_size: int = 256, chunk_overlap: int = 128
-) -> list[str]:
-    """
-    Create traditional text chunks using LlamaIndex SentenceSplitter.
-
-    Args:
-        documents: List of documents to chunk
-        chunk_size: Size of each chunk in characters
-        chunk_overlap: Overlap between chunks
-
-    Returns:
-        List of text chunks
-    """
-    # Handle invalid chunk_size values
-    if chunk_size <= 0:
-        logger.warning(f"Invalid chunk_size={chunk_size}, using default value of 256")
-        chunk_size = 256
-
-    # Ensure chunk_overlap is not negative and not larger than chunk_size
-    if chunk_overlap < 0:
-        chunk_overlap = 0
-    if chunk_overlap >= chunk_size:
-        chunk_overlap = chunk_size // 2
-
-    node_parser = SentenceSplitter(
-        chunk_size=chunk_size,
-        chunk_overlap=chunk_overlap,
-        separator=" ",
-        paragraph_separator="\n\n",
-    )
-
-    all_texts = []
-    for doc in documents:
-        try:
-            nodes = node_parser.get_nodes_from_documents([doc])
-            if nodes:
-                chunk_texts = [node.get_content() for node in nodes]
-                all_texts.extend(chunk_texts)
-                logger.debug(f"Created {len(chunk_texts)} traditional chunks from document")
-        except Exception as e:
-            logger.error(f"Traditional chunking failed for document: {e}")
-            # As last resort, add the raw content
-            content = doc.get_content()
-            if content and content.strip():
-                all_texts.append(content.strip())
-
-    return all_texts
-
-
-def create_text_chunks(
-    documents,
-    chunk_size: int = 256,
-    chunk_overlap: int = 128,
-    use_ast_chunking: bool = False,
-    ast_chunk_size: int = 512,
-    ast_chunk_overlap: int = 64,
-    code_file_extensions: Optional[list[str]] = None,
-    ast_fallback_traditional: bool = True,
-) -> list[str]:
-    """
-    Create text chunks from documents with optional AST support for code files.
-
-    Args:
-        documents: List of LlamaIndex Document objects
-        chunk_size: Size for traditional text chunks
-        chunk_overlap: Overlap for traditional text chunks
-        use_ast_chunking: Whether to use AST chunking for code files
-        ast_chunk_size: Size for AST chunks
-        ast_chunk_overlap: Overlap for AST chunks
-        code_file_extensions: Custom list of code file extensions
-        ast_fallback_traditional: Fall back to traditional chunking on AST errors
-
-    Returns:
-        List of text chunks
-    """
-    if not documents:
-        logger.warning("No documents provided for chunking")
-        return []
-
-    # Create a local copy of supported extensions for this function call
-    local_code_extensions = CODE_EXTENSIONS.copy()
-
-    # Update supported extensions if provided
-    if code_file_extensions:
-        # Map extensions to languages (simplified mapping)
-        ext_mapping = {
-            ".py": "python",
-            ".java": "java",
-            ".cs": "c_sharp",
-            ".ts": "typescript",
-            ".tsx": "typescript",
-        }
-        for ext in code_file_extensions:
-            if ext.lower() not in local_code_extensions:
-                # Try to guess language from extension
-                if ext.lower() in ext_mapping:
-                    local_code_extensions[ext.lower()] = ext_mapping[ext.lower()]
-                else:
-                    logger.warning(f"Unsupported extension {ext}, will use traditional chunking")
-
-    all_chunks = []
-
-    if use_ast_chunking:
-        # Separate code and text documents using local extensions
-        code_docs, text_docs = detect_code_files(documents, local_code_extensions)
-
-        # Process code files with AST chunking
-        if code_docs:
-            logger.info(f"Processing {len(code_docs)} code files with AST chunking")
-            try:
-                ast_chunks = create_ast_chunks(
-                    code_docs, max_chunk_size=ast_chunk_size, chunk_overlap=ast_chunk_overlap
-                )
-                all_chunks.extend(ast_chunks)
-                logger.info(f"Created {len(ast_chunks)} AST chunks from code files")
-            except Exception as e:
-                logger.error(f"AST chunking failed: {e}")
-                if ast_fallback_traditional:
-                    logger.info("Falling back to traditional chunking for code files")
-                    traditional_code_chunks = create_traditional_chunks(
-                        code_docs, chunk_size, chunk_overlap
-                    )
-                    all_chunks.extend(traditional_code_chunks)
-                else:
-                    raise
-
-        # Process text files with traditional chunking
-        if text_docs:
-            logger.info(f"Processing {len(text_docs)} text files with traditional chunking")
-            text_chunks = create_traditional_chunks(text_docs, chunk_size, chunk_overlap)
-            all_chunks.extend(text_chunks)
-            logger.info(f"Created {len(text_chunks)} traditional chunks from text files")
-    else:
-        # Use traditional chunking for all files
-        logger.info(f"Processing {len(documents)} documents with traditional chunking")
-        all_chunks = create_traditional_chunks(documents, chunk_size, chunk_overlap)
-
-    logger.info(f"Total chunks created: {len(all_chunks)}")
-    return all_chunks
--- a/apps/code_rag.py
+++ b/apps/code_rag.py
@@ -1,211 +0,0 @@
-"""
-Code RAG example using AST-aware chunking for optimal code understanding.
-Specialized for code repositories with automatic language detection and
-optimized chunking parameters.
-"""
-
-import sys
-from pathlib import Path
-
-# Add parent directory to path for imports
-sys.path.insert(0, str(Path(__file__).parent))
-
-from base_rag_example import BaseRAGExample
-from chunking import CODE_EXTENSIONS, create_text_chunks
-from llama_index.core import SimpleDirectoryReader
-
-
-class CodeRAG(BaseRAGExample):
-    """Specialized RAG example for code repositories with AST-aware chunking."""
-
-    def __init__(self):
-        super().__init__(
-            name="Code",
-            description="Process and query code repositories with AST-aware chunking",
-            default_index_name="code_index",
-        )
-        # Override defaults for code-specific usage
-        self.embedding_model_default = "facebook/contriever"  # Good for code
-        self.max_items_default = -1  # Process all code files by default
-
-    def _add_specific_arguments(self, parser):
-        """Add code-specific arguments."""
-        code_group = parser.add_argument_group("Code Repository Parameters")
-
-        code_group.add_argument(
-            "--repo-dir",
-            type=str,
-            default=".",
-            help="Code repository directory to index (default: current directory)",
-        )
-        code_group.add_argument(
-            "--include-extensions",
-            nargs="+",
-            default=list(CODE_EXTENSIONS.keys()),
-            help="File extensions to include (default: supported code extensions)",
-        )
-        code_group.add_argument(
-            "--exclude-dirs",
-            nargs="+",
-            default=[
-                ".git",
-                "__pycache__",
-                "node_modules",
-                "venv",
-                ".venv",
-                "build",
-                "dist",
-                "target",
-            ],
-            help="Directories to exclude from indexing",
-        )
-        code_group.add_argument(
-            "--max-file-size",
-            type=int,
-            default=1000000,  # 1MB
-            help="Maximum file size in bytes to process (default: 1MB)",
-        )
-        code_group.add_argument(
-            "--include-comments",
-            action="store_true",
-            help="Include comments in chunking (useful for documentation)",
-        )
-        code_group.add_argument(
-            "--preserve-imports",
-            action="store_true",
-            default=True,
-            help="Try to preserve import statements in chunks (default: True)",
-        )
-
-    async def load_data(self, args) -> list[str]:
-        """Load code files and convert to AST-aware chunks."""
-        print(f"🔍 Scanning code repository: {args.repo_dir}")
-        print(f"📁 Including extensions: {args.include_extensions}")
-        print(f"🚫 Excluding directories: {args.exclude_dirs}")
-
-        # Check if repository directory exists
-        repo_path = Path(args.repo_dir)
-        if not repo_path.exists():
-            raise ValueError(f"Repository directory not found: {args.repo_dir}")
-
-        # Load code files with filtering
-        reader_kwargs = {
-            "recursive": True,
-            "encoding": "utf-8",
-            "required_exts": args.include_extensions,
-            "exclude_hidden": True,
-        }
-
-        # Create exclusion filter
-        def file_filter(file_path: str) -> bool:
-            """Filter out unwanted files and directories."""
-            path = Path(file_path)
-
-            # Check file size
-            try:
-                if path.stat().st_size > args.max_file_size:
-                    print(f"⚠️ Skipping large file: {path.name} ({path.stat().st_size} bytes)")
-                    return False
-            except Exception:
-                return False
-
-            # Check if in excluded directory
-            for exclude_dir in args.exclude_dirs:
-                if exclude_dir in path.parts:
-                    return False
-
-            return True
-
-        try:
-            # Load documents with file filtering
-            documents = SimpleDirectoryReader(
-                args.repo_dir,
-                file_extractor=None,  # Use default extractors
-                **reader_kwargs,
-            ).load_data(show_progress=True)
-
-            # Apply custom filtering
-            filtered_docs = []
-            for doc in documents:
-                file_path = doc.metadata.get("file_path", "")
-                if file_filter(file_path):
-                    filtered_docs.append(doc)
-
-            documents = filtered_docs
-
-        except Exception as e:
-            print(f"❌ Error loading code files: {e}")
-            return []
-
-        if not documents:
-            print(
-                f"❌ No code files found in {args.repo_dir} with extensions {args.include_extensions}"
-            )
-            return []
-
-        print(f"✅ Loaded {len(documents)} code files")
-
-        # Show breakdown by language/extension
-        ext_counts = {}
-        for doc in documents:
-            file_path = doc.metadata.get("file_path", "")
-            if file_path:
-                ext = Path(file_path).suffix.lower()
-                ext_counts[ext] = ext_counts.get(ext, 0) + 1
-
-        print("📊 Files by extension:")
-        for ext, count in sorted(ext_counts.items()):
-            print(f"   {ext}: {count} files")
-
-        # Use AST-aware chunking by default for code
-        print(
-            f"🧠 Using AST-aware chunking (chunk_size: {args.ast_chunk_size}, overlap: {args.ast_chunk_overlap})"
-        )
-
-        all_texts = create_text_chunks(
-            documents,
-            chunk_size=256,  # Fallback for non-code files
-            chunk_overlap=64,
-            use_ast_chunking=True,  # Always use AST for code RAG
-            ast_chunk_size=args.ast_chunk_size,
-            ast_chunk_overlap=args.ast_chunk_overlap,
-            code_file_extensions=args.include_extensions,
-            ast_fallback_traditional=True,
-        )
-
-        # Apply max_items limit if specified
-        if args.max_items > 0 and len(all_texts) > args.max_items:
-            print(f"⏳ Limiting to {args.max_items} chunks (from {len(all_texts)})")
-            all_texts = all_texts[: args.max_items]
-
-        print(f"✅ Generated {len(all_texts)} code chunks")
-        return all_texts
-
-
-if __name__ == "__main__":
-    import asyncio
-
-    # Example queries for code RAG
-    print("\n💻 Code RAG Example")
-    print("=" * 50)
-    print("\nExample queries you can try:")
-    print("- 'How does the embedding computation work?'")
-    print("- 'What are the main classes in this codebase?'")
-    print("- 'Show me the search implementation'")
-    print("- 'How is error handling implemented?'")
-    print("- 'What design patterns are used?'")
-    print("- 'Explain the chunking logic'")
-    print("\n🚀 Features:")
-    print("- ✅ AST-aware chunking preserves code structure")
-    print("- ✅ Automatic language detection")
-    print("- ✅ Smart filtering of large files and common excludes")
-    print("- ✅ Optimized for code understanding")
-    print("\nUsage examples:")
-    print("  python -m apps.code_rag --repo-dir ./my_project")
-    print(
-        "  python -m apps.code_rag --include-extensions .py .js --query 'How does authentication work?'"
-    )
-    print("\nOr run without --query for interactive mode\n")
-
-    rag = CodeRAG()
-    asyncio.run(rag.run())
--- a/apps/document_rag.py
+++ b/apps/document_rag.py
@@ -9,8 +9,7 @@ from pathlib import Path
 # Add parent directory to path for imports
 sys.path.insert(0, str(Path(__file__).parent))

-from base_rag_example import BaseRAGExample
-from chunking import create_text_chunks
+from base_rag_example import BaseRAGExample, create_text_chunks
 from llama_index.core import SimpleDirectoryReader


@@ -45,11 +44,6 @@ class DocumentRAG(BaseRAGExample):
        doc_group.add_argument(
            "--chunk-overlap", type=int, default=128, help="Text chunk overlap (default: 128)"
        )
-        doc_group.add_argument(
-            "--enable-code-chunking",
-            action="store_true",
-            help="Enable AST-aware chunking for code files in the data directory",
-        )

    async def load_data(self, args) -> list[str]:
        """Load documents and convert to text chunks."""
@@ -82,22 +76,9 @@ class DocumentRAG(BaseRAGExample):

        print(f"Loaded {len(documents)} documents")

-        # Determine chunking strategy
-        use_ast = args.enable_code_chunking or getattr(args, "use_ast_chunking", False)
-
-        if use_ast:
-            print("Using AST-aware chunking for code files")
-
-        # Convert to text chunks with optional AST support
+        # Convert to text chunks
        all_texts = create_text_chunks(
-            documents,
-            chunk_size=args.chunk_size,
-            chunk_overlap=args.chunk_overlap,
-            use_ast_chunking=use_ast,
-            ast_chunk_size=getattr(args, "ast_chunk_size", 512),
-            ast_chunk_overlap=getattr(args, "ast_chunk_overlap", 64),
-            code_file_extensions=getattr(args, "code_file_extensions", None),
-            ast_fallback_traditional=getattr(args, "ast_fallback_traditional", True),
+            documents, chunk_size=args.chunk_size, chunk_overlap=args.chunk_overlap
        )

        # Apply max_items limit if specified
@@ -121,10 +102,6 @@ if __name__ == "__main__":
    print(
        "- 'What is the problem of developing pan gu model Huawei meets? (盘古大模型开发中遇到什么问题?)'"
    )
-    print("\n🚀 NEW: Code-aware chunking available!")
-    print("- Use --enable-code-chunking to enable AST-aware chunking for code files")
-    print("- Supports Python, Java, C#, TypeScript files")
-    print("- Better semantic understanding of code structure")
    print("\nOr run without --query for interactive mode\n")

    rag = DocumentRAG()
--- a/benchmarks/data/.gitattributes
+++ b/benchmarks/data/.gitattributes
@@ -0,0 +1,82 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.lz4 filter=lfs diff=lfs merge=lfs -text
+*.mds filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Audio files - uncompressed
+*.pcm filter=lfs diff=lfs merge=lfs -text
+*.sam filter=lfs diff=lfs merge=lfs -text
+*.raw filter=lfs diff=lfs merge=lfs -text
+# Audio files - compressed
+*.aac filter=lfs diff=lfs merge=lfs -text
+*.flac filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.ogg filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+# Image files - uncompressed
+*.bmp filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.tiff filter=lfs diff=lfs merge=lfs -text
+# Image files - compressed
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text
+# Video files - compressed
+*.mp4 filter=lfs diff=lfs merge=lfs -text
+*.webm filter=lfs diff=lfs merge=lfs -text
+ground_truth/dpr/id_map.json filter=lfs diff=lfs merge=lfs -text
+indices/dpr/dpr_diskann.passages.idx filter=lfs diff=lfs merge=lfs -text
+indices/dpr/dpr_diskann.passages.jsonl filter=lfs diff=lfs merge=lfs -text
+indices/dpr/dpr_diskann_disk.index filter=lfs diff=lfs merge=lfs -text
+indices/dpr/leann.labels.map filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/leann.labels.map filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.index filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.0.idx filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.0.jsonl filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.1.idx filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.1.jsonl filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.2.idx filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.2.jsonl filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.3.idx filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.3.jsonl filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.4.idx filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.4.jsonl filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.5.idx filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.5.jsonl filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.6.idx filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.6.jsonl filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.7.idx filter=lfs diff=lfs merge=lfs -text
+indices/rpj_wiki/rpj_wiki.passages.7.jsonl filter=lfs diff=lfs merge=lfs -text
--- a/benchmarks/data/README.md
+++ b/benchmarks/data/README.md
@@ -1,44 +0,0 @@
---
-license: mit
---
-
-# LEANN-RAG Evaluation Data
-
-This repository contains the necessary data to run the recall evaluation scripts for the [LEANN-RAG](https://huggingface.co/LEANN-RAG) project.
-
-## Dataset Components
-
-This dataset is structured into three main parts:
-
-1.  **Pre-built LEANN Indices**:
-    *   `dpr/`: A pre-built index for the DPR dataset.
-    *   `rpj_wiki/`: A pre-built index for the RPJ-Wiki dataset.
-    These indices were created using the `leann-core` library and are required by the `LeannSearcher`.
-
-2.  **Ground Truth Data**:
-    *   `ground_truth/`: Contains the ground truth files (`flat_results_nq_k3.json`) for both the DPR and RPJ-Wiki datasets. These files map queries to the original passage IDs from the Natural Questions benchmark, evaluated using the Contriever model.
-
-3.  **Queries**:
-    *   `queries/`: Contains the `nq_open.jsonl` file with the Natural Questions queries used for the evaluation.
-
-## Usage
-
-To use this data, you can download it locally using the `huggingface-hub` library. First, install the library:
-
-```bash
-pip install huggingface-hub
-```
-
-Then, you can download the entire dataset to a local directory (e.g., `data/`) with the following Python script:
-
-```python
-from huggingface_hub import snapshot_download
-
-snapshot_download(
-    repo_id="LEANN-RAG/leann-rag-evaluation-data",
-    repo_type="dataset",
-    local_dir="data"
-)
-```
-
-This will download all the necessary files into a local `data` folder, preserving the repository structure. The evaluation scripts in the main [LEANN-RAG Space](https://huggingface.co/LEANN-RAG) are configured to work with this data structure.
--- a/benchmarks/run_evaluation.py
+++ b/benchmarks/run_evaluation.py
@@ -12,7 +12,7 @@ import time
 from pathlib import Path

 import numpy as np
-from leann.api import LeannBuilder, LeannChat, LeannSearcher
+from leann.api import LeannBuilder, LeannSearcher


 def download_data_if_needed(data_root: Path, download_embeddings: bool = False):
@@ -197,25 +197,6 @@ def main():
    parser.add_argument(
        "--ef-search", type=int, default=120, help="The 'efSearch' parameter for HNSW."
    )
-    parser.add_argument(
-        "--batch-size",
-        type=int,
-        default=0,
-        help="Batch size for HNSW batched search (0 disables batching)",
-    )
-    parser.add_argument(
-        "--llm-type",
-        type=str,
-        choices=["ollama", "hf", "openai", "gemini", "simulated"],
-        default="ollama",
-        help="LLM backend type to optionally query during evaluation (default: ollama)",
-    )
-    parser.add_argument(
-        "--llm-model",
-        type=str,
-        default="qwen3:1.7b",
-        help="LLM model identifier for the chosen backend (default: qwen3:1.7b)",
-    )
    args = parser.parse_args()

    # --- Path Configuration ---
@@ -337,24 +318,9 @@ def main():

        for i in range(num_eval_queries):
            start_time = time.time()
-            new_results = searcher.search(
-                queries[i],
-                top_k=args.top_k,
-                complexity=args.ef_search,
-                batch_size=args.batch_size,
-            )
+            new_results = searcher.search(queries[i], top_k=args.top_k, ef=args.ef_search)
            search_times.append(time.time() - start_time)

-            # Optional: also call the LLM with configurable backend/model (does not affect recall)
-            llm_config = {"type": args.llm_type, "model": args.llm_model}
-            chat = LeannChat(args.index_path, llm_config=llm_config, searcher=searcher)
-            answer = chat.ask(
-                queries[i],
-                top_k=args.top_k,
-                complexity=args.ef_search,
-                batch_size=args.batch_size,
-            )
-            print(f"Answer: {answer}")
            # Correct Recall Calculation: Based on TEXT content
            new_texts = {result.text for result in new_results}

--- a/benchmarks/simple_mac_tpt_test.py
+++ b/benchmarks/simple_mac_tpt_test.py
@@ -20,7 +20,7 @@ except ImportError:

@dataclass
 class BenchmarkConfig:
-    model_path: str = "facebook/contriever-msmarco"
+    model_path: str = "facebook/contriever"
    batch_sizes: list[int] = None
    seq_length: int = 256
    num_runs: int = 5
@@ -34,7 +34,7 @@ class BenchmarkConfig:

    def __post_init__(self):
        if self.batch_sizes is None:
-            self.batch_sizes = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024]
+            self.batch_sizes = [1, 2, 4, 8, 16, 32, 64]


 class MLXBenchmark:
@@ -179,16 +179,10 @@ class Benchmark:

    def _run_inference(self, input_ids: torch.Tensor) -> float:
        attention_mask = torch.ones_like(input_ids)
-        # print shape of input_ids and attention_mask
-        print(f"input_ids shape: {input_ids.shape}")
-        print(f"attention_mask shape: {attention_mask.shape}")
+
        start_time = time.time()
        with torch.no_grad():
            self.model(input_ids=input_ids, attention_mask=attention_mask)
-        if torch.cuda.is_available():
-            torch.cuda.synchronize()
-        if torch.backends.mps.is_available():
-            torch.mps.synchronize()
        end_time = time.time()

        return end_time - start_time
--- a/docs/ast_chunking_guide.md
+++ b/docs/ast_chunking_guide.md
@@ -1,128 +0,0 @@
-# AST-Aware Code chunking guide
-
-## Overview
-
-This guide covers best practices for using AST-aware code chunking in LEANN. AST chunking provides better semantic understanding of code structure compared to traditional text-based chunking.
-
-## Quick Start
-
-### Basic Usage
-
-```bash
-# Enable AST chunking for mixed content (code + docs)
-python -m apps.document_rag --enable-code-chunking --data-dir ./my_project
-
-# Specialized code repository indexing
-python -m apps.code_rag --repo-dir ./my_codebase
-
-# Global CLI with AST support
-leann build my-code-index --docs ./src --use-ast-chunking
-```
-
-### Installation
-
-```bash
-# Install LEANN with AST chunking support
-uv pip install -e "."
-```
-
-## Best Practices
-
-### When to Use AST Chunking
-
-✅ **Recommended for:**
- Code repositories with multiple languages
- Mixed documentation and code content
- Complex codebases with deep function/class hierarchies
- When working with Claude Code for code assistance
-
-❌ **Not recommended for:**
- Pure text documents
- Very large files (>1MB)
- Languages not supported by tree-sitter
-
-### Optimal Configuration
-
-```bash
-# Recommended settings for most codebases
-python -m apps.code_rag \
-    --repo-dir ./src \
-    --ast-chunk-size 768 \
-    --ast-chunk-overlap 96 \
-    --exclude-dirs .git __pycache__ node_modules build dist
-```
-
-### Supported Languages
-
-| Extension | Language | Status |
-|-----------|----------|--------|
-| `.py` | Python | ✅ Full support |
-| `.java` | Java | ✅ Full support |
-| `.cs` | C# | ✅ Full support |
-| `.ts`, `.tsx` | TypeScript | ✅ Full support |
-| `.js`, `.jsx` | JavaScript | ✅ Via TypeScript parser |
-
-## Integration Examples
-
-### Document RAG with Code Support
-
-```python
-# Enable code chunking in document RAG
-python -m apps.document_rag \
-    --enable-code-chunking \
-    --data-dir ./project \
-    --query "How does authentication work in the codebase?"
-```
-
-### Claude Code Integration
-
-When using with Claude Code MCP server, AST chunking provides better context for:
- Code completion and suggestions
- Bug analysis and debugging
- Architecture understanding
- Refactoring assistance
-
-## Troubleshooting
-
-### Common Issues
-
-1. **Fallback to Traditional Chunking**
-   - Normal behavior for unsupported languages
-   - Check logs for specific language support
-
-2. **Performance with Large Files**
-   - Adjust `--max-file-size` parameter
-   - Use `--exclude-dirs` to skip unnecessary directories
-
-3. **Quality Issues**
-   - Try different `--ast-chunk-size` values (512, 768, 1024)
-   - Adjust overlap for better context preservation
-
-### Debug Mode
-
-```bash
-export LEANN_LOG_LEVEL=DEBUG
-python -m apps.code_rag --repo-dir ./my_code
-```
-
-## Migration from Traditional Chunking
-
-Existing workflows continue to work without changes. To enable AST chunking:
-
-```bash
-# Before
-python -m apps.document_rag --chunk-size 256
-
-# After (maintains traditional chunking for non-code files)
-python -m apps.document_rag --enable-code-chunking --chunk-size 256 --ast-chunk-size 768
-```
-
-## References
-
- [astchunk GitHub Repository](https://github.com/yilinjz/astchunk)
- [LEANN MCP Integration](../packages/leann-mcp/README.md)
- [Research Paper](https://arxiv.org/html/2506.15655v1)
-
---
-
-**Note**: AST chunking maintains full backward compatibility while enhancing code understanding capabilities.
--- a/docs/features.md
+++ b/docs/features.md
@@ -3,7 +3,6 @@
 ## 🔥 Core Features

 - **🔄 Real-time Embeddings** - Eliminate heavy embedding storage with dynamic computation using optimized ZMQ servers and highly optimized search paradigm (overlapping and batching) with highly optimized embedding engine
- **🧠 AST-Aware Code Chunking** - Intelligent code chunking that preserves semantic boundaries (functions, classes, methods) for Python, Java, C#, and TypeScript files
 - **📈 Scalable Architecture** - Handles millions of documents on consumer hardware; the larger your dataset, the more LEANN can save
 - **🎯 Graph Pruning** - Advanced techniques to minimize the storage overhead of vector search to a limited footprint
 - **🏗️ Pluggable Backends** - HNSW/FAISS (default), with optional DiskANN for large-scale deployments
--- a/packages/leann-backend-diskann/leann_backend_diskann/diskann_embedding_server.py
+++ b/packages/leann-backend-diskann/leann_backend_diskann/diskann_embedding_server.py
@@ -83,7 +83,9 @@ def create_diskann_embedding_server(

    logger.info(f"Loading PassageManager with metadata_file_path: {passages_file}")
    passages = PassageManager(meta["passage_sources"], metadata_file_path=passages_file)
-    logger.info(f"Loaded PassageManager with {len(passages)} passages from metadata")
+    logger.info(
+        f"Loaded PassageManager with {len(passages.global_offset_map)} passages from metadata"
+    )

    # Import protobuf after ensuring the path is correct
    try:
--- a/packages/leann-backend-diskann/pyproject.toml
+++ b/packages/leann-backend-diskann/pyproject.toml
@@ -4,8 +4,8 @@ build-backend = "scikit_build_core.build"

 [project]
 name = "leann-backend-diskann"
-version = "0.3.2"
-dependencies = ["leann-core==0.3.2", "numpy", "protobuf>=3.19.0"]
+version = "0.3.0"
+dependencies = ["leann-core==0.3.0", "numpy", "protobuf>=3.19.0"]

 [tool.scikit-build]
 # Key: simplified CMake path
--- a/packages/leann-backend-diskann/third_party/DiskANN
+++ b/packages/leann-backend-diskann/third_party/DiskANN
--- a/packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_backend.py
+++ b/packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_backend.py
@@ -1,7 +1,6 @@
 import logging
 import os
 import shutil
-import time
 from pathlib import Path
 from typing import Any, Literal, Optional

@@ -237,7 +236,6 @@ class HNSWSearcher(BaseSearcher):
        distances = np.empty((batch_size_query, top_k), dtype=np.float32)
        labels = np.empty((batch_size_query, top_k), dtype=np.int64)

-        search_time = time.time()
        self._index.search(
            query.shape[0],
            faiss.swig_ptr(query),
@@ -246,8 +244,7 @@ class HNSWSearcher(BaseSearcher):
            faiss.swig_ptr(labels),
            params,
        )
-        search_time = time.time() - search_time
-        logger.info(f"  Search time in HNSWSearcher.search() backend: {search_time} seconds")
+
        string_labels = [[str(int_label) for int_label in batch_labels] for batch_labels in labels]

        return {"labels": string_labels, "distances": distances}
--- a/packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_embedding_server.py
+++ b/packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_embedding_server.py
@@ -90,7 +90,9 @@ def create_hnsw_embedding_server(
        embedding_dim: int = int(meta.get("dimensions", 0))
    except Exception:
        embedding_dim = 0
-    logger.info(f"Loaded PassageManager with {len(passages)} passages from metadata")
+    logger.info(
+        f"Loaded PassageManager with {len(passages.global_offset_map)} passages from metadata"
+    )

    # (legacy ZMQ thread removed; using shutdown-capable server only)

--- a/packages/leann-backend-hnsw/pyproject.toml
+++ b/packages/leann-backend-hnsw/pyproject.toml
@@ -6,10 +6,10 @@ build-backend = "scikit_build_core.build"

 [project]
 name = "leann-backend-hnsw"
-version = "0.3.2"
+version = "0.3.0"
 description = "Custom-built HNSW (Faiss) backend for the Leann toolkit."
 dependencies = [
-    "leann-core==0.3.2",
+    "leann-core==0.3.0",
    "numpy",
    "pyzmq>=23.0.0",
    "msgpack>=1.0.0",
--- a/packages/leann-core/pyproject.toml
+++ b/packages/leann-core/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"

 [project]
 name = "leann-core"
-version = "0.3.2"
+version = "0.3.0"
 description = "Core API and plugin system for LEANN"
 readme = "README.md"
 requires-python = ">=3.9"
--- a/packages/leann-core/src/leann/api.py
+++ b/packages/leann-core/src/leann/api.py
@@ -119,12 +119,9 @@ class PassageManager:
    def __init__(
        self, passage_sources: list[dict[str, Any]], metadata_file_path: Optional[str] = None
    ):
-        self.offset_maps: dict[str, dict[str, int]] = {}
-        self.passage_files: dict[str, str] = {}
-        # Avoid materializing a single gigantic global map to reduce memory
-        # footprint on very large corpora (e.g., 60M+ passages). Instead, keep
-        # per-shard maps and do a lightweight per-shard lookup on demand.
-        self._total_count: int = 0
+        self.offset_maps = {}
+        self.passage_files = {}
+        self.global_offset_map = {}  # Combined map for fast lookup

        # Derive index base name for standard sibling fallbacks, e.g., <index_name>.passages.*
        index_name_base = None
@@ -145,25 +142,12 @@ class PassageManager:
                default_name: Optional[str],
                source_dict: dict[str, Any],
            ) -> list[Path]:
-                """
-                Build an ordered list of candidate paths. For relative paths specified in
-                metadata, prefer resolution relative to the metadata file directory first,
-                then fall back to CWD-based resolution, and finally to conventional
-                sibling defaults (e.g., <index_base>.passages.idx / .jsonl).
-                """
                candidates: list[Path] = []
-                # 1) Primary path
+                # 1) Primary as-is (absolute or relative)
                if primary:
                    p = Path(primary)
-                    if p.is_absolute():
-                        candidates.append(p)
-                    else:
-                        # Prefer metadata-relative resolution for relative paths
-                        if metadata_file_path:
-                            candidates.append(Path(metadata_file_path).parent / p)
-                        # Also consider CWD-relative as a fallback for legacy layouts
-                        candidates.append(Path.cwd() / p)
-                # 2) metadata-relative explicit relative key (if present)
+                    candidates.append(p if p.is_absolute() else (Path.cwd() / p))
+                # 2) metadata-relative explicit relative key
                if metadata_file_path and source_dict.get(relative_key):
                    candidates.append(Path(metadata_file_path).parent / source_dict[relative_key])
                # 3) metadata-relative standard sibling filename
@@ -193,28 +177,23 @@ class PassageManager:
                raise FileNotFoundError(f"Passage index file not found: {index_file}")

            with open(index_file, "rb") as f:
-                offset_map: dict[str, int] = pickle.load(f)
+                offset_map = pickle.load(f)
                self.offset_maps[passage_file] = offset_map
                self.passage_files[passage_file] = passage_file
-                self._total_count += len(offset_map)
+
+                # Build global map for O(1) lookup
+                for passage_id, offset in offset_map.items():
+                    self.global_offset_map[passage_id] = (passage_file, offset)

    def get_passage(self, passage_id: str) -> dict[str, Any]:
-        # Fast path: check each shard map (there are typically few shards).
-        # This avoids building a massive combined dict while keeping lookups
-        # bounded by the number of shards.
-        for passage_file, offset_map in self.offset_maps.items():
-            try:
-                offset = offset_map[passage_id]
-                with open(passage_file, encoding="utf-8") as f:
-                    f.seek(offset)
-                    return json.loads(f.readline())
-            except KeyError:
-                continue
+        if passage_id in self.global_offset_map:
+            passage_file, offset = self.global_offset_map[passage_id]
+            # Lazy file opening - only open when needed
+            with open(passage_file, encoding="utf-8") as f:
+                f.seek(offset)
+                return json.loads(f.readline())
        raise KeyError(f"Passage ID not found: {passage_id}")

-    def __len__(self) -> int:
-        return self._total_count
-

 class LeannBuilder:
    def __init__(
@@ -578,8 +557,6 @@ class LeannSearcher:
        self.passage_manager = PassageManager(
            self.meta_data.get("passage_sources", []), metadata_file_path=self.meta_path_str
        )
-        # Preserve backend name for conditional parameter forwarding
-        self.backend_name = backend_name
        backend_factory = BACKEND_REGISTRY.get(backend_name)
        if backend_factory is None:
            raise ValueError(f"Backend '{backend_name}' not found.")
@@ -599,7 +576,6 @@ class LeannSearcher:
        recompute_embeddings: bool = True,
        pruning_strategy: Literal["global", "local", "proportional"] = "global",
        expected_zmq_port: int = 5557,
-        batch_size: int = 0,
        **kwargs,
    ) -> list[SearchResult]:
        logger.info("🔍 LeannSearcher.search() called:")
@@ -608,9 +584,7 @@ class LeannSearcher:
        logger.info(f"  Additional kwargs: {kwargs}")

        # Smart top_k detection and adjustment
-        # Use PassageManager length (sum of shard sizes) to avoid
-        # depending on a massive combined map
-        total_docs = len(self.passage_manager)
+        total_docs = len(self.passage_manager.global_offset_map)
        original_top_k = top_k
        if top_k > total_docs:
            top_k = total_docs
@@ -639,33 +613,23 @@ class LeannSearcher:
            use_server_if_available=recompute_embeddings,
            zmq_port=zmq_port,
        )
-        logger.info(f"  Generated embedding shape: {query_embedding.shape}")
-        embedding_time = time.time() - start_time
-        logger.info(f"  Embedding time: {embedding_time} seconds")
+        # logger.info(f"  Generated embedding shape: {query_embedding.shape}")
+        time.time() - start_time
+        # logger.info(f"  Embedding time: {embedding_time} seconds")

        start_time = time.time()
-        backend_search_kwargs: dict[str, Any] = {
-            "complexity": complexity,
-            "beam_width": beam_width,
-            "prune_ratio": prune_ratio,
-            "recompute_embeddings": recompute_embeddings,
-            "pruning_strategy": pruning_strategy,
-            "zmq_port": zmq_port,
-        }
-        # Only HNSW supports batching; forward conditionally
-        if self.backend_name == "hnsw":
-            backend_search_kwargs["batch_size"] = batch_size
-
-        # Merge any extra kwargs last
-        backend_search_kwargs.update(kwargs)
-
        results = self.backend_impl.search(
            query_embedding,
            top_k,
-            **backend_search_kwargs,
+            complexity=complexity,
+            beam_width=beam_width,
+            prune_ratio=prune_ratio,
+            recompute_embeddings=recompute_embeddings,
+            pruning_strategy=pruning_strategy,
+            zmq_port=zmq_port,
+            **kwargs,
        )
-        search_time = time.time() - start_time
-        logger.info(f"  Search time in search() LEANN searcher: {search_time} seconds")
+        # logger.info(f"  Search time: {search_time} seconds")
        logger.info(f"  Backend returned: labels={len(results.get('labels', [[]])[0])} results")

        enriched_results = []
@@ -716,9 +680,8 @@ class LeannSearcher:
        This method should be called after you're done using the searcher,
        especially in test environments or batch processing scenarios.
        """
-        backend = getattr(self.backend_impl, "embedding_server_manager", None)
-        if backend is not None:
-            backend.stop_server()
+        if hasattr(self.backend_impl, "embedding_server_manager"):
+            self.backend_impl.embedding_server_manager.stop_server()

    # Enable automatic cleanup patterns
    def __enter__(self):
@@ -744,15 +707,9 @@ class LeannChat:
        index_path: str,
        llm_config: Optional[dict[str, Any]] = None,
        enable_warmup: bool = False,
-        searcher: Optional[LeannSearcher] = None,
        **kwargs,
    ):
-        if searcher is None:
-            self.searcher = LeannSearcher(index_path, enable_warmup=enable_warmup, **kwargs)
-            self._owns_searcher = True
-        else:
-            self.searcher = searcher
-            self._owns_searcher = False
+        self.searcher = LeannSearcher(index_path, enable_warmup=enable_warmup, **kwargs)
        self.llm = get_llm(llm_config)

    def ask(
@@ -766,7 +723,6 @@ class LeannChat:
        pruning_strategy: Literal["global", "local", "proportional"] = "global",
        llm_kwargs: Optional[dict[str, Any]] = None,
        expected_zmq_port: int = 5557,
-        batch_size: int = 0,
        **search_kwargs,
    ):
        if llm_kwargs is None:
@@ -781,11 +737,10 @@ class LeannChat:
            recompute_embeddings=recompute_embeddings,
            pruning_strategy=pruning_strategy,
            expected_zmq_port=expected_zmq_port,
-            batch_size=batch_size,
            **search_kwargs,
        )
        search_time = time.time() - search_time
-        logger.info(f"  Search time: {search_time} seconds")
+        # logger.info(f"  Search time: {search_time} seconds")
        context = "\n\n".join([r.text for r in results])
        prompt = (
            "Here is some retrieved context that might help answer your question:\n\n"
@@ -821,9 +776,7 @@ class LeannChat:
        This method should be called after you're done using the chat interface,
        especially in test environments or batch processing scenarios.
        """
-        # Only stop the embedding server if this LeannChat instance created the searcher.
-        # When a shared searcher is passed in, avoid shutting down the server to enable reuse.
-        if getattr(self, "_owns_searcher", False) and hasattr(self.searcher, "cleanup"):
+        if hasattr(self.searcher, "cleanup"):
            self.searcher.cleanup()

    # Enable automatic cleanup patterns
--- a/packages/leann-core/src/leann/chat.py
+++ b/packages/leann-core/src/leann/chat.py
@@ -707,28 +707,20 @@ class GeminiChat(LLMInterface):
        logger.info(f"Sending request to Gemini with model {self.model}")

        try:
-            from google.genai.types import GenerateContentConfig
-
-            generation_config = GenerateContentConfig(
-                temperature=kwargs.get("temperature", 0.7),
-                max_output_tokens=kwargs.get("max_tokens", 1000),
-            )
+            # Set generation configuration
+            generation_config = {
+                "temperature": kwargs.get("temperature", 0.7),
+                "max_output_tokens": kwargs.get("max_tokens", 1000),
+            }

            # Handle top_p parameter
            if "top_p" in kwargs:
-                generation_config.top_p = kwargs["top_p"]
+                generation_config["top_p"] = kwargs["top_p"]

            response = self.client.models.generate_content(
-                model=self.model,
-                contents=prompt,
-                config=generation_config,
+                model=self.model, contents=prompt, config=generation_config
            )
-            # Handle potential None response text
-            response_text = response.text
-            if response_text is None:
-                logger.warning("Gemini returned None response text")
-                return ""
-            return response_text.strip()
+            return response.text.strip()
        except Exception as e:
            logger.error(f"Error communicating with Gemini: {e}")
            return f"Error: Could not get a response from Gemini. Details: {e}"
--- a/packages/leann-core/src/leann/cli.py
+++ b/packages/leann-core/src/leann/cli.py
@@ -1,15 +1,13 @@
 import argparse
 import asyncio
-import sys
 from pathlib import Path
-from typing import Any, Optional, Union
+from typing import Union

 from llama_index.core import SimpleDirectoryReader
 from llama_index.core.node_parser import SentenceSplitter
 from tqdm import tqdm

 from .api import LeannBuilder, LeannChat, LeannSearcher
-from .registry import register_project_directory


 def extract_pdf_text_with_pymupdf(file_path: str) -> str:
@@ -181,29 +179,6 @@ Examples:
            default=50,
            help="Code chunk overlap (default: 50)",
        )
-        build_parser.add_argument(
-            "--use-ast-chunking",
-            action="store_true",
-            help="Enable AST-aware chunking for code files (requires astchunk)",
-        )
-        build_parser.add_argument(
-            "--ast-chunk-size",
-            type=int,
-            default=768,
-            help="AST chunk size in characters (default: 768)",
-        )
-        build_parser.add_argument(
-            "--ast-chunk-overlap",
-            type=int,
-            default=96,
-            help="AST chunk overlap in characters (default: 96)",
-        )
-        build_parser.add_argument(
-            "--ast-fallback-traditional",
-            action="store_true",
-            default=True,
-            help="Fall back to traditional chunking if AST chunking fails (default: True)",
-        )

        # Search command
        search_parser = subparsers.add_parser("search", help="Search documents")
@@ -230,11 +205,6 @@ Examples:
            default="global",
            help="Pruning strategy (default: global)",
        )
-        search_parser.add_argument(
-            "--non-interactive",
-            action="store_true",
-            help="Non-interactive mode: automatically select index without prompting",
-        )

        # Ask command
        ask_parser = subparsers.add_parser("ask", help="Ask questions")
@@ -293,7 +263,31 @@ Examples:

    def register_project_dir(self):
        """Register current project directory in global registry"""
-        register_project_directory()
+        global_registry = Path.home() / ".leann" / "projects.json"
+        global_registry.parent.mkdir(exist_ok=True)
+
+        current_dir = str(Path.cwd())
+
+        # Load existing registry
+        projects = []
+        if global_registry.exists():
+            try:
+                import json
+
+                with open(global_registry) as f:
+                    projects = json.load(f)
+            except Exception:
+                projects = []
+
+        # Add current directory if not already present
+        if current_dir not in projects:
+            projects.append(current_dir)
+
+        # Save registry
+        import json
+
+        with open(global_registry, "w") as f:
+            json.dump(projects, f, indent=2)

    def _build_gitignore_parser(self, docs_dir: str):
        """Build gitignore parser using gitignore-parser library."""
@@ -379,10 +373,13 @@ Examples:
            valid_projects.append(current_path)

        # Separate current and other projects
+        current_project = None
        other_projects = []

        for project_path in valid_projects:
-            if project_path != current_path:
+            if project_path == current_path:
+                current_project = project_path
+            else:
                other_projects.append(project_path)

        print("📚 LEANN Indexes")
@@ -392,20 +389,35 @@ Examples:
        current_indexes_count = 0

        # Show current project first (most important)
-        print("\n🏠 Current Project")
-        print(f"   {current_path}")
-        print("   " + "─" * 45)
+        if current_project:
+            current_indexes_dir = current_project / ".leann" / "indexes"
+            if current_indexes_dir.exists():
+                current_index_dirs = [d for d in current_indexes_dir.iterdir() if d.is_dir()]

-        current_indexes = self._discover_indexes_in_project(current_path)
-        if current_indexes:
-            for idx in current_indexes:
-                total_indexes += 1
-                current_indexes_count += 1
-                type_icon = "📁" if idx["type"] == "cli" else "📄"
-                print(f"   {current_indexes_count}. {type_icon} {idx['name']} {idx['status']}")
-                if idx["size_mb"] > 0:
-                    print(f"      📦 Size: {idx['size_mb']:.1f} MB")
+                print("\n🏠 Current Project")
+                print(f"   {current_project}")
+                print("   " + "─" * 45)
+
+                if current_index_dirs:
+                    for index_dir in current_index_dirs:
+                        total_indexes += 1
+                        current_indexes_count += 1
+                        index_name = index_dir.name
+                        meta_file = index_dir / "documents.leann.meta.json"
+                        status = "✅" if meta_file.exists() else "❌"
+
+                        print(f"   {current_indexes_count}. {index_name} {status}")
+                        if meta_file.exists():
+                            size_mb = sum(
+                                f.stat().st_size for f in index_dir.iterdir() if f.is_file()
+                            ) / (1024 * 1024)
+                            print(f"      📦 Size: {size_mb:.1f} MB")
+                else:
+                    print("   📭 No indexes in current project")
        else:
+            print("\n🏠 Current Project")
+            print(f"   {current_path}")
+            print("   " + "─" * 45)
            print("   📭 No indexes in current project")

        # Show other projects (reference information)
@@ -414,19 +426,29 @@ Examples:
            print("   " + "─" * 45)

            for project_path in other_projects:
-                project_indexes = self._discover_indexes_in_project(project_path)
-                if not project_indexes:
+                indexes_dir = project_path / ".leann" / "indexes"
+                if not indexes_dir.exists():
+                    continue
+
+                index_dirs = [d for d in indexes_dir.iterdir() if d.is_dir()]
+                if not index_dirs:
                    continue

                print(f"\n   📂 {project_path.name}")
                print(f"      {project_path}")

-                for idx in project_indexes:
+                for index_dir in index_dirs:
                    total_indexes += 1
-                    type_icon = "📁" if idx["type"] == "cli" else "📄"
-                    print(f"      • {type_icon} {idx['name']} {idx['status']}")
-                    if idx["size_mb"] > 0:
-                        print(f"        📦 {idx['size_mb']:.1f} MB")
+                    index_name = index_dir.name
+                    meta_file = index_dir / "documents.leann.meta.json"
+                    status = "✅" if meta_file.exists() else "❌"
+
+                    print(f"      • {index_name} {status}")
+                    if meta_file.exists():
+                        size_mb = sum(
+                            f.stat().st_size for f in index_dir.iterdir() if f.is_file()
+                        ) / (1024 * 1024)
+                        print(f"        📦 {size_mb:.1f} MB")

        # Summary and usage info
        print("\n" + "=" * 50)
@@ -434,9 +456,13 @@ Examples:
            print("💡 Get started:")
            print("   leann build my-docs --docs ./documents")
        else:
-            # Count only projects that have at least one discoverable index
-            projects_count = sum(
-                1 for p in valid_projects if len(self._discover_indexes_in_project(p)) > 0
+            projects_count = len(
+                [
+                    p
+                    for p in valid_projects
+                    if (p / ".leann" / "indexes").exists()
+                    and list((p / ".leann" / "indexes").iterdir())
+                ]
            )
            print(f"📊 Total: {total_indexes} indexes across {projects_count} projects")

@@ -454,76 +480,6 @@ Examples:
                print("\n💡 Create your first index:")
                print("   leann build my-docs --docs ./documents")

-    def _discover_indexes_in_project(self, project_path: Path):
-        """Discover all indexes in a project directory (both CLI and apps formats)"""
-        indexes = []
-
-        # 1. CLI format: .leann/indexes/index_name/
-        cli_indexes_dir = project_path / ".leann" / "indexes"
-        if cli_indexes_dir.exists():
-            for index_dir in cli_indexes_dir.iterdir():
-                if index_dir.is_dir():
-                    meta_file = index_dir / "documents.leann.meta.json"
-                    status = "✅" if meta_file.exists() else "❌"
-
-                    size_mb = 0
-                    if meta_file.exists():
-                        try:
-                            size_mb = sum(
-                                f.stat().st_size for f in index_dir.iterdir() if f.is_file()
-                            ) / (1024 * 1024)
-                        except (OSError, PermissionError):
-                            pass
-
-                    indexes.append(
-                        {
-                            "name": index_dir.name,
-                            "type": "cli",
-                            "status": status,
-                            "size_mb": size_mb,
-                            "path": index_dir,
-                        }
-                    )
-
-        # 2. Apps format: *.leann.meta.json files anywhere in the project
-        cli_indexes_dir = project_path / ".leann" / "indexes"
-        for meta_file in project_path.rglob("*.leann.meta.json"):
-            if meta_file.is_file():
-                # Skip CLI-built indexes (which store meta under .leann/indexes/<name>/)
-                try:
-                    if cli_indexes_dir.exists() and cli_indexes_dir in meta_file.parents:
-                        continue
-                except Exception:
-                    pass
-                # Use the parent directory name as the app index display name
-                display_name = meta_file.parent.name
-                # Extract file base used to store files
-                file_base = meta_file.name.replace(".leann.meta.json", "")
-
-                # Apps indexes are considered complete if the .leann.meta.json file exists
-                status = "✅"
-
-                # Calculate total size of all related files (use file base)
-                size_mb = 0
-                try:
-                    index_dir = meta_file.parent
-                    for related_file in index_dir.glob(f"{file_base}.leann*"):
-                        size_mb += related_file.stat().st_size / (1024 * 1024)
-                except (OSError, PermissionError):
-                    pass
-
-                indexes.append(
-                    {
-                        "name": display_name,
-                        "type": "app",
-                        "status": status,
-                        "size_mb": size_mb,
-                        "path": meta_file,
-                    }
-                )
-
-        return indexes
-
    def remove_index(self, index_name: str, force: bool = False):
        """Safely remove an index - always show all matches for transparency"""

@@ -568,79 +524,13 @@ Examples:
            if not project_path.exists():
                continue

-            # 1) CLI-format index under .leann/indexes/<name>
            index_dir = project_path / ".leann" / "indexes" / index_name
            if index_dir.exists():
                is_current = project_path == current_path
                matches.append(
-                    {
-                        "project_path": project_path,
-                        "index_dir": index_dir,
-                        "is_current": is_current,
-                        "kind": "cli",
-                    }
+                    {"project_path": project_path, "index_dir": index_dir, "is_current": is_current}
                )

-            # 2) App-format indexes
-            # We support two ways of addressing apps:
-            #   a) by the file base (e.g., `pdf_documents`)
-            #   b) by the parent directory name (e.g., `new_txt`)
-            seen_app_meta = set()
-
-            # 2a) by file base
-            for meta_file in project_path.rglob(f"{index_name}.leann.meta.json"):
-                if meta_file.is_file():
-                    # Skip CLI-built indexes' meta under .leann/indexes
-                    try:
-                        cli_indexes_dir = project_path / ".leann" / "indexes"
-                        if cli_indexes_dir.exists() and cli_indexes_dir in meta_file.parents:
-                            continue
-                    except Exception:
-                        pass
-                    is_current = project_path == current_path
-                    key = (str(project_path), str(meta_file))
-                    if key in seen_app_meta:
-                        continue
-                    seen_app_meta.add(key)
-                    matches.append(
-                        {
-                            "project_path": project_path,
-                            "files_dir": meta_file.parent,
-                            "meta_file": meta_file,
-                            "is_current": is_current,
-                            "kind": "app",
-                            "display_name": meta_file.parent.name,
-                            "file_base": meta_file.name.replace(".leann.meta.json", ""),
-                        }
-                    )
-
-            # 2b) by parent directory name
-            for meta_file in project_path.rglob("*.leann.meta.json"):
-                if meta_file.is_file() and meta_file.parent.name == index_name:
-                    # Skip CLI-built indexes' meta under .leann/indexes
-                    try:
-                        cli_indexes_dir = project_path / ".leann" / "indexes"
-                        if cli_indexes_dir.exists() and cli_indexes_dir in meta_file.parents:
-                            continue
-                    except Exception:
-                        pass
-                    is_current = project_path == current_path
-                    key = (str(project_path), str(meta_file))
-                    if key in seen_app_meta:
-                        continue
-                    seen_app_meta.add(key)
-                    matches.append(
-                        {
-                            "project_path": project_path,
-                            "files_dir": meta_file.parent,
-                            "meta_file": meta_file,
-                            "is_current": is_current,
-                            "kind": "app",
-                            "display_name": meta_file.parent.name,
-                            "file_base": meta_file.name.replace(".leann.meta.json", ""),
-                        }
-                    )
-
        # Sort: current project first, then by project name
        matches.sort(key=lambda x: (not x["is_current"], x["project_path"].name))
        return matches
@@ -648,8 +538,8 @@ Examples:
    def _remove_single_match(self, match, index_name: str, force: bool):
        """Handle removal when only one match is found"""
        project_path = match["project_path"]
+        index_dir = match["index_dir"]
        is_current = match["is_current"]
-        kind = match.get("kind", "cli")

        if is_current:
            location_info = "current project"
@@ -660,10 +550,7 @@ Examples:

        print(f"✅ Found 1 index named '{index_name}':")
        print(f"   {emoji} Location: {location_info}")
-        if kind == "cli":
-            print(f"   📍 Path: {project_path / '.leann' / 'indexes' / index_name}")
-        else:
-            print(f"   📍 Meta: {match['meta_file']}")
+        print(f"   📍 Path: {project_path}")

        if not force:
            if not is_current:
@@ -675,22 +562,9 @@ Examples:
                print("   ❌ Removal cancelled.")
                return False

-        if kind == "cli":
-            return self._delete_index_directory(
-                match["index_dir"],
-                index_name,
-                project_path if not is_current else None,
-                is_app=False,
-            )
-        else:
-            return self._delete_index_directory(
-                match["files_dir"],
-                match.get("display_name", index_name),
-                project_path if not is_current else None,
-                is_app=True,
-                meta_file=match.get("meta_file"),
-                app_file_base=match.get("file_base"),
-            )
+        return self._delete_index_directory(
+            index_dir, index_name, project_path if not is_current else None
+        )

    def _remove_from_multiple_matches(self, matches, index_name: str, force: bool):
        """Handle removal when multiple matches are found"""
@@ -701,34 +575,19 @@ Examples:
        for i, match in enumerate(matches, 1):
            project_path = match["project_path"]
            is_current = match["is_current"]
-            kind = match.get("kind", "cli")

            if is_current:
-                print(f"   {i}. 🏠 Current project ({'CLI' if kind == 'cli' else 'APP'})")
+                print(f"   {i}. 🏠 Current project")
+                print(f"      📍 {project_path}")
            else:
-                print(f"   {i}. 📂 {project_path.name} ({'CLI' if kind == 'cli' else 'APP'})")
-
-            # Show path details
-            if kind == "cli":
-                print(f"      📍 {project_path / '.leann' / 'indexes' / index_name}")
-            else:
-                print(f"      📍 {match['meta_file']}")
+                print(f"   {i}. 📂 {project_path.name}")
+                print(f"      📍 {project_path}")

            # Show size info
            try:
-                if kind == "cli":
-                    size_mb = sum(
-                        f.stat().st_size for f in match["index_dir"].iterdir() if f.is_file()
-                    ) / (1024 * 1024)
-                else:
-                    file_base = match.get("file_base")
-                    size_mb = 0.0
-                    if file_base:
-                        size_mb = sum(
-                            f.stat().st_size
-                            for f in match["files_dir"].glob(f"{file_base}.leann*")
-                            if f.is_file()
-                        ) / (1024 * 1024)
+                size_mb = sum(
+                    f.stat().st_size for f in match["index_dir"].iterdir() if f.is_file()
+                ) / (1024 * 1024)
                print(f"      📦 Size: {size_mb:.1f} MB")
            except (OSError, PermissionError):
                pass
@@ -752,8 +611,8 @@ Examples:
            if 0 <= choice_idx < len(matches):
                selected_match = matches[choice_idx]
                project_path = selected_match["project_path"]
+                index_dir = selected_match["index_dir"]
                is_current = selected_match["is_current"]
-                kind = selected_match.get("kind", "cli")

                location = "current project" if is_current else f"'{project_path.name}' project"
                print(f"   🎯 Selected: Remove from {location}")
@@ -766,22 +625,9 @@ Examples:
                    print("   ❌ Confirmation failed. Removal cancelled.")
                    return False

-                if kind == "cli":
-                    return self._delete_index_directory(
-                        selected_match["index_dir"],
-                        index_name,
-                        project_path if not is_current else None,
-                        is_app=False,
-                    )
-                else:
-                    return self._delete_index_directory(
-                        selected_match["files_dir"],
-                        selected_match.get("display_name", index_name),
-                        project_path if not is_current else None,
-                        is_app=True,
-                        meta_file=selected_match.get("meta_file"),
-                        app_file_base=selected_match.get("file_base"),
-                    )
+                return self._delete_index_directory(
+                    index_dir, index_name, project_path if not is_current else None
+                )
            else:
                print("   ❌ Invalid choice. Removal cancelled.")
                return False
@@ -791,65 +637,21 @@ Examples:
            return False

    def _delete_index_directory(
-        self,
-        index_dir: Path,
-        index_display_name: str,
-        project_path: Optional[Path] = None,
-        is_app: bool = False,
-        meta_file: Optional[Path] = None,
-        app_file_base: Optional[str] = None,
+        self, index_dir: Path, index_name: str, project_path: Path | None = None
    ):
-        """Delete a CLI index directory or APP index files safely."""
+        """Actually delete the index directory"""
        try:
-            if is_app:
-                removed = 0
-                errors = 0
-                # Delete only files that belong to this app index (based on file base)
-                pattern_base = app_file_base or ""
-                for f in index_dir.glob(f"{pattern_base}.leann*"):
-                    try:
-                        f.unlink()
-                        removed += 1
-                    except Exception:
-                        errors += 1
-                # Best-effort: also remove the meta file if specified and still exists
-                if meta_file and meta_file.exists():
-                    try:
-                        meta_file.unlink()
-                        removed += 1
-                    except Exception:
-                        errors += 1
+            import shutil

-                if removed > 0 and errors == 0:
-                    if project_path:
-                        print(
-                            f"✅ App index '{index_display_name}' removed from {project_path.name}"
-                        )
-                    else:
-                        print(f"✅ App index '{index_display_name}' removed successfully")
-                    return True
-                elif removed > 0 and errors > 0:
-                    print(
-                        f"⚠️  App index '{index_display_name}' partially removed (some files couldn't be deleted)"
-                    )
-                    return True
-                else:
-                    print(
-                        f"❌ No files found to remove for app index '{index_display_name}' in {index_dir}"
-                    )
-                    return False
+            shutil.rmtree(index_dir)
+
+            if project_path:
+                print(f"✅ Index '{index_name}' removed from {project_path.name}")
            else:
-                import shutil
-
-                shutil.rmtree(index_dir)
-
-                if project_path:
-                    print(f"✅ Index '{index_display_name}' removed from {project_path.name}")
-                else:
-                    print(f"✅ Index '{index_display_name}' removed successfully")
-                return True
+                print(f"✅ Index '{index_name}' removed successfully")
+            return True
        except Exception as e:
-            print(f"❌ Error removing index '{index_display_name}': {e}")
+            print(f"❌ Error removing index '{index_name}': {e}")
            return False

    def load_documents(
@@ -857,7 +659,6 @@ Examples:
        docs_paths: Union[str, list],
        custom_file_types: Union[str, None] = None,
        include_hidden: bool = False,
-        args: Optional[dict[str, Any]] = None,
    ):
        # Handle both single path (string) and multiple paths (list) for backward compatibility
        if isinstance(docs_paths, str):
@@ -1163,50 +964,18 @@ Examples:
        }

        print("start chunking documents")
+        # Add progress bar for document chunking
+        for doc in tqdm(documents, desc="Chunking documents", unit="doc"):
+            # Check if this is a code file based on source path
+            source_path = doc.metadata.get("source", "")
+            is_code_file = any(source_path.endswith(ext) for ext in code_file_exts)

-        # Check if AST chunking is requested
-        use_ast = getattr(args, "use_ast_chunking", False)
+            # Use appropriate parser based on file type
+            parser = self.code_parser if is_code_file else self.node_parser
+            nodes = parser.get_nodes_from_documents([doc])

-        if use_ast:
-            print("🧠 Using AST-aware chunking for code files")
-            try:
-                # Import enhanced chunking utilities
-                # Add apps directory to path to import chunking utilities
-                apps_dir = Path(__file__).parent.parent.parent.parent.parent / "apps"
-                if apps_dir.exists():
-                    sys.path.insert(0, str(apps_dir))
-
-                from chunking import create_text_chunks
-
-                # Use enhanced chunking with AST support
-                all_texts = create_text_chunks(
-                    documents,
-                    chunk_size=self.node_parser.chunk_size,
-                    chunk_overlap=self.node_parser.chunk_overlap,
-                    use_ast_chunking=True,
-                    ast_chunk_size=getattr(args, "ast_chunk_size", 768),
-                    ast_chunk_overlap=getattr(args, "ast_chunk_overlap", 96),
-                    code_file_extensions=None,  # Use defaults
-                    ast_fallback_traditional=getattr(args, "ast_fallback_traditional", True),
-                )
-
-            except ImportError as e:
-                print(f"⚠️  AST chunking not available ({e}), falling back to traditional chunking")
-                use_ast = False
-
-        if not use_ast:
-            # Use traditional chunking logic
-            for doc in tqdm(documents, desc="Chunking documents", unit="doc"):
-                # Check if this is a code file based on source path
-                source_path = doc.metadata.get("source", "")
-                is_code_file = any(source_path.endswith(ext) for ext in code_file_exts)
-
-                # Use appropriate parser based on file type
-                parser = self.code_parser if is_code_file else self.node_parser
-                nodes = parser.get_nodes_from_documents([doc])
-
-                for node in nodes:
-                    all_texts.append(node.get_content())
+            for node in nodes:
+                all_texts.append(node.get_content())

        print(f"Loaded {len(documents)} documents, {len(all_texts)} chunks")
        return all_texts
@@ -1273,7 +1042,7 @@ Examples:
        )

        all_texts = self.load_documents(
-            docs_paths, args.file_types, include_hidden=args.include_hidden, args=args
+            docs_paths, args.file_types, include_hidden=args.include_hidden
        )
        if not all_texts:
            print("No documents found")
@@ -1306,101 +1075,13 @@ Examples:
    async def search_documents(self, args):
        index_name = args.index_name
        query = args.query
-
-        # First try to find the index in current project
        index_path = self.get_index_path(index_name)
-        if self.index_exists(index_name):
-            # Found in current project, use it
-            pass
-        else:
-            # Search across all registered projects (like list_indexes does)
-            all_matches = self._find_all_matching_indexes(index_name)
-            if not all_matches:
-                print(
-                    f"Index '{index_name}' not found. Use 'leann build {index_name} --docs <dir> [<dir2> ...]' to create it."
-                )
-                return
-            elif len(all_matches) == 1:
-                # Found exactly one match, use it
-                match = all_matches[0]
-                if match["kind"] == "cli":
-                    index_path = str(match["index_dir"] / "documents.leann")
-                else:
-                    # App format: use the meta file to construct the path
-                    meta_file = match["meta_file"]
-                    file_base = match["file_base"]
-                    index_path = str(meta_file.parent / f"{file_base}.leann")

-                project_info = (
-                    "current project"
-                    if match["is_current"]
-                    else f"project '{match['project_path'].name}'"
-                )
-                print(f"Using index '{index_name}' from {project_info}")
-            else:
-                # Multiple matches found
-                if args.non_interactive:
-                    # Non-interactive mode: automatically select the best match
-                    # Priority: current project first, then first available
-                    current_matches = [m for m in all_matches if m["is_current"]]
-                    if current_matches:
-                        match = current_matches[0]
-                        location_desc = "current project"
-                    else:
-                        match = all_matches[0]
-                        location_desc = f"project '{match['project_path'].name}'"
-
-                    if match["kind"] == "cli":
-                        index_path = str(match["index_dir"] / "documents.leann")
-                    else:
-                        meta_file = match["meta_file"]
-                        file_base = match["file_base"]
-                        index_path = str(meta_file.parent / f"{file_base}.leann")
-
-                    print(
-                        f"Found {len(all_matches)} indexes named '{index_name}', using index from {location_desc}"
-                    )
-                else:
-                    # Interactive mode: ask user to choose
-                    print(f"Found {len(all_matches)} indexes named '{index_name}':")
-                    for i, match in enumerate(all_matches, 1):
-                        project_path = match["project_path"]
-                        is_current = match["is_current"]
-                        kind = match.get("kind", "cli")
-
-                        if is_current:
-                            print(
-                                f"   {i}. 🏠 Current project ({'CLI' if kind == 'cli' else 'APP'})"
-                            )
-                        else:
-                            print(
-                                f"   {i}. 📂 {project_path.name} ({'CLI' if kind == 'cli' else 'APP'})"
-                            )
-
-                    try:
-                        choice = input(f"Which index to search? (1-{len(all_matches)}): ").strip()
-                        choice_idx = int(choice) - 1
-                        if 0 <= choice_idx < len(all_matches):
-                            match = all_matches[choice_idx]
-                            if match["kind"] == "cli":
-                                index_path = str(match["index_dir"] / "documents.leann")
-                            else:
-                                meta_file = match["meta_file"]
-                                file_base = match["file_base"]
-                                index_path = str(meta_file.parent / f"{file_base}.leann")
-
-                            project_info = (
-                                "current project"
-                                if match["is_current"]
-                                else f"project '{match['project_path'].name}'"
-                            )
-                            print(f"Using index '{index_name}' from {project_info}")
-                        else:
-                            print("Invalid choice. Aborting search.")
-                            return
-                    except (ValueError, KeyboardInterrupt):
-                        print("Invalid input. Aborting search.")
-                        return
+        if not self.index_exists(index_name):
+            print(
+                f"Index '{index_name}' not found. Use 'leann build {index_name} --docs <dir> [<dir2> ...]' to create it."
+            )
+            return

        searcher = LeannSearcher(index_path=index_path)
        results = searcher.search(
--- a/packages/leann-core/src/leann/embedding_compute.py
+++ b/packages/leann-core/src/leann/embedding_compute.py
@@ -6,7 +6,6 @@ Preserves all optimization parameters to ensure performance

 import logging
 import os
-import time
 from typing import Any

 import numpy as np
@@ -29,8 +28,6 @@ def compute_embeddings(
    is_build: bool = False,
    batch_size: int = 32,
    adaptive_optimization: bool = True,
-    manual_tokenize: bool = False,
-    max_length: int = 512,
 ) -> np.ndarray:
    """
    Unified embedding computation entry point
@@ -53,8 +50,6 @@ def compute_embeddings(
            is_build=is_build,
            batch_size=batch_size,
            adaptive_optimization=adaptive_optimization,
-            manual_tokenize=manual_tokenize,
-            max_length=max_length,
        )
    elif mode == "openai":
        return compute_embeddings_openai(texts, model_name)
@@ -76,8 +71,6 @@ def compute_embeddings_sentence_transformers(
    batch_size: int = 32,
    is_build: bool = False,
    adaptive_optimization: bool = True,
-    manual_tokenize: bool = False,
-    max_length: int = 512,
 ) -> np.ndarray:
    """
    Compute embeddings using SentenceTransformer with model caching and adaptive optimization
@@ -221,130 +214,20 @@ def compute_embeddings_sentence_transformers(
        logger.info(f"Model cached: {cache_key}")

    # Compute embeddings with optimized inference mode
-    logger.info(
-        f"Starting embedding computation... (batch_size: {batch_size}, manual_tokenize={manual_tokenize})"
-    )
+    logger.info(f"Starting embedding computation... (batch_size: {batch_size})")

-    start_time = time.time()
-    if not manual_tokenize:
-        # Use SentenceTransformer's optimized encode path (default)
-        with torch.inference_mode():
-            embeddings = model.encode(
-                texts,
-                batch_size=batch_size,
-                show_progress_bar=is_build,  # Don't show progress bar in server environment
-                convert_to_numpy=True,
-                normalize_embeddings=False,
-                device=device,
-            )
-        # Synchronize if CUDA to measure accurate wall time
-        try:
-            if torch.cuda.is_available():
-                torch.cuda.synchronize()
-        except Exception:
-            pass
-    else:
-        # Manual tokenization + forward pass using HF AutoTokenizer/AutoModel
-        try:
-            from transformers import AutoModel, AutoTokenizer  # type: ignore
-        except Exception as e:
-            raise ImportError(f"transformers is required for manual_tokenize=True: {e}")
+    # Use torch.inference_mode for optimal performance
+    with torch.inference_mode():
+        embeddings = model.encode(
+            texts,
+            batch_size=batch_size,
+            show_progress_bar=is_build,  # Don't show progress bar in server environment
+            convert_to_numpy=True,
+            normalize_embeddings=False,
+            device=device,
+        )

-        # Cache tokenizer and model
-        tok_cache_key = f"hf_tokenizer_{model_name}"
-        mdl_cache_key = f"hf_model_{model_name}_{device}_{use_fp16}"
-        if tok_cache_key in _model_cache and mdl_cache_key in _model_cache:
-            hf_tokenizer = _model_cache[tok_cache_key]
-            hf_model = _model_cache[mdl_cache_key]
-            logger.info("Using cached HF tokenizer/model for manual path")
-        else:
-            logger.info("Loading HF tokenizer/model for manual tokenization path")
-            hf_tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)
-            torch_dtype = torch.float16 if (use_fp16 and device == "cuda") else torch.float32
-            hf_model = AutoModel.from_pretrained(model_name, torch_dtype=torch_dtype)
-            hf_model.to(device)
-            hf_model.eval()
-            # Optional compile on supported devices
-            if device in ["cuda", "mps"]:
-                try:
-                    hf_model = torch.compile(hf_model, mode="reduce-overhead", dynamic=True)  # type: ignore
-                except Exception:
-                    pass
-            _model_cache[tok_cache_key] = hf_tokenizer
-            _model_cache[mdl_cache_key] = hf_model
-
-        all_embeddings: list[np.ndarray] = []
-        # Progress bar when building or for large inputs
-        show_progress = is_build or len(texts) > 32
-        try:
-            if show_progress:
-                from tqdm import tqdm  # type: ignore
-
-                batch_iter = tqdm(
-                    range(0, len(texts), batch_size),
-                    desc="Embedding (manual)",
-                    unit="batch",
-                )
-            else:
-                batch_iter = range(0, len(texts), batch_size)
-        except Exception:
-            batch_iter = range(0, len(texts), batch_size)
-
-        start_time_manual = time.time()
-        with torch.inference_mode():
-            for start_index in batch_iter:
-                end_index = min(start_index + batch_size, len(texts))
-                batch_texts = texts[start_index:end_index]
-                tokenize_start_time = time.time()
-                inputs = hf_tokenizer(
-                    batch_texts,
-                    padding=True,
-                    truncation=True,
-                    max_length=max_length,
-                    return_tensors="pt",
-                )
-                tokenize_end_time = time.time()
-                logger.info(
-                    f"Tokenize time taken: {tokenize_end_time - tokenize_start_time} seconds"
-                )
-                # Print shapes of all input tensors for debugging
-                for k, v in inputs.items():
-                    print(f"inputs[{k!r}] shape: {getattr(v, 'shape', type(v))}")
-                to_device_start_time = time.time()
-                inputs = {k: v.to(device) for k, v in inputs.items()}
-                to_device_end_time = time.time()
-                logger.info(
-                    f"To device time taken: {to_device_end_time - to_device_start_time} seconds"
-                )
-                forward_start_time = time.time()
-                outputs = hf_model(**inputs)
-                forward_end_time = time.time()
-                logger.info(f"Forward time taken: {forward_end_time - forward_start_time} seconds")
-                last_hidden_state = outputs.last_hidden_state  # (B, L, H)
-                attention_mask = inputs.get("attention_mask")
-                if attention_mask is None:
-                    # Fallback: assume all tokens are valid
-                    pooled = last_hidden_state.mean(dim=1)
-                else:
-                    mask = attention_mask.unsqueeze(-1).to(last_hidden_state.dtype)
-                    masked = last_hidden_state * mask
-                    lengths = mask.sum(dim=1).clamp(min=1)
-                    pooled = masked.sum(dim=1) / lengths
-                # Move to CPU float32
-                batch_embeddings = pooled.detach().to("cpu").float().numpy()
-                all_embeddings.append(batch_embeddings)
-
-        embeddings = np.vstack(all_embeddings).astype(np.float32, copy=False)
-        try:
-            if torch.cuda.is_available():
-                torch.cuda.synchronize()
-        except Exception:
-            pass
-        end_time = time.time()
-        logger.info(f"Manual tokenize time taken: {end_time - start_time_manual} seconds")
-    end_time = time.time()
    logger.info(f"Generated {len(embeddings)} embeddings, dimension: {embeddings.shape[1]}")
-    logger.info(f"Time taken: {end_time - start_time} seconds")

    # Validate results
    if np.isnan(embeddings).any() or np.isinf(embeddings).any():
--- a/packages/leann-core/src/leann/embedding_server_manager.py
+++ b/packages/leann-core/src/leann/embedding_server_manager.py
@@ -192,7 +192,6 @@ class EmbeddingServerManager:
            stderr_target = None  # Direct to console for visible logs

        # Start embedding server subprocess
-        logger.info(f"Starting server process with command: {' '.join(command)}")
        self.server_process = subprocess.Popen(
            command,
            cwd=project_root,
--- a/packages/leann-core/src/leann/mcp.py
+++ b/packages/leann-core/src/leann/mcp.py
@@ -94,7 +94,7 @@ def handle_request(request):
                        },
                    }

-                # Build simplified command with non-interactive flag for MCP compatibility
+                # Build simplified command
                cmd = [
                    "leann",
                    "search",
@@ -102,7 +102,6 @@ def handle_request(request):
                    args["query"],
                    f"--top-k={args.get('top_k', 5)}",
                    f"--complexity={args.get('complexity', 32)}",
-                    "--non-interactive",
                ]
                result = subprocess.run(cmd, capture_output=True, text=True)

--- a/packages/leann-core/src/leann/registry.py
+++ b/packages/leann-core/src/leann/registry.py
@@ -2,10 +2,8 @@

 import importlib
 import importlib.metadata
-import json
 import logging
-from pathlib import Path
-from typing import TYPE_CHECKING, Optional, Union
+from typing import TYPE_CHECKING

 if TYPE_CHECKING:
    from leann.interface import LeannBackendFactoryInterface
@@ -45,54 +43,3 @@ def autodiscover_backends():
            # print(f"WARN: Could not import backend module '{backend_module_name}': {e}")
            pass
    # print("INFO: Backend auto-discovery finished.")
-
-
-def register_project_directory(project_dir: Optional[Union[str, Path]] = None):
-    """
-    Register a project directory in the global LEANN registry.
-
-    This allows `leann list` to discover indexes created by apps or other tools.
-
-    Args:
-        project_dir: Directory to register. If None, uses current working directory.
-    """
-    if project_dir is None:
-        project_dir = Path.cwd()
-    else:
-        project_dir = Path(project_dir)
-
-    # Only register directories that have some kind of LEANN content
-    # Either .leann/indexes/ (CLI format) or *.leann.meta.json files (apps format)
-    has_cli_indexes = (project_dir / ".leann" / "indexes").exists()
-    has_app_indexes = any(project_dir.rglob("*.leann.meta.json"))
-
-    if not (has_cli_indexes or has_app_indexes):
-        # Don't register if there are no LEANN indexes
-        return
-
-    global_registry = Path.home() / ".leann" / "projects.json"
-    global_registry.parent.mkdir(exist_ok=True)
-
-    project_str = str(project_dir.resolve())
-
-    # Load existing registry
-    projects = []
-    if global_registry.exists():
-        try:
-            with open(global_registry) as f:
-                projects = json.load(f)
-        except Exception:
-            logger.debug("Could not load existing project registry")
-            projects = []
-
-    # Add project if not already present
-    if project_str not in projects:
-        projects.append(project_str)
-
-        # Save updated registry
-        try:
-            with open(global_registry, "w") as f:
-                json.dump(projects, f, indent=2)
-            logger.debug(f"Registered project directory: {project_str}")
-        except Exception as e:
-            logger.warning(f"Could not save project registry: {e}")
--- a/packages/leann/pyproject.toml
+++ b/packages/leann/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"

 [project]
 name = "leann"
-version = "0.3.2"
+version = "0.3.0"
 description = "LEANN - The smallest vector index in the world. RAG Everything with LEANN!"
 readme = "README.md"
 requires-python = ">=3.9"
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -14,6 +14,8 @@ dependencies = [
    "numpy>=1.26.0",
    "torch",
    "tqdm",
+    "flask",
+    "flask_compress",
    "datasets>=2.15.0",
    "evaluate",
    "colorama",
@@ -46,13 +48,6 @@ dependencies = [
    "pathspec>=0.12.1",
    "nbconvert>=7.16.6",
    "gitignore-parser>=0.1.12",
-    # AST-aware code chunking dependencies
-    "astchunk>=0.1.0",
-    "tree-sitter>=0.20.0",
-    "tree-sitter-python>=0.20.0",
-    "tree-sitter-java>=0.20.0",
-    "tree-sitter-c-sharp>=0.20.0",
-    "tree-sitter-typescript>=0.20.0",
 ]

 [project.optional-dependencies]
@@ -71,7 +66,9 @@ test = [
    "pytest>=7.0",
    "pytest-timeout>=2.0",
    "llama-index-core>=0.12.0",
+    "llama-index-readers-file>=0.4.0",
    "python-dotenv>=1.0.0",
+    "sentence-transformers>=2.2.0",
 ]

 diskann = [
@@ -103,8 +100,13 @@ leann-backend-hnsw = { path = "packages/leann-backend-hnsw", editable = true }
 [tool.ruff]
 target-version = "py39"
 line-length = 100
-extend-exclude = ["third_party"]
-
+extend-exclude = [
+    "third_party",
+    "*.egg-info",
+    "__pycache__",
+    ".git",
+    ".venv",
+]

 [tool.ruff.lint]
 select = [
@@ -127,12 +129,21 @@ ignore = [
    "RUF012", # mutable class attributes should be annotated with typing.ClassVar
 ]

+[tool.ruff.lint.per-file-ignores]
+"test/**/*.py" = ["E402"]      # module level import not at top of file (common in tests)
+"examples/**/*.py" = ["E402"]  # module level import not at top of file (common in examples)
+
 [tool.ruff.format]
 quote-style = "double"
 indent-style = "space"
 skip-magic-trailing-comma = false
 line-ending = "auto"

+[dependency-groups]
+dev = [
+    "ruff>=0.12.4",
+]
+
 [tool.lychee]
 accept = ["200", "403", "429", "503"]
 timeout = 20
--- a/tests/test_astchunk_integration.py
+++ b/tests/test_astchunk_integration.py
@@ -1,397 +0,0 @@
-"""
-Test suite for astchunk integration with LEANN.
-Tests AST-aware chunking functionality, language detection, and fallback mechanisms.
-"""
-
-import os
-import subprocess
-import sys
-import tempfile
-from pathlib import Path
-from unittest.mock import patch
-
-import pytest
-
-# Add apps directory to path for imports
-sys.path.insert(0, str(Path(__file__).parent.parent / "apps"))
-
-from typing import Optional
-
-from chunking import (
-    create_ast_chunks,
-    create_text_chunks,
-    create_traditional_chunks,
-    detect_code_files,
-    get_language_from_extension,
-)
-
-
-class MockDocument:
-    """Mock LlamaIndex Document for testing."""
-
-    def __init__(self, content: str, file_path: str = "", metadata: Optional[dict] = None):
-        self.content = content
-        self.metadata = metadata or {}
-        if file_path:
-            self.metadata["file_path"] = file_path
-
-    def get_content(self) -> str:
-        return self.content
-
-
-class TestCodeFileDetection:
-    """Test code file detection and language mapping."""
-
-    def test_detect_code_files_python(self):
-        """Test detection of Python files."""
-        docs = [
-            MockDocument("print('hello')", "/path/to/file.py"),
-            MockDocument("This is text", "/path/to/file.txt"),
-        ]
-
-        code_docs, text_docs = detect_code_files(docs)
-
-        assert len(code_docs) == 1
-        assert len(text_docs) == 1
-        assert code_docs[0].metadata["language"] == "python"
-        assert code_docs[0].metadata["is_code"] is True
-        assert text_docs[0].metadata["is_code"] is False
-
-    def test_detect_code_files_multiple_languages(self):
-        """Test detection of multiple programming languages."""
-        docs = [
-            MockDocument("def func():", "/path/to/script.py"),
-            MockDocument("public class Test {}", "/path/to/Test.java"),
-            MockDocument("interface ITest {}", "/path/to/test.ts"),
-            MockDocument("using System;", "/path/to/Program.cs"),
-            MockDocument("Regular text content", "/path/to/document.txt"),
-        ]
-
-        code_docs, text_docs = detect_code_files(docs)
-
-        assert len(code_docs) == 4
-        assert len(text_docs) == 1
-
-        languages = [doc.metadata["language"] for doc in code_docs]
-        assert "python" in languages
-        assert "java" in languages
-        assert "typescript" in languages
-        assert "csharp" in languages
-
-    def test_detect_code_files_no_file_path(self):
-        """Test handling of documents without file paths."""
-        docs = [
-            MockDocument("some content"),
-            MockDocument("other content", metadata={"some_key": "value"}),
-        ]
-
-        code_docs, text_docs = detect_code_files(docs)
-
-        assert len(code_docs) == 0
-        assert len(text_docs) == 2
-        for doc in text_docs:
-            assert doc.metadata["is_code"] is False
-
-    def test_get_language_from_extension(self):
-        """Test language detection from file extensions."""
-        assert get_language_from_extension("test.py") == "python"
-        assert get_language_from_extension("Test.java") == "java"
-        assert get_language_from_extension("component.tsx") == "typescript"
-        assert get_language_from_extension("Program.cs") == "csharp"
-        assert get_language_from_extension("document.txt") is None
-        assert get_language_from_extension("") is None
-
-
-class TestChunkingFunctions:
-    """Test various chunking functionality."""
-
-    def test_create_traditional_chunks(self):
-        """Test traditional text chunking."""
-        docs = [
-            MockDocument(
-                "This is a test document. It has multiple sentences. We want to test chunking."
-            )
-        ]
-
-        chunks = create_traditional_chunks(docs, chunk_size=50, chunk_overlap=10)
-
-        assert len(chunks) > 0
-        assert all(isinstance(chunk, str) for chunk in chunks)
-        assert all(len(chunk.strip()) > 0 for chunk in chunks)
-
-    def test_create_traditional_chunks_empty_docs(self):
-        """Test traditional chunking with empty documents."""
-        chunks = create_traditional_chunks([], chunk_size=50, chunk_overlap=10)
-        assert chunks == []
-
-    @pytest.mark.skipif(
-        os.environ.get("CI") == "true",
-        reason="Skip astchunk tests in CI - dependency may not be available",
-    )
-    def test_create_ast_chunks_with_astchunk_available(self):
-        """Test AST chunking when astchunk is available."""
-        python_code = '''
-def hello_world():
-    """Print hello world message."""
-    print("Hello, World!")
-
-def add_numbers(a, b):
-    """Add two numbers and return the result."""
-    return a + b
-
-class Calculator:
-    """A simple calculator class."""
-
-    def __init__(self):
-        self.history = []
-
-    def add(self, a, b):
-        result = a + b
-        self.history.append(f"{a} + {b} = {result}")
-        return result
-'''
-
-        docs = [MockDocument(python_code, "/test/calculator.py", {"language": "python"})]
-
-        try:
-            chunks = create_ast_chunks(docs, max_chunk_size=200, chunk_overlap=50)
-
-            # Should have multiple chunks due to different functions/classes
-            assert len(chunks) > 0
-            assert all(isinstance(chunk, str) for chunk in chunks)
-            assert all(len(chunk.strip()) > 0 for chunk in chunks)
-
-            # Check that code structure is somewhat preserved
-            combined_content = " ".join(chunks)
-            assert "def hello_world" in combined_content
-            assert "class Calculator" in combined_content
-
-        except ImportError:
-            # astchunk not available, should fall back to traditional chunking
-            chunks = create_ast_chunks(docs, max_chunk_size=200, chunk_overlap=50)
-            assert len(chunks) > 0  # Should still get chunks from fallback
-
-    def test_create_ast_chunks_fallback_to_traditional(self):
-        """Test AST chunking falls back to traditional when astchunk is not available."""
-        docs = [MockDocument("def test(): pass", "/test/script.py", {"language": "python"})]
-
-        # Mock astchunk import to fail
-        with patch("chunking.create_ast_chunks"):
-            # First call (actual test) should import astchunk and potentially fail
-            # Let's call the actual function to test the import error handling
-            chunks = create_ast_chunks(docs)
-
-            # Should return some chunks (either from astchunk or fallback)
-            assert isinstance(chunks, list)
-
-    def test_create_text_chunks_traditional_mode(self):
-        """Test text chunking in traditional mode."""
-        docs = [
-            MockDocument("def test(): pass", "/test/script.py"),
-            MockDocument("This is regular text.", "/test/doc.txt"),
-        ]
-
-        chunks = create_text_chunks(docs, use_ast_chunking=False, chunk_size=50, chunk_overlap=10)
-
-        assert len(chunks) > 0
-        assert all(isinstance(chunk, str) for chunk in chunks)
-
-    def test_create_text_chunks_ast_mode(self):
-        """Test text chunking in AST mode."""
-        docs = [
-            MockDocument("def test(): pass", "/test/script.py"),
-            MockDocument("This is regular text.", "/test/doc.txt"),
-        ]
-
-        chunks = create_text_chunks(
-            docs,
-            use_ast_chunking=True,
-            ast_chunk_size=100,
-            ast_chunk_overlap=20,
-            chunk_size=50,
-            chunk_overlap=10,
-        )
-
-        assert len(chunks) > 0
-        assert all(isinstance(chunk, str) for chunk in chunks)
-
-    def test_create_text_chunks_custom_extensions(self):
-        """Test text chunking with custom code file extensions."""
-        docs = [
-            MockDocument("function test() {}", "/test/script.js"),  # Not in default extensions
-            MockDocument("Regular text", "/test/doc.txt"),
-        ]
-
-        # First without custom extensions - should treat .js as text
-        chunks_without = create_text_chunks(docs, use_ast_chunking=True, code_file_extensions=None)
-
-        # Then with custom extensions - should treat .js as code
-        chunks_with = create_text_chunks(
-            docs, use_ast_chunking=True, code_file_extensions=[".js", ".jsx"]
-        )
-
-        # Both should return chunks
-        assert len(chunks_without) > 0
-        assert len(chunks_with) > 0
-
-
-class TestIntegrationWithDocumentRAG:
-    """Integration tests with the document RAG system."""
-
-    @pytest.fixture
-    def temp_code_dir(self):
-        """Create a temporary directory with sample code files."""
-        with tempfile.TemporaryDirectory() as temp_dir:
-            temp_path = Path(temp_dir)
-
-            # Create sample Python file
-            python_file = temp_path / "example.py"
-            python_file.write_text('''
-def fibonacci(n):
-    """Calculate fibonacci number."""
-    if n <= 1:
-        return n
-    return fibonacci(n-1) + fibonacci(n-2)
-
-class MathUtils:
-    @staticmethod
-    def factorial(n):
-        if n <= 1:
-            return 1
-        return n * MathUtils.factorial(n-1)
-''')
-
-            # Create sample text file
-            text_file = temp_path / "readme.txt"
-            text_file.write_text("This is a sample text file for testing purposes.")
-
-            yield temp_path
-
-    @pytest.mark.skipif(
-        os.environ.get("CI") == "true",
-        reason="Skip integration tests in CI to avoid dependency issues",
-    )
-    def test_document_rag_with_ast_chunking(self, temp_code_dir):
-        """Test document RAG with AST chunking enabled."""
-        with tempfile.TemporaryDirectory() as index_dir:
-            cmd = [
-                sys.executable,
-                "apps/document_rag.py",
-                "--llm",
-                "simulated",
-                "--embedding-model",
-                "facebook/contriever",
-                "--embedding-mode",
-                "sentence-transformers",
-                "--index-dir",
-                index_dir,
-                "--data-dir",
-                str(temp_code_dir),
-                "--enable-code-chunking",
-                "--query",
-                "How does the fibonacci function work?",
-            ]
-
-            env = os.environ.copy()
-            env["HF_HUB_DISABLE_SYMLINKS"] = "1"
-            env["TOKENIZERS_PARALLELISM"] = "false"
-
-            try:
-                result = subprocess.run(
-                    cmd,
-                    capture_output=True,
-                    text=True,
-                    timeout=300,  # 5 minutes
-                    env=env,
-                )
-
-                # Should succeed even if astchunk is not available (fallback)
-                assert result.returncode == 0, f"Command failed: {result.stderr}"
-
-                output = result.stdout + result.stderr
-                assert "Index saved to" in output or "Using existing index" in output
-
-            except subprocess.TimeoutExpired:
-                pytest.skip("Test timed out - likely due to model download in CI")
-
-    @pytest.mark.skipif(
-        os.environ.get("CI") == "true",
-        reason="Skip integration tests in CI to avoid dependency issues",
-    )
-    def test_code_rag_application(self, temp_code_dir):
-        """Test the specialized code RAG application."""
-        with tempfile.TemporaryDirectory() as index_dir:
-            cmd = [
-                sys.executable,
-                "apps/code_rag.py",
-                "--llm",
-                "simulated",
-                "--embedding-model",
-                "facebook/contriever",
-                "--index-dir",
-                index_dir,
-                "--repo-dir",
-                str(temp_code_dir),
-                "--query",
-                "What classes are defined in this code?",
-            ]
-
-            env = os.environ.copy()
-            env["HF_HUB_DISABLE_SYMLINKS"] = "1"
-            env["TOKENIZERS_PARALLELISM"] = "false"
-
-            try:
-                result = subprocess.run(cmd, capture_output=True, text=True, timeout=300, env=env)
-
-                # Should succeed
-                assert result.returncode == 0, f"Command failed: {result.stderr}"
-
-                output = result.stdout + result.stderr
-                assert "Using AST-aware chunking" in output or "traditional chunking" in output
-
-            except subprocess.TimeoutExpired:
-                pytest.skip("Test timed out - likely due to model download in CI")
-
-
-class TestErrorHandling:
-    """Test error handling and edge cases."""
-
-    def test_text_chunking_empty_documents(self):
-        """Test text chunking with empty document list."""
-        chunks = create_text_chunks([])
-        assert chunks == []
-
-    def test_text_chunking_invalid_parameters(self):
-        """Test text chunking with invalid parameters."""
-        docs = [MockDocument("test content")]
-
-        # Should handle negative chunk sizes gracefully
-        chunks = create_text_chunks(
-            docs, chunk_size=0, chunk_overlap=0, ast_chunk_size=0, ast_chunk_overlap=0
-        )
-
-        # Should still return some result
-        assert isinstance(chunks, list)
-
-    def test_create_ast_chunks_no_language(self):
-        """Test AST chunking with documents missing language metadata."""
-        docs = [MockDocument("def test(): pass", "/test/script.py")]  # No language set
-
-        chunks = create_ast_chunks(docs)
-
-        # Should fall back to traditional chunking
-        assert isinstance(chunks, list)
-        assert len(chunks) >= 0  # May be empty if fallback also fails
-
-    def test_create_ast_chunks_empty_content(self):
-        """Test AST chunking with empty content."""
-        docs = [MockDocument("", "/test/script.py", {"language": "python"})]
-
-        chunks = create_ast_chunks(docs)
-
-        # Should handle empty content gracefully
-        assert isinstance(chunks, list)
-
-
-if __name__ == "__main__":
-    pytest.main([__file__, "-v"])
--- a/tests/test_document_rag.py
+++ b/tests/test_document_rag.py
@@ -57,51 +57,6 @@ def test_document_rag_simulated(test_data_dir):
        assert "This is a simulated answer" in output


-@pytest.mark.skipif(
-    os.environ.get("CI") == "true",
-    reason="Skip AST chunking tests in CI to avoid dependency issues",
-)
-def test_document_rag_with_ast_chunking(test_data_dir):
-    """Test document_rag with AST-aware chunking enabled."""
-    with tempfile.TemporaryDirectory() as temp_dir:
-        # Use a subdirectory that doesn't exist yet to force index creation
-        index_dir = Path(temp_dir) / "test_ast_index"
-        cmd = [
-            sys.executable,
-            "apps/document_rag.py",
-            "--llm",
-            "simulated",
-            "--embedding-model",
-            "facebook/contriever",
-            "--embedding-mode",
-            "sentence-transformers",
-            "--index-dir",
-            str(index_dir),
-            "--data-dir",
-            str(test_data_dir),
-            "--enable-code-chunking",  # Enable AST chunking
-            "--query",
-            "What is Pride and Prejudice about?",
-        ]
-
-        env = os.environ.copy()
-        env["HF_HUB_DISABLE_SYMLINKS"] = "1"
-        env["TOKENIZERS_PARALLELISM"] = "false"
-
-        result = subprocess.run(cmd, capture_output=True, text=True, timeout=600, env=env)
-
-        # Check return code
-        assert result.returncode == 0, f"Command failed: {result.stderr}"
-
-        # Verify output
-        output = result.stdout + result.stderr
-        assert "Index saved to" in output or "Using existing index" in output
-        assert "This is a simulated answer" in output
-
-        # Should mention AST chunking if code files are present
-        # (might not be relevant for the test data, but command should succeed)
-
-
@pytest.mark.skipif(not os.environ.get("OPENAI_API_KEY"), reason="OpenAI API key not available")
@pytest.mark.skipif(
    os.environ.get("CI") == "true", reason="Skip OpenAI tests in CI to avoid API costs"
--- a/uv.lock
+++ b/uv.lock
				`@@ -0,0 +1 @@`
				`paper_plot/data/big_graph_degree_data.npz filter=lfs diff=lfs merge=lfs -text`