Merge remote-tracking branch 'origin/main' into fix/openai-embeddings-cosine-distance

- Allow using existing index without rebuilding
- Useful for testing pre-built indices

.github/workflows/build-reusable.yml

@@ -54,26 +54,16 @@ jobs:
             python: '3.12'
           - os: ubuntu-22.04
             python: '3.13'
-          - os: macos-14
+          - os: macos-latest
             python: '3.9'
-          - os: macos-14
+          - os: macos-latest
             python: '3.10'
-          - os: macos-14
+          - os: macos-latest
             python: '3.11'
-          - os: macos-14
+          - os: macos-latest
             python: '3.12'
-          - os: macos-14
+          - os: macos-latest
             python: '3.13'
-          - os: macos-13
-            python: '3.9'
-          - os: macos-13
-            python: '3.10'
-          - os: macos-13
-            python: '3.11'
-          - os: macos-13
-            python: '3.12'
-          # Note: macos-13 + Python 3.13 excluded due to PyTorch compatibility
-          # (PyTorch 2.5+ supports Python 3.13 but not Intel Mac x86_64)
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -107,8 +97,7 @@ jobs:
       - name: Install system dependencies (macOS)
         if: runner.os == 'macOS'
         run: |
-          # Don't install LLVM, use system clang for better compatibility
-          brew install libomp boost protobuf zeromq
+          brew install llvm libomp boost protobuf zeromq
 
       - name: Install build dependencies
         run: |
@@ -119,59 +108,39 @@ jobs:
             uv pip install --system delocate
           fi
 
-      - name: Set macOS environment variables
-        if: runner.os == 'macOS'
-        run: |
-          # Use brew --prefix to automatically detect Homebrew installation path
-          HOMEBREW_PREFIX=$(brew --prefix)
-          echo "HOMEBREW_PREFIX=${HOMEBREW_PREFIX}" >> $GITHUB_ENV
-          echo "OpenMP_ROOT=${HOMEBREW_PREFIX}/opt/libomp" >> $GITHUB_ENV
-
-          # Set CMAKE_PREFIX_PATH to let CMake find all packages automatically
-          echo "CMAKE_PREFIX_PATH=${HOMEBREW_PREFIX}" >> $GITHUB_ENV
-
-          # Set compiler flags for OpenMP (required for both backends)
-          echo "LDFLAGS=-L${HOMEBREW_PREFIX}/opt/libomp/lib" >> $GITHUB_ENV
-          echo "CPPFLAGS=-I${HOMEBREW_PREFIX}/opt/libomp/include" >> $GITHUB_ENV
-
       - name: Build packages
         run: |
           # Build core (platform independent)
-          cd packages/leann-core
-          uv build
-          cd ../..
+          if [[ "${{ matrix.os }}" == ubuntu-* ]]; then
+            cd packages/leann-core
+            uv build
+            cd ../..
+          fi
 
           # Build HNSW backend
           cd packages/leann-backend-hnsw
-          if [[ "${{ matrix.os }}" == macos-* ]]; then
-            # Use system clang for better compatibility
-            export CC=clang
-            export CXX=clang++
-            export MACOSX_DEPLOYMENT_TARGET=11.0
-            uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
+          if [ "${{ matrix.os }}" == "macos-latest" ]; then
+            CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build --wheel --python python
           else
-            uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
+            uv build --wheel --python python
           fi
           cd ../..
 
           # Build DiskANN backend
           cd packages/leann-backend-diskann
-          if [[ "${{ matrix.os }}" == macos-* ]]; then
-            # Use system clang for better compatibility
-            export CC=clang
-            export CXX=clang++
-            # DiskANN requires macOS 13.3+ for sgesdd_ LAPACK function
-            export MACOSX_DEPLOYMENT_TARGET=13.3
-            uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
+          if [ "${{ matrix.os }}" == "macos-latest" ]; then
+            CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build --wheel --python python
           else
-            uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
+            uv build --wheel --python python
           fi
           cd ../..
 
           # Build meta package (platform independent)
-          cd packages/leann
-          uv build
-          cd ../..
+          if [[ "${{ matrix.os }}" == ubuntu-* ]]; then
+            cd packages/leann
+            uv build
+            cd ../..
+          fi
 
       - name: Repair wheels (Linux)
         if: runner.os == 'Linux'
@@ -220,49 +189,6 @@ jobs:
           echo "📦 Built packages:"
           find packages/*/dist -name "*.whl" -o -name "*.tar.gz" | sort
 
-
-      - name: Install built packages for testing
-        run: |
-          # Create a virtual environment with the correct Python version
-          uv venv --python ${{ matrix.python }}
-          source .venv/bin/activate || source .venv/Scripts/activate
-
-          # Install packages using --find-links to prioritize local builds
-          uv pip install --find-links packages/leann-core/dist --find-links packages/leann-backend-hnsw/dist --find-links packages/leann-backend-diskann/dist packages/leann-core/dist/*.whl || uv pip install --find-links packages/leann-core/dist packages/leann-core/dist/*.tar.gz
-          uv pip install --find-links packages/leann-core/dist packages/leann-backend-hnsw/dist/*.whl
-          uv pip install --find-links packages/leann-core/dist packages/leann-backend-diskann/dist/*.whl
-          uv pip install packages/leann/dist/*.whl || uv pip install packages/leann/dist/*.tar.gz
-
-          # Install test dependencies using extras
-          uv pip install -e ".[test]"
-
-      - name: Run tests with pytest
-        env:
-          CI: true  # Mark as CI environment to skip memory-intensive tests
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-          HF_HUB_DISABLE_SYMLINKS: 1
-          TOKENIZERS_PARALLELISM: false
-          PYTORCH_ENABLE_MPS_FALLBACK: 0  # Disable MPS on macOS CI to avoid memory issues
-          OMP_NUM_THREADS: 1  # Disable OpenMP parallelism to avoid libomp crashes
-          MKL_NUM_THREADS: 1  # Single thread for MKL operations
-        run: |
-          # Activate virtual environment
-          source .venv/bin/activate || source .venv/Scripts/activate
-
-          # Run all tests
-          pytest tests/
-
-      - name: Run sanity checks (optional)
-        run: |
-          # Activate virtual environment
-          source .venv/bin/activate || source .venv/Scripts/activate
-
-          # Run distance function tests if available
-          if [ -f test/sanity_checks/test_distance_functions.py ]; then
-            echo "Running distance function sanity checks..."
-            python test/sanity_checks/test_distance_functions.py || echo "⚠️ Distance function test failed, continuing..."
-          fi
-
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         with:

.github/workflows/link-check.yml

@@ -1,19 +0,0 @@
-name: Link Check
-
-on:
-  push:
-    branches: [ main, master ]
-  pull_request:
-  schedule:
-    - cron: "0 3 * * 1"
-
-jobs:
-  link-check:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: lycheeverse/lychee-action@v2
-        with:
-          args: --no-progress --insecure README.md docs/ apps/ examples/ benchmarks/
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -34,15 +34,11 @@ build/
 nprobe_logs/
 micro/results
 micro/contriever-INT8
-data/*
-!data/2501.14312v1 (1).pdf
-!data/2506.08276v1.pdf
-!data/PrideandPrejudice.txt
-!data/huawei_pangu.md
-!data/ground_truth/
-!data/indices/
-!data/queries/
-!data/.gitattributes
+examples/data/*
+!examples/data/2501.14312v1 (1).pdf
+!examples/data/2506.08276v1.pdf
+!examples/data/PrideandPrejudice.txt
+!examples/data/README.md
 *.qdstrm
 benchmark_results/
 results/
@@ -90,5 +86,3 @@ packages/leann-backend-diskann/third_party/DiskANN/_deps/
 *.passages.json
 
 batchtest.py
-tests/__pytest_cache__/
-tests/__pycache__/

README.md

@@ -3,11 +3,9 @@
 </p>
 
 <p align="center">
-  <img src="https://img.shields.io/badge/Python-3.9%20%7C%203.10%20%7C%203.11%20%7C%203.12%20%7C%203.13-blue.svg" alt="Python Versions">
-  <img src="https://github.com/yichuan-w/LEANN/actions/workflows/build-and-publish.yml/badge.svg" alt="CI Status">
-  <img src="https://img.shields.io/badge/Platform-Ubuntu%20%7C%20macOS%20(ARM64%2FIntel)-lightgrey" alt="Platform">
+  <img src="https://img.shields.io/badge/Python-3.9%2B-blue.svg" alt="Python 3.9+">
   <img src="https://img.shields.io/badge/License-MIT-green.svg" alt="MIT License">
-  <img src="https://img.shields.io/badge/MCP-Native%20Integration-blue" alt="MCP Integration">
+  <img src="https://img.shields.io/badge/Platform-Linux%20%7C%20macOS-lightgrey" alt="Platform">
 </p>
 
 <h2 align="center" tabindex="-1" class="heading-element" dir="auto">
@@ -18,10 +16,7 @@ LEANN is an innovative vector database that democratizes personal AI. Transform
 
 LEANN achieves this through *graph-based selective recomputation* with *high-degree preserving pruning*, computing embeddings on-demand instead of storing them all. [Illustration Fig →](#️-architecture--how-it-works) | [Paper →](https://arxiv.org/abs/2506.08276)
 
-**Ready to RAG Everything?** Transform your laptop into a personal AI assistant that can semantic search your **[file system](#-personal-data-manager-process-any-documents-pdf-txt-md)**, **[emails](#-your-personal-email-secretary-rag-on-apple-mail)**, **[browser history](#-time-machine-for-the-web-rag-your-entire-browser-history)**, **[chat history](#-wechat-detective-unlock-your-golden-memories)**, **[codebase](#-claude-code-integration-transform-your-development-workflow)**\* , or external knowledge bases (i.e., 60M documents) - all on your laptop, with zero cloud costs and complete privacy.
-
-
-\* Claude Code only supports basic `grep`-style keyword search. **LEANN** is a drop-in **semantic search MCP service fully compatible with Claude Code**, unlocking intelligent retrieval without changing your workflow. 🔥 Check out [the easy setup →](packages/leann-mcp/README.md)
+**Ready to RAG Everything?** Transform your laptop into a personal AI assistant that can search your **[file system](#-personal-data-manager-process-any-documents-pdf-txt-md)**, **[emails](#-your-personal-email-secretary-rag-on-apple-mail)**, **[browser history](#-time-machine-for-the-web-rag-your-entire-browser-history)**, **[chat history](#-wechat-detective-unlock-your-golden-memories)**, or external knowledge bases (i.e., 60M documents) - all on your laptop, with zero cloud costs and complete privacy.
 
 
 
@@ -31,7 +26,7 @@ LEANN achieves this through *graph-based selective recomputation* with *high-deg
   <img src="assets/effects.png" alt="LEANN vs Traditional Vector DB Storage Comparison" width="70%">
 </p>
 
-> **The numbers speak for themselves:** Index 60 million text chunks in just 6GB instead of 201GB. From emails to browser history, everything fits on your laptop. [See detailed benchmarks for different applications below ↓](#storage-comparison)
+> **The numbers speak for themselves:** Index 60 million Wikipedia chunks in just 6GB instead of 201GB. From emails to browser history, everything fits on your laptop. [See detailed benchmarks for different applications below ↓](#storage-comparison)
 
 
 🔒 **Privacy:** Your data never leaves your laptop. No OpenAI, no cloud, no "terms of service".
@@ -46,40 +41,40 @@ LEANN achieves this through *graph-based selective recomputation* with *high-deg
 
 ## Installation
 
-### 📦 Prerequisites: Install uv
+<details>
+<summary><strong>📦 Prerequisites: Install uv (if you don't have it)</strong></summary>
 
-[Install uv](https://docs.astral.sh/uv/getting-started/installation/#installation-methods) first if you don't have it. Typically, you can install it with:
+Install uv first if you don't have it:
 
 ```bash
 curl -LsSf https://astral.sh/uv/install.sh | sh
 ```
 
-### 🚀 Quick Install
+📖 [Detailed uv installation methods →](https://docs.astral.sh/uv/getting-started/installation/#installation-methods)
 
-Clone the repository to access all examples and try amazing applications,
+</details>
+
+
+LEANN provides two installation methods: **pip install** (quick and easy) and **build from source** (recommended for development).
+
+
+
+### 🚀 Quick Install (Recommended for most users)
+
+Clone the repository to access all examples and install LEANN from [PyPI](https://pypi.org/project/leann/) to run them immediately:
 
 ```bash
-git clone https://github.com/yichuan-w/LEANN.git leann
+git clone git@github.com:yichuan-w/LEANN.git leann
 cd leann
-```
-
-and install LEANN from [PyPI](https://pypi.org/project/leann/) to run them immediately:
-
-```bash
 uv venv
 source .venv/bin/activate
 uv pip install leann
 ```
 
-<details>
-<summary>
-<strong>🔧 Build from Source (Recommended for development)</strong>
-</summary>
-
-
+### 🔧 Build from Source (Recommended for development)
 
 ```bash
-git clone https://github.com/yichuan-w/LEANN.git leann
+git clone git@github.com:yichuan-w/LEANN.git leann
 cd leann
 git submodule update --init --recursive
 ```
@@ -96,14 +91,14 @@ sudo apt-get install libomp-dev libboost-all-dev protobuf-compiler libabsl-dev l
 uv sync
 ```
 
-</details>
+
 
 
 ## Quick Start
 
 Our declarative API makes RAG as easy as writing a config file.
 
-Check out [demo.ipynb](demo.ipynb) or [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/yichuan-w/LEANN/blob/main/demo.ipynb)
+[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/yichuan-w/LEANN/blob/main/demo.ipynb) [Try in this ipynb file →](demo.ipynb)
 
 ```python
 from leann import LeannBuilder, LeannSearcher, LeannChat
@@ -127,11 +122,11 @@ response = chat.ask("How much storage does LEANN save?", top_k=1)
 
 ## RAG on Everything!
 
-LEANN supports RAG on various data sources including documents (`.pdf`, `.txt`, `.md`), Apple Mail, Google Search History, WeChat, and more.
+LEANN supports RAG on various data sources including documents (.pdf, .txt, .md), Apple Mail, Google Search History, WeChat, and more.
 
-### Generation Model Setup
 
-LEANN supports multiple LLM providers for text generation (OpenAI API, HuggingFace, Ollama).
+> **Generation Model Setup**
+> LEANN supports multiple LLM providers for text generation (OpenAI API, HuggingFace, Ollama).
 
 <details>
 <summary><strong>🔑 OpenAI API Setup (Default)</strong></summary>
@@ -171,52 +166,7 @@ ollama pull llama3.2:1b
 
 </details>
 
-### ⭐ Flexible Configuration
-
-LEANN provides flexible parameters for embedding models, search strategies, and data processing to fit your specific needs.
-
-📚 **Need configuration best practices?** Check our [Configuration Guide](docs/configuration-guide.md) for detailed optimization tips, model selection advice, and solutions to common issues like slow embeddings or poor search quality.
-
-<details>
-<summary><strong>📋 Click to expand: Common Parameters (Available in All Examples)</strong></summary>
-
-All RAG examples share these common parameters. **Interactive mode** is available in all examples - simply run without `--query` to start a continuous Q&A session where you can ask multiple questions. Type 'quit' to exit.
-
-```bash
-# Core Parameters (General preprocessing for all examples)
---index-dir DIR          # Directory to store the index (default: current directory)
---query "YOUR QUESTION"  # Single query mode. Omit for interactive chat (type 'quit' to exit), and now you can play with your index interactively
---max-items N           # Limit data preprocessing (default: -1, process all data)
---force-rebuild         # Force rebuild index even if it exists
-
-# Embedding Parameters
---embedding-model MODEL  # e.g., facebook/contriever, text-embedding-3-small, nomic-embed-text,mlx-community/Qwen3-Embedding-0.6B-8bit or nomic-embed-text
---embedding-mode MODE    # sentence-transformers, openai, mlx, or ollama
-
-# LLM Parameters (Text generation models)
---llm TYPE              # LLM backend: openai, ollama, or hf (default: openai)
---llm-model MODEL       # Model name (default: gpt-4o) e.g., gpt-4o-mini, llama3.2:1b, Qwen/Qwen2.5-1.5B-Instruct
---thinking-budget LEVEL # Thinking budget for reasoning models: low/medium/high (supported by o3, o3-mini, GPT-Oss:20b, and other reasoning models)
-
-# Search Parameters
---top-k N               # Number of results to retrieve (default: 20)
---search-complexity N   # Search complexity for graph traversal (default: 32)
-
-# Chunking Parameters
---chunk-size N          # Size of text chunks (default varies by source: 256 for most, 192 for WeChat)
---chunk-overlap N       # Overlap between chunks (default varies: 25-128 depending on source)
-
-# Index Building Parameters
---backend-name NAME     # Backend to use: hnsw or diskann (default: hnsw)
---graph-degree N        # Graph degree for index construction (default: 32)
---build-complexity N    # Build complexity for index construction (default: 64)
---no-compact           # Disable compact index storage (compact storage IS enabled to save storage by default)
---no-recompute         # Disable embedding recomputation (recomputation IS enabled to save storage by default)
-```
-
-</details>
-
-### 📄 Personal Data Manager: Process Any Documents (`.pdf`, `.txt`, `.md`)!
+### 📄 Personal Data Manager: Process Any Documents (.pdf, .txt, .md)!
 
 Ask questions directly about your personal PDFs, documents, and any directory containing your files!
 
@@ -224,32 +174,15 @@ Ask questions directly about your personal PDFs, documents, and any directory co
   <img src="videos/paper_clear.gif" alt="LEANN Document Search Demo" width="600">
 </p>
 
-The example below asks a question about summarizing our paper (uses default data in `data/`, which is a directory with diverse data sources: two papers, Pride and Prejudice, and a Technical report about LLM in Huawei in Chinese), and this is the **easiest example** to run here:
+The example below asks a question about summarizing two papers (uses default data in `examples/data`):
 
-```bash
-source .venv/bin/activate # Don't forget to activate the virtual environment
-python -m apps.document_rag --query "What are the main techniques LEANN explores?"
+```
+# Or use python directly
+source .venv/bin/activate
+python ./examples/main_cli_example.py
 ```
 
-<details>
-<summary><strong>📋 Click to expand: Document-Specific Arguments</strong></summary>
 
-#### Parameters
-```bash
---data-dir DIR           # Directory containing documents to process (default: data)
---file-types .ext .ext   # Filter by specific file types (optional - all LlamaIndex supported types if omitted)
-```
-
-#### Example Commands
-```bash
-# Process all documents with larger chunks for academic papers
-python -m apps.document_rag --data-dir "~/Documents/Papers" --chunk-size 1024
-
-# Filter only markdown and Python files with smaller chunks
-python -m apps.document_rag --data-dir "./docs" --chunk-size 256 --file-types .md .py
-```
-
-</details>
 
 ### 📧 Your Personal Email Secretary: RAG on Apple Mail!
 
@@ -260,29 +193,30 @@ python -m apps.document_rag --data-dir "./docs" --chunk-size 256 --file-types .m
   <img src="videos/mail_clear.gif" alt="LEANN Email Search Demo" width="600">
 </p>
 
-Before running the example below, you need to grant full disk access to your terminal/VS Code in System Preferences → Privacy & Security → Full Disk Access.
-
+**Note:** You need to grant full disk access to your terminal/VS Code in System Preferences → Privacy & Security → Full Disk Access.
 ```bash
-python -m apps.email_rag --query "What's the food I ordered by DoorDash or Uber Eats mostly?"
+python examples/mail_reader_leann.py --query "What's the food I ordered by doordash or Uber eat mostly?"
 ```
-**780K email chunks → 78MB storage.** Finally, search your email like you search Google.
+**780K email chunks → 78MB storage** Finally, search your email like you search Google.
 
 <details>
-<summary><strong>📋 Click to expand: Email-Specific Arguments</strong></summary>
+<summary><strong>📋 Click to expand: Command Examples</strong></summary>
 
-#### Parameters
 ```bash
---mail-path PATH         # Path to specific mail directory (auto-detects if omitted)
---include-html          # Include HTML content in processing (useful for newsletters)
-```
+# Use default mail path (works for most macOS setups)
+python examples/mail_reader_leann.py
 
-#### Example Commands
-```bash
-# Search work emails from a specific account
-python -m apps.email_rag --mail-path "~/Library/Mail/V10/WORK_ACCOUNT"
+# Run with custom index directory
+python examples/mail_reader_leann.py --index-dir "./my_mail_index"
 
-# Find all receipts and order confirmations (includes HTML)
-python -m apps.email_rag --query "receipt order confirmation invoice" --include-html
+# Process all emails (may take time but indexes everything)
+python examples/mail_reader_leann.py --max-emails -1
+
+# Limit number of emails processed (useful for testing)
+python examples/mail_reader_leann.py --max-emails 1000
+
+# Run a single query
+python examples/mail_reader_leann.py --query "What did my boss say about deadlines?"
 ```
 
 </details>
@@ -303,25 +237,25 @@ Once the index is built, you can ask questions like:
 </p>
 
 ```bash
-python -m apps.browser_rag --query "Tell me my browser history about machine learning?"
+python examples/google_history_reader_leann.py --query "Tell me my browser history about machine learning?"
 ```
 **38K browser entries → 6MB storage.** Your browser history becomes your personal search engine.
 
 <details>
-<summary><strong>📋 Click to expand: Browser-Specific Arguments</strong></summary>
+<summary><strong>📋 Click to expand: Command Examples</strong></summary>
 
-#### Parameters
 ```bash
---chrome-profile PATH    # Path to Chrome profile directory (auto-detects if omitted)
-```
+# Use default Chrome profile (auto-finds all profiles)
+python examples/google_history_reader_leann.py
 
-#### Example Commands
-```bash
-# Search academic research from your browsing history
-python -m apps.browser_rag --query "arxiv papers machine learning transformer architecture"
+# Run with custom index directory
+python examples/google_history_reader_leann.py --index-dir "./my_chrome_index"
 
-# Track competitor analysis across work profile
-python -m apps.browser_rag --chrome-profile "~/Library/Application Support/Google/Chrome/Work Profile" --max-items 5000
+# Limit number of history entries processed (useful for testing)
+python examples/google_history_reader_leann.py --max-entries 500
+
+# Run a single query
+python examples/google_history_reader_leann.py --query "What websites did I visit about machine learning?"
 ```
 
 </details>
@@ -361,7 +295,7 @@ Once the index is built, you can ask questions like:
 </p>
 
 ```bash
-python -m apps.wechat_rag --query "Show me all group chats about weekend plans"
+python examples/wechat_history_reader_leann.py --query "Show me all group chats about weekend plans"
 ```
 **400K messages → 64MB storage** Search years of chat history in any language.
 
@@ -369,13 +303,7 @@ python -m apps.wechat_rag --query "Show me all group chats about weekend plans"
 <details>
 <summary><strong>🔧 Click to expand: Installation Requirements</strong></summary>
 
-First, you need to install the [WeChat exporter](https://github.com/sunnyyoung/WeChatTweak-CLI),
-
-```bash
-brew install sunnyyoung/repo/wechattweak-cli
-```
-
-or install it manually (if you have issues with Homebrew):
+First, you need to install the WeChat exporter:
 
 ```bash
 sudo packages/wechat-exporter/wechattweak-cli install
@@ -384,28 +312,30 @@ sudo packages/wechat-exporter/wechattweak-cli install
 **Troubleshooting:**
 - **Installation issues**: Check the [WeChatTweak-CLI issues page](https://github.com/sunnyyoung/WeChatTweak-CLI/issues/41)
 - **Export errors**: If you encounter the error below, try restarting WeChat
-  ```bash
-  Failed to export WeChat data. Please ensure WeChat is running and WeChatTweak is installed.
-  Failed to find or export WeChat data. Exiting.
-  ```
+```
+Failed to export WeChat data. Please ensure WeChat is running and WeChatTweak is installed.
+Failed to find or export WeChat data. Exiting.
+```
 </details>
 
 <details>
-<summary><strong>📋 Click to expand: WeChat-Specific Arguments</strong></summary>
+<summary><strong>📋 Click to expand: Command Examples</strong></summary>
 
-#### Parameters
 ```bash
---export-dir DIR         # Directory to store exported WeChat data (default: wechat_export_direct)
---force-export          # Force re-export even if data exists
-```
+# Use default settings (recommended for first run)
+python examples/wechat_history_reader_leann.py
 
-#### Example Commands
-```bash
-# Search for travel plans discussed in group chats
-python -m apps.wechat_rag --query "travel plans" --max-items 10000
+# Run with custom export directory and wehn we run the first time, LEANN will export all chat history automatically for you
+python examples/wechat_history_reader_leann.py --export-dir "./my_wechat_exports"
 
-# Re-export and search recent chats (useful after new messages)
-python -m apps.wechat_rag --force-export --query "work schedule"
+# Run with custom index directory
+python examples/wechat_history_reader_leann.py --index-dir "./my_wechat_index"
+
+# Limit number of chat entries processed (useful for testing)
+python examples/wechat_history_reader_leann.py --max-entries 1000
+
+# Run a single query
+python examples/wechat_history_reader_leann.py --query "Show me conversations about travel plans"
 ```
 
 </details>
@@ -419,57 +349,15 @@ Once the index is built, you can ask questions like:
 
 </details>
 
-### 🚀 Claude Code Integration: Transform Your Development Workflow!
 
-**The future of code assistance is here.** Transform your development workflow with LEANN's native MCP integration for Claude Code. Index your entire codebase and get intelligent code assistance directly in your IDE.
-
-**Key features:**
-- 🔍 **Semantic code search** across your entire project
-- 📚 **Context-aware assistance** for debugging and development
-- 🚀 **Zero-config setup** with automatic language detection
-
-```bash
-# Install LEANN globally for MCP integration
-uv tool install leann-core
-
-# Setup is automatic - just start using Claude Code!
-```
-Try our fully agentic pipeline with auto query rewriting, semantic search planning, and more:
-
-![LEANN MCP Integration](assets/mcp_leann.png)
-
-**Ready to supercharge your coding?** [Complete Setup Guide →](packages/leann-mcp/README.md)
 
 ## 🖥️ Command Line Interface
 
 LEANN includes a powerful CLI for document processing and search. Perfect for quick document indexing and interactive chat.
 
-### Installation
-
-If you followed the Quick Start, `leann` is already installed in your virtual environment:
 ```bash
-source .venv/bin/activate
-leann --help
-```
-
-**To make it globally available:**
-```bash
-# Install the LEANN CLI globally using uv tool
-uv tool install leann
-
-# Now you can use leann from anywhere without activating venv
-leann --help
-```
-
-> **Note**: Global installation is required for Claude Code integration. The `leann_mcp` server depends on the globally available `leann` command.
-
-
-
-### Usage Examples
-
-```bash
-# build from a specific directory, and my_docs is the index name(Here you can also build from multiple dict or multiple files)
-leann build my-docs --docs ./your_documents
+# Build an index from documents
+leann build my-docs --docs ./documents
 
 # Search your documents
 leann search my-docs "machine learning concepts"
@@ -543,13 +431,13 @@ Options:
 - **Dynamic batching:** Efficiently batch embedding computations for GPU utilization
 - **Two-level search:** Smart graph traversal that prioritizes promising nodes
 
-**Backends:** HNSW (default) for most use cases, with optional DiskANN support for billion-scale datasets.
+**Backends:** DiskANN or HNSW - pick what works for your data size.
 
 ## Benchmarks
 
 
-**[Simple Example: Compare LEANN vs FAISS →](benchmarks/compare_faiss_vs_leann.py)**
-### 📊 Storage Comparison
+📊 **[Simple Example: Compare LEANN vs FAISS →](examples/compare_faiss_vs_leann.py)**
+### Storage Comparison
 
 | System | DPR (2.1M) | Wiki (60M) | Chat (400K) | Email (780K) | Browser (38K) |
 |--------|-------------|------------|-------------|--------------|---------------|
@@ -563,7 +451,8 @@ Options:
 
 ```bash
 uv pip install -e ".[dev]"  # Install dev dependencies
-python benchmarks/run_evaluation.py    # Will auto-download evaluation data and run benchmarks
+python examples/run_evaluation.py data/indices/dpr/dpr_diskann      # DPR dataset
+python examples/run_evaluation.py data/indices/rpj_wiki/rpj_wiki.index  # Wikipedia
 ```
 
 The evaluation script downloads data automatically on first run. The last three results were tested with partial personal data, and you can reproduce them with your own data!
@@ -601,15 +490,9 @@ MIT License - see [LICENSE](LICENSE) for details.
 
 ## 🙏 Acknowledgments
 
-Core Contributors: [Yichuan Wang](https://yichuan-w.github.io/) & [Zhifei Li](https://github.com/andylizf).
+This work is done at [**Berkeley Sky Computing Lab**](https://sky.cs.berkeley.edu/)
+---
 
-We welcome more contributors! Feel free to open issues or submit PRs.
-
-This work is done at [**Berkeley Sky Computing Lab**](https://sky.cs.berkeley.edu/).
-
-## Star History
-
-[![Star History Chart](https://api.star-history.com/svg?repos=yichuan-w/LEANN&type=Date)](https://www.star-history.com/#yichuan-w/LEANN&Date)
 <p align="center">
   <strong>⭐ Star us on GitHub if Leann is useful for your research or applications!</strong>
 </p>

apps/base_rag_example.py

@@ -1,321 +0,0 @@
-"""
-Base class for unified RAG examples interface.
-Provides common parameters and functionality for all RAG examples.
-"""
-
-import argparse
-from abc import ABC, abstractmethod
-from pathlib import Path
-from typing import Any
-
-import dotenv
-from leann.api import LeannBuilder, LeannChat
-from llama_index.core.node_parser import SentenceSplitter
-
-dotenv.load_dotenv()
-
-
-class BaseRAGExample(ABC):
-    """Base class for all RAG examples with unified interface."""
-
-    def __init__(
-        self,
-        name: str,
-        description: str,
-        default_index_name: str,
-    ):
-        self.name = name
-        self.description = description
-        self.default_index_name = default_index_name
-        self.parser = self._create_parser()
-
-    def _create_parser(self) -> argparse.ArgumentParser:
-        """Create argument parser with common parameters."""
-        parser = argparse.ArgumentParser(
-            description=self.description, formatter_class=argparse.RawDescriptionHelpFormatter
-        )
-
-        # Core parameters (all examples share these)
-        core_group = parser.add_argument_group("Core Parameters")
-        core_group.add_argument(
-            "--index-dir",
-            type=str,
-            default=f"./{self.default_index_name}",
-            help=f"Directory to store the index (default: ./{self.default_index_name})",
-        )
-        core_group.add_argument(
-            "--query",
-            type=str,
-            default=None,
-            help="Query to run (if not provided, will run in interactive mode)",
-        )
-        # Allow subclasses to override default max_items
-        max_items_default = getattr(self, "max_items_default", -1)
-        core_group.add_argument(
-            "--max-items",
-            type=int,
-            default=max_items_default,
-            help="Maximum number of items to process  -1 for all, means index all documents, and you should set it to a reasonable number if you have a large dataset and try at the first time)",
-        )
-        core_group.add_argument(
-            "--force-rebuild", action="store_true", help="Force rebuild index even if it exists"
-        )
-
-        # Embedding parameters
-        embedding_group = parser.add_argument_group("Embedding Parameters")
-        # Allow subclasses to override default embedding_model
-        embedding_model_default = getattr(self, "embedding_model_default", "facebook/contriever")
-        embedding_group.add_argument(
-            "--embedding-model",
-            type=str,
-            default=embedding_model_default,
-            help=f"Embedding model to use (default: {embedding_model_default})",
-        )
-        embedding_group.add_argument(
-            "--embedding-mode",
-            type=str,
-            default="sentence-transformers",
-            choices=["sentence-transformers", "openai", "mlx", "ollama"],
-            help="Embedding backend mode (default: sentence-transformers)",
-        )
-
-        # LLM parameters
-        llm_group = parser.add_argument_group("LLM Parameters")
-        llm_group.add_argument(
-            "--llm",
-            type=str,
-            default="openai",
-            choices=["openai", "ollama", "hf", "simulated"],
-            help="LLM backend to use (default: openai)",
-        )
-        llm_group.add_argument(
-            "--llm-model",
-            type=str,
-            default=None,
-            help="LLM model name (default: gpt-4o for openai, llama3.2:1b for ollama)",
-        )
-        llm_group.add_argument(
-            "--llm-host",
-            type=str,
-            default="http://localhost:11434",
-            help="Host for Ollama API (default: http://localhost:11434)",
-        )
-        llm_group.add_argument(
-            "--thinking-budget",
-            type=str,
-            choices=["low", "medium", "high"],
-            default=None,
-            help="Thinking budget for reasoning models (low/medium/high). Supported by GPT-Oss:20b and other reasoning models.",
-        )
-
-        # Search parameters
-        search_group = parser.add_argument_group("Search Parameters")
-        search_group.add_argument(
-            "--top-k", type=int, default=20, help="Number of results to retrieve (default: 20)"
-        )
-        search_group.add_argument(
-            "--search-complexity",
-            type=int,
-            default=32,
-            help="Search complexity for graph traversal (default: 64)",
-        )
-
-        # Index building parameters
-        index_group = parser.add_argument_group("Index Building Parameters")
-        index_group.add_argument(
-            "--backend-name",
-            type=str,
-            default="hnsw",
-            choices=["hnsw", "diskann"],
-            help="Backend to use for index (default: hnsw)",
-        )
-        index_group.add_argument(
-            "--graph-degree",
-            type=int,
-            default=32,
-            help="Graph degree for index construction (default: 32)",
-        )
-        index_group.add_argument(
-            "--build-complexity",
-            type=int,
-            default=64,
-            help="Build complexity for index construction (default: 64)",
-        )
-        index_group.add_argument(
-            "--no-compact",
-            action="store_true",
-            help="Disable compact index storage",
-        )
-        index_group.add_argument(
-            "--no-recompute",
-            action="store_true",
-            help="Disable embedding recomputation",
-        )
-
-        # Add source-specific parameters
-        self._add_specific_arguments(parser)
-
-        return parser
-
-    @abstractmethod
-    def _add_specific_arguments(self, parser: argparse.ArgumentParser):
-        """Add source-specific arguments. Override in subclasses."""
-        pass
-
-    @abstractmethod
-    async def load_data(self, args) -> list[str]:
-        """Load data from the source. Returns list of text chunks."""
-        pass
-
-    def get_llm_config(self, args) -> dict[str, Any]:
-        """Get LLM configuration based on arguments."""
-        config = {"type": args.llm}
-
-        if args.llm == "openai":
-            config["model"] = args.llm_model or "gpt-4o"
-        elif args.llm == "ollama":
-            config["model"] = args.llm_model or "llama3.2:1b"
-            config["host"] = args.llm_host
-        elif args.llm == "hf":
-            config["model"] = args.llm_model or "Qwen/Qwen2.5-1.5B-Instruct"
-
-        return config
-
-    async def build_index(self, args, texts: list[str]) -> str:
-        """Build LEANN index from texts."""
-        index_path = str(Path(args.index_dir) / f"{self.default_index_name}.leann")
-
-        print(f"\n[Building Index] Creating {self.name} index...")
-        print(f"Total text chunks: {len(texts)}")
-
-        builder = LeannBuilder(
-            backend_name=args.backend_name,
-            embedding_model=args.embedding_model,
-            embedding_mode=args.embedding_mode,
-            graph_degree=args.graph_degree,
-            complexity=args.build_complexity,
-            is_compact=not args.no_compact,
-            is_recompute=not args.no_recompute,
-            num_threads=1,  # Force single-threaded mode
-        )
-
-        # Add texts in batches for better progress tracking
-        batch_size = 1000
-        for i in range(0, len(texts), batch_size):
-            batch = texts[i : i + batch_size]
-            for text in batch:
-                builder.add_text(text)
-            print(f"Added {min(i + batch_size, len(texts))}/{len(texts)} texts...")
-
-        print("Building index structure...")
-        builder.build_index(index_path)
-        print(f"Index saved to: {index_path}")
-
-        return index_path
-
-    async def run_interactive_chat(self, args, index_path: str):
-        """Run interactive chat with the index."""
-        chat = LeannChat(
-            index_path,
-            llm_config=self.get_llm_config(args),
-            system_prompt=f"You are a helpful assistant that answers questions about {self.name} data.",
-            complexity=args.search_complexity,
-        )
-
-        print(f"\n[Interactive Mode] Chat with your {self.name} data!")
-        print("Type 'quit' or 'exit' to stop.\n")
-
-        while True:
-            try:
-                query = input("You: ").strip()
-                if query.lower() in ["quit", "exit", "q"]:
-                    print("Goodbye!")
-                    break
-
-                if not query:
-                    continue
-
-                # Prepare LLM kwargs with thinking budget if specified
-                llm_kwargs = {}
-                if hasattr(args, "thinking_budget") and args.thinking_budget:
-                    llm_kwargs["thinking_budget"] = args.thinking_budget
-
-                response = chat.ask(
-                    query,
-                    top_k=args.top_k,
-                    complexity=args.search_complexity,
-                    llm_kwargs=llm_kwargs,
-                )
-                print(f"\nAssistant: {response}\n")
-
-            except KeyboardInterrupt:
-                print("\nGoodbye!")
-                break
-            except Exception as e:
-                print(f"Error: {e}")
-
-    async def run_single_query(self, args, index_path: str, query: str):
-        """Run a single query against the index."""
-        chat = LeannChat(
-            index_path,
-            llm_config=self.get_llm_config(args),
-            system_prompt=f"You are a helpful assistant that answers questions about {self.name} data.",
-            complexity=args.search_complexity,
-        )
-
-        print(f"\n[Query]: \033[36m{query}\033[0m")
-
-        # Prepare LLM kwargs with thinking budget if specified
-        llm_kwargs = {}
-        if hasattr(args, "thinking_budget") and args.thinking_budget:
-            llm_kwargs["thinking_budget"] = args.thinking_budget
-
-        response = chat.ask(
-            query, top_k=args.top_k, complexity=args.search_complexity, llm_kwargs=llm_kwargs
-        )
-        print(f"\n[Response]: \033[36m{response}\033[0m")
-
-    async def run(self):
-        """Main entry point for the example."""
-        args = self.parser.parse_args()
-
-        # Check if index exists
-        index_path = str(Path(args.index_dir) / f"{self.default_index_name}.leann")
-        index_exists = Path(args.index_dir).exists()
-
-        if not index_exists or args.force_rebuild:
-            # Load data and build index
-            print(f"\n{'Rebuilding' if index_exists else 'Building'} index...")
-            texts = await self.load_data(args)
-
-            if not texts:
-                print("No data found to index!")
-                return
-
-            index_path = await self.build_index(args, texts)
-        else:
-            print(f"\nUsing existing index in {args.index_dir}")
-
-        # Run query or interactive mode
-        if args.query:
-            await self.run_single_query(args, index_path, args.query)
-        else:
-            await self.run_interactive_chat(args, index_path)
-
-
-def create_text_chunks(documents, chunk_size=256, chunk_overlap=25) -> list[str]:
-    """Helper function to create text chunks from documents."""
-    node_parser = SentenceSplitter(
-        chunk_size=chunk_size,
-        chunk_overlap=chunk_overlap,
-        separator=" ",
-        paragraph_separator="\n\n",
-    )
-
-    all_texts = []
-    for doc in documents:
-        nodes = node_parser.get_nodes_from_documents([doc])
-        if nodes:
-            all_texts.extend(node.get_content() for node in nodes)
-
-    return all_texts

apps/browser_rag.py

@@ -1,170 +0,0 @@
-"""
-Browser History RAG example using the unified interface.
-Supports Chrome browser history.
-"""
-
-import os
-import sys
-from pathlib import Path
-
-# Add parent directory to path for imports
-sys.path.insert(0, str(Path(__file__).parent))
-
-from base_rag_example import BaseRAGExample, create_text_chunks
-
-from .history_data.history import ChromeHistoryReader
-
-
-class BrowserRAG(BaseRAGExample):
-    """RAG example for Chrome browser history."""
-
-    def __init__(self):
-        # Set default values BEFORE calling super().__init__
-        self.embedding_model_default = (
-            "sentence-transformers/all-MiniLM-L6-v2"  # Fast 384-dim model
-        )
-
-        super().__init__(
-            name="Browser History",
-            description="Process and query Chrome browser history with LEANN",
-            default_index_name="google_history_index",
-        )
-
-    def _add_specific_arguments(self, parser):
-        """Add browser-specific arguments."""
-        browser_group = parser.add_argument_group("Browser Parameters")
-        browser_group.add_argument(
-            "--chrome-profile",
-            type=str,
-            default=None,
-            help="Path to Chrome profile directory (auto-detected if not specified)",
-        )
-        browser_group.add_argument(
-            "--auto-find-profiles",
-            action="store_true",
-            default=True,
-            help="Automatically find all Chrome profiles (default: True)",
-        )
-        browser_group.add_argument(
-            "--chunk-size", type=int, default=256, help="Text chunk size (default: 256)"
-        )
-        browser_group.add_argument(
-            "--chunk-overlap", type=int, default=128, help="Text chunk overlap (default: 128)"
-        )
-
-    def _get_chrome_base_path(self) -> Path:
-        """Get the base Chrome profile path based on OS."""
-        if sys.platform == "darwin":
-            return Path.home() / "Library" / "Application Support" / "Google" / "Chrome"
-        elif sys.platform.startswith("linux"):
-            return Path.home() / ".config" / "google-chrome"
-        elif sys.platform == "win32":
-            return Path(os.environ["LOCALAPPDATA"]) / "Google" / "Chrome" / "User Data"
-        else:
-            raise ValueError(f"Unsupported platform: {sys.platform}")
-
-    def _find_chrome_profiles(self) -> list[Path]:
-        """Auto-detect all Chrome profiles."""
-        base_path = self._get_chrome_base_path()
-        if not base_path.exists():
-            return []
-
-        profiles = []
-
-        # Check Default profile
-        default_profile = base_path / "Default"
-        if default_profile.exists() and (default_profile / "History").exists():
-            profiles.append(default_profile)
-
-        # Check numbered profiles
-        for item in base_path.iterdir():
-            if item.is_dir() and item.name.startswith("Profile "):
-                if (item / "History").exists():
-                    profiles.append(item)
-
-        return profiles
-
-    async def load_data(self, args) -> list[str]:
-        """Load browser history and convert to text chunks."""
-        # Determine Chrome profiles
-        if args.chrome_profile and not args.auto_find_profiles:
-            profile_dirs = [Path(args.chrome_profile)]
-        else:
-            print("Auto-detecting Chrome profiles...")
-            profile_dirs = self._find_chrome_profiles()
-
-            # If specific profile given, filter to just that one
-            if args.chrome_profile:
-                profile_path = Path(args.chrome_profile)
-                profile_dirs = [p for p in profile_dirs if p == profile_path]
-
-        if not profile_dirs:
-            print("No Chrome profiles found!")
-            print("Please specify --chrome-profile manually")
-            return []
-
-        print(f"Found {len(profile_dirs)} Chrome profiles")
-
-        # Create reader
-        reader = ChromeHistoryReader()
-
-        # Process each profile
-        all_documents = []
-        total_processed = 0
-
-        for i, profile_dir in enumerate(profile_dirs):
-            print(f"\nProcessing profile {i + 1}/{len(profile_dirs)}: {profile_dir.name}")
-
-            try:
-                # Apply max_items limit per profile
-                max_per_profile = -1
-                if args.max_items > 0:
-                    remaining = args.max_items - total_processed
-                    if remaining <= 0:
-                        break
-                    max_per_profile = remaining
-
-                # Load history
-                documents = reader.load_data(
-                    chrome_profile_path=str(profile_dir),
-                    max_count=max_per_profile,
-                )
-
-                if documents:
-                    all_documents.extend(documents)
-                    total_processed += len(documents)
-                    print(f"Processed {len(documents)} history entries from this profile")
-
-            except Exception as e:
-                print(f"Error processing {profile_dir}: {e}")
-                continue
-
-        if not all_documents:
-            print("No browser history found to process!")
-            return []
-
-        print(f"\nTotal history entries processed: {len(all_documents)}")
-
-        # Convert to text chunks
-        all_texts = create_text_chunks(
-            all_documents, chunk_size=args.chunk_size, chunk_overlap=args.chunk_overlap
-        )
-
-        return all_texts
-
-
-if __name__ == "__main__":
-    import asyncio
-
-    # Example queries for browser history RAG
-    print("\n🌐 Browser History RAG Example")
-    print("=" * 50)
-    print("\nExample queries you can try:")
-    print("- 'What websites did I visit about machine learning?'")
-    print("- 'Find my search history about programming'")
-    print("- 'What YouTube videos did I watch recently?'")
-    print("- 'Show me websites about travel planning'")
-    print("\nNote: Make sure Chrome is closed before running\n")
-
-    rag = BrowserRAG()
-    asyncio.run(rag.run())

apps/document_rag.py

@@ -1,108 +0,0 @@
-"""
-Document RAG example using the unified interface.
-Supports PDF, TXT, MD, and other document formats.
-"""
-
-import sys
-from pathlib import Path
-
-# Add parent directory to path for imports
-sys.path.insert(0, str(Path(__file__).parent))
-
-from base_rag_example import BaseRAGExample, create_text_chunks
-from llama_index.core import SimpleDirectoryReader
-
-
-class DocumentRAG(BaseRAGExample):
-    """RAG example for document processing (PDF, TXT, MD, etc.)."""
-
-    def __init__(self):
-        super().__init__(
-            name="Document",
-            description="Process and query documents (PDF, TXT, MD, etc.) with LEANN",
-            default_index_name="test_doc_files",
-        )
-
-    def _add_specific_arguments(self, parser):
-        """Add document-specific arguments."""
-        doc_group = parser.add_argument_group("Document Parameters")
-        doc_group.add_argument(
-            "--data-dir",
-            type=str,
-            default="data",
-            help="Directory containing documents to index (default: data)",
-        )
-        doc_group.add_argument(
-            "--file-types",
-            nargs="+",
-            default=None,
-            help="Filter by file types (e.g., .pdf .txt .md). If not specified, all supported types are processed",
-        )
-        doc_group.add_argument(
-            "--chunk-size", type=int, default=256, help="Text chunk size (default: 256)"
-        )
-        doc_group.add_argument(
-            "--chunk-overlap", type=int, default=128, help="Text chunk overlap (default: 128)"
-        )
-
-    async def load_data(self, args) -> list[str]:
-        """Load documents and convert to text chunks."""
-        print(f"Loading documents from: {args.data_dir}")
-        if args.file_types:
-            print(f"Filtering by file types: {args.file_types}")
-        else:
-            print("Processing all supported file types")
-
-        # Check if data directory exists
-        data_path = Path(args.data_dir)
-        if not data_path.exists():
-            raise ValueError(f"Data directory not found: {args.data_dir}")
-
-        # Load documents
-        reader_kwargs = {
-            "recursive": True,
-            "encoding": "utf-8",
-        }
-        if args.file_types:
-            reader_kwargs["required_exts"] = args.file_types
-
-        documents = SimpleDirectoryReader(args.data_dir, **reader_kwargs).load_data(
-            show_progress=True
-        )
-
-        if not documents:
-            print(f"No documents found in {args.data_dir} with extensions {args.file_types}")
-            return []
-
-        print(f"Loaded {len(documents)} documents")
-
-        # Convert to text chunks
-        all_texts = create_text_chunks(
-            documents, chunk_size=args.chunk_size, chunk_overlap=args.chunk_overlap
-        )
-
-        # Apply max_items limit if specified
-        if args.max_items > 0 and len(all_texts) > args.max_items:
-            print(f"Limiting to {args.max_items} chunks (from {len(all_texts)})")
-            all_texts = all_texts[: args.max_items]
-
-        return all_texts
-
-
-if __name__ == "__main__":
-    import asyncio
-
-    # Example queries for document RAG
-    print("\n📄 Document RAG Example")
-    print("=" * 50)
-    print("\nExample queries you can try:")
-    print("- 'What are the main techniques LEANN uses?'")
-    print("- 'What is the technique DLPM?'")
-    print("- 'Who does Elizabeth Bennet marry?'")
-    print(
-        "- 'What is the problem of developing pan gu model Huawei meets? (盘古大模型开发中遇到什么问题?)'"
-    )
-    print("\nOr run without --query for interactive mode\n")
-
-    rag = DocumentRAG()
-    asyncio.run(rag.run())

apps/email_rag.py

@@ -1,156 +0,0 @@
-"""
-Email RAG example using the unified interface.
-Supports Apple Mail on macOS.
-"""
-
-import sys
-from pathlib import Path
-
-# Add parent directory to path for imports
-sys.path.insert(0, str(Path(__file__).parent))
-
-from base_rag_example import BaseRAGExample, create_text_chunks
-
-from .email_data.LEANN_email_reader import EmlxReader
-
-
-class EmailRAG(BaseRAGExample):
-    """RAG example for Apple Mail processing."""
-
-    def __init__(self):
-        # Set default values BEFORE calling super().__init__
-        self.max_items_default = -1  # Process all emails by default
-        self.embedding_model_default = (
-            "sentence-transformers/all-MiniLM-L6-v2"  # Fast 384-dim model
-        )
-
-        super().__init__(
-            name="Email",
-            description="Process and query Apple Mail emails with LEANN",
-            default_index_name="mail_index",
-        )
-
-    def _add_specific_arguments(self, parser):
-        """Add email-specific arguments."""
-        email_group = parser.add_argument_group("Email Parameters")
-        email_group.add_argument(
-            "--mail-path",
-            type=str,
-            default=None,
-            help="Path to Apple Mail directory (auto-detected if not specified)",
-        )
-        email_group.add_argument(
-            "--include-html", action="store_true", help="Include HTML content in email processing"
-        )
-        email_group.add_argument(
-            "--chunk-size", type=int, default=256, help="Text chunk size (default: 256)"
-        )
-        email_group.add_argument(
-            "--chunk-overlap", type=int, default=25, help="Text chunk overlap (default: 25)"
-        )
-
-    def _find_mail_directories(self) -> list[Path]:
-        """Auto-detect all Apple Mail directories."""
-        mail_base = Path.home() / "Library" / "Mail"
-        if not mail_base.exists():
-            return []
-
-        # Find all Messages directories
-        messages_dirs = []
-        for item in mail_base.rglob("Messages"):
-            if item.is_dir():
-                messages_dirs.append(item)
-
-        return messages_dirs
-
-    async def load_data(self, args) -> list[str]:
-        """Load emails and convert to text chunks."""
-        # Determine mail directories
-        if args.mail_path:
-            messages_dirs = [Path(args.mail_path)]
-        else:
-            print("Auto-detecting Apple Mail directories...")
-            messages_dirs = self._find_mail_directories()
-
-        if not messages_dirs:
-            print("No Apple Mail directories found!")
-            print("Please specify --mail-path manually")
-            return []
-
-        print(f"Found {len(messages_dirs)} mail directories")
-
-        # Create reader
-        reader = EmlxReader(include_html=args.include_html)
-
-        # Process each directory
-        all_documents = []
-        total_processed = 0
-
-        for i, messages_dir in enumerate(messages_dirs):
-            print(f"\nProcessing directory {i + 1}/{len(messages_dirs)}: {messages_dir}")
-
-            try:
-                # Count emlx files
-                emlx_files = list(messages_dir.glob("*.emlx"))
-                print(f"Found {len(emlx_files)} email files")
-
-                # Apply max_items limit per directory
-                max_per_dir = -1  # Default to process all
-                if args.max_items > 0:
-                    remaining = args.max_items - total_processed
-                    if remaining <= 0:
-                        break
-                    max_per_dir = remaining
-                # If args.max_items == -1, max_per_dir stays -1 (process all)
-
-                # Load emails - fix the parameter passing
-                documents = reader.load_data(
-                    input_dir=str(messages_dir),
-                    max_count=max_per_dir,
-                )
-
-                if documents:
-                    all_documents.extend(documents)
-                    total_processed += len(documents)
-                    print(f"Processed {len(documents)} emails from this directory")
-
-            except Exception as e:
-                print(f"Error processing {messages_dir}: {e}")
-                continue
-
-        if not all_documents:
-            print("No emails found to process!")
-            return []
-
-        print(f"\nTotal emails processed: {len(all_documents)}")
-        print("now starting to split into text chunks ... take some time")
-
-        # Convert to text chunks
-        # Email reader uses chunk_overlap=25 as in original
-        all_texts = create_text_chunks(
-            all_documents, chunk_size=args.chunk_size, chunk_overlap=args.chunk_overlap
-        )
-
-        return all_texts
-
-
-if __name__ == "__main__":
-    import asyncio
-
-    # Check platform
-    if sys.platform != "darwin":
-        print("\n⚠️  Warning: This example is designed for macOS (Apple Mail)")
-        print("   Windows/Linux support coming soon!\n")
-
-    # Example queries for email RAG
-    print("\n📧 Email RAG Example")
-    print("=" * 50)
-    print("\nExample queries you can try:")
-    print("- 'What did my boss say about deadlines?'")
-    print("- 'Find emails about travel expenses'")
-    print("- 'Show me emails from last month about the project'")
-    print("- 'What food did I order from DoorDash?'")
-    print("\nNote: You may need to grant Full Disk Access to your terminal\n")
-
-    rag = EmailRAG()
-    asyncio.run(rag.run())

apps/wechat_rag.py

@@ -1,189 +0,0 @@
-"""
-WeChat History RAG example using the unified interface.
-Supports WeChat chat history export and search.
-"""
-
-import subprocess
-import sys
-from pathlib import Path
-
-# Add parent directory to path for imports
-sys.path.insert(0, str(Path(__file__).parent))
-
-from base_rag_example import BaseRAGExample
-
-from .history_data.wechat_history import WeChatHistoryReader
-
-
-class WeChatRAG(BaseRAGExample):
-    """RAG example for WeChat chat history."""
-
-    def __init__(self):
-        # Set default values BEFORE calling super().__init__
-        self.max_items_default = -1  # Match original default
-        self.embedding_model_default = (
-            "sentence-transformers/all-MiniLM-L6-v2"  # Fast 384-dim model
-        )
-
-        super().__init__(
-            name="WeChat History",
-            description="Process and query WeChat chat history with LEANN",
-            default_index_name="wechat_history_magic_test_11Debug_new",
-        )
-
-    def _add_specific_arguments(self, parser):
-        """Add WeChat-specific arguments."""
-        wechat_group = parser.add_argument_group("WeChat Parameters")
-        wechat_group.add_argument(
-            "--export-dir",
-            type=str,
-            default="./wechat_export",
-            help="Directory to store WeChat exports (default: ./wechat_export)",
-        )
-        wechat_group.add_argument(
-            "--force-export",
-            action="store_true",
-            help="Force re-export of WeChat data even if exports exist",
-        )
-        wechat_group.add_argument(
-            "--chunk-size", type=int, default=192, help="Text chunk size (default: 192)"
-        )
-        wechat_group.add_argument(
-            "--chunk-overlap", type=int, default=64, help="Text chunk overlap (default: 64)"
-        )
-
-    def _export_wechat_data(self, export_dir: Path) -> bool:
-        """Export WeChat data using wechattweak-cli."""
-        print("Exporting WeChat data...")
-
-        # Check if WeChat is running
-        try:
-            result = subprocess.run(["pgrep", "WeChat"], capture_output=True, text=True)
-            if result.returncode != 0:
-                print("WeChat is not running. Please start WeChat first.")
-                return False
-        except Exception:
-            pass  # pgrep might not be available on all systems
-
-        # Create export directory
-        export_dir.mkdir(parents=True, exist_ok=True)
-
-        # Run export command
-        cmd = ["packages/wechat-exporter/wechattweak-cli", "export", str(export_dir)]
-
-        try:
-            print(f"Running: {' '.join(cmd)}")
-            result = subprocess.run(cmd, capture_output=True, text=True)
-
-            if result.returncode == 0:
-                print("WeChat data exported successfully!")
-                return True
-            else:
-                print(f"Export failed: {result.stderr}")
-                return False
-
-        except FileNotFoundError:
-            print("\nError: wechattweak-cli not found!")
-            print("Please install it first:")
-            print("  sudo packages/wechat-exporter/wechattweak-cli install")
-            return False
-        except Exception as e:
-            print(f"Export error: {e}")
-            return False
-
-    async def load_data(self, args) -> list[str]:
-        """Load WeChat history and convert to text chunks."""
-        # Initialize WeChat reader with export capabilities
-        reader = WeChatHistoryReader()
-
-        # Find existing exports or create new ones using the centralized method
-        export_dirs = reader.find_or_export_wechat_data(args.export_dir)
-        if not export_dirs:
-            print("Failed to find or export WeChat data. Trying to find any existing exports...")
-            # Try to find any existing exports in common locations
-            export_dirs = reader.find_wechat_export_dirs()
-            if not export_dirs:
-                print("No WeChat data found. Please ensure WeChat exports exist.")
-                return []
-
-        # Load documents from all found export directories
-        all_documents = []
-        total_processed = 0
-
-        for i, export_dir in enumerate(export_dirs):
-            print(f"\nProcessing WeChat export {i + 1}/{len(export_dirs)}: {export_dir}")
-
-            try:
-                # Apply max_items limit per export
-                max_per_export = -1
-                if args.max_items > 0:
-                    remaining = args.max_items - total_processed
-                    if remaining <= 0:
-                        break
-                    max_per_export = remaining
-
-                documents = reader.load_data(
-                    wechat_export_dir=str(export_dir),
-                    max_count=max_per_export,
-                    concatenate_messages=True,  # Enable message concatenation for better context
-                )
-
-                if documents:
-                    print(f"Loaded {len(documents)} chat documents from {export_dir}")
-                    all_documents.extend(documents)
-                    total_processed += len(documents)
-                else:
-                    print(f"No documents loaded from {export_dir}")
-
-            except Exception as e:
-                print(f"Error processing {export_dir}: {e}")
-                continue
-
-        if not all_documents:
-            print("No documents loaded from any source. Exiting.")
-            return []
-
-        print(f"\nTotal loaded {len(all_documents)} chat documents from {len(export_dirs)} exports")
-        print("now starting to split into text chunks ... take some time")
-
-        # Convert to text chunks with contact information
-        all_texts = []
-        for doc in all_documents:
-            # Split the document into chunks
-            from llama_index.core.node_parser import SentenceSplitter
-
-            text_splitter = SentenceSplitter(
-                chunk_size=args.chunk_size, chunk_overlap=args.chunk_overlap
-            )
-            nodes = text_splitter.get_nodes_from_documents([doc])
-
-            for node in nodes:
-                # Add contact information to each chunk
-                contact_name = doc.metadata.get("contact_name", "Unknown")
-                text = f"[Contact] means the message is from: {contact_name}\n" + node.get_content()
-                all_texts.append(text)
-
-        print(f"Created {len(all_texts)} text chunks from {len(all_documents)} documents")
-        return all_texts
-
-
-if __name__ == "__main__":
-    import asyncio
-
-    # Check platform
-    if sys.platform != "darwin":
-        print("\n⚠️  Warning: WeChat export is only supported on macOS")
-        print("   You can still query existing exports on other platforms\n")
-
-    # Example queries for WeChat RAG
-    print("\n💬 WeChat History RAG Example")
-    print("=" * 50)
-    print("\nExample queries you can try:")
-    print("- 'Show me conversations about travel plans'")
-    print("- 'Find group chats about weekend activities'")
-    print("- '我想买魔术师约翰逊的球衣,给我一些对应聊天记录?'")
-    print("- 'What did we discuss about the project last month?'")
-    print("\nNote: WeChat must be running for export to work\n")
-
-    rag = WeChatRAG()
-    asyncio.run(rag.run())

data/README.md

@@ -0,0 +1,44 @@
+---
+license: mit
+---
+
+# LEANN-RAG Evaluation Data
+
+This repository contains the necessary data to run the recall evaluation scripts for the [LEANN-RAG](https://huggingface.co/LEANN-RAG) project.
+
+## Dataset Components
+
+This dataset is structured into three main parts:
+
+1.  **Pre-built LEANN Indices**:
+    *   `dpr/`: A pre-built index for the DPR dataset.
+    *   `rpj_wiki/`: A pre-built index for the RPJ-Wiki dataset.
+    These indices were created using the `leann-core` library and are required by the `LeannSearcher`.
+
+2.  **Ground Truth Data**:
+    *   `ground_truth/`: Contains the ground truth files (`flat_results_nq_k3.json`) for both the DPR and RPJ-Wiki datasets. These files map queries to the original passage IDs from the Natural Questions benchmark, evaluated using the Contriever model.
+
+3.  **Queries**:
+    *   `queries/`: Contains the `nq_open.jsonl` file with the Natural Questions queries used for the evaluation.
+
+## Usage
+
+To use this data, you can download it locally using the `huggingface-hub` library. First, install the library:
+
+```bash
+pip install huggingface-hub
+```
+
+Then, you can download the entire dataset to a local directory (e.g., `data/`) with the following Python script:
+
+```python
+from huggingface_hub import snapshot_download
+
+snapshot_download(
+    repo_id="LEANN-RAG/leann-rag-evaluation-data",
+    repo_type="dataset",
+    local_dir="data"
+)
+```
+
+This will download all the necessary files into a local `data` folder, preserving the repository structure. The evaluation scripts in the main [LEANN-RAG Space](https://huggingface.co/LEANN-RAG) are configured to work with this data structure.

docs/THINKING_BUDGET_FEATURE.md

@@ -1,123 +0,0 @@
-# Thinking Budget Feature Implementation
-
-## Overview
-
-This document describes the implementation of the **thinking budget** feature for LEANN, which allows users to control the computational effort for reasoning models like GPT-Oss:20b.
-
-## Feature Description
-
-The thinking budget feature provides three levels of computational effort for reasoning models:
-- **`low`**: Fast responses, basic reasoning (default for simple queries)
-- **`medium`**: Balanced speed and reasoning depth
-- **`high`**: Maximum reasoning effort, best for complex analytical questions
-
-## Implementation Details
-
-### 1. Command Line Interface
-
-Added `--thinking-budget` parameter to both CLI and RAG examples:
-
-```bash
-# LEANN CLI
-leann ask my-index --llm ollama --model gpt-oss:20b --thinking-budget high
-
-# RAG Examples
-python apps/email_rag.py --llm ollama --llm-model gpt-oss:20b --thinking-budget high
-python apps/document_rag.py --llm openai --llm-model o3 --thinking-budget medium
-```
-
-### 2. LLM Backend Support
-
-#### Ollama Backend (`packages/leann-core/src/leann/chat.py`)
-
-```python
-def ask(self, prompt: str, **kwargs) -> str:
-    # Handle thinking budget for reasoning models
-    options = kwargs.copy()
-    thinking_budget = kwargs.get("thinking_budget")
-    if thinking_budget:
-        options.pop("thinking_budget", None)
-        if thinking_budget in ["low", "medium", "high"]:
-            options["reasoning"] = {"effort": thinking_budget, "exclude": False}
-```
-
-**API Format**: Uses Ollama's `reasoning` parameter with `effort` and `exclude` fields.
-
-#### OpenAI Backend (`packages/leann-core/src/leann/chat.py`)
-
-```python
-def ask(self, prompt: str, **kwargs) -> str:
-    # Handle thinking budget for reasoning models
-    thinking_budget = kwargs.get("thinking_budget")
-    if thinking_budget and thinking_budget in ["low", "medium", "high"]:
-        # Check if this is an o-series model
-        o_series_models = ["o3", "o3-mini", "o4-mini", "o1", "o3-pro", "o3-deep-research"]
-        if any(model in self.model for model in o_series_models):
-            params["reasoning_effort"] = thinking_budget
-```
-
-**API Format**: Uses OpenAI's `reasoning_effort` parameter for o-series models.
-
-### 3. Parameter Propagation
-
-The thinking budget parameter is properly propagated through the LEANN architecture:
-
-1. **CLI** (`packages/leann-core/src/leann/cli.py`): Captures `--thinking-budget` argument
-2. **Base RAG** (`apps/base_rag_example.py`): Adds parameter to argument parser
-3. **LeannChat** (`packages/leann-core/src/leann/api.py`): Passes `llm_kwargs` to LLM
-4. **LLM Interface**: Handles the parameter in backend-specific implementations
-
-## Files Modified
-
-### Core Implementation
-- `packages/leann-core/src/leann/chat.py`: Added thinking budget support to OllamaChat and OpenAIChat
-- `packages/leann-core/src/leann/cli.py`: Added `--thinking-budget` argument
-- `apps/base_rag_example.py`: Added thinking budget parameter to RAG examples
-
-### Documentation
-- `README.md`: Added thinking budget parameter to usage examples
-- `docs/configuration-guide.md`: Added detailed documentation and usage guidelines
-
-### Examples
-- `examples/thinking_budget_demo.py`: Comprehensive demo script with usage examples
-
-## Usage Examples
-
-### Basic Usage
-```bash
-# High reasoning effort for complex questions
-leann ask my-index --llm ollama --model gpt-oss:20b --thinking-budget high
-
-# Medium reasoning for balanced performance
-leann ask my-index --llm openai --model gpt-4o --thinking-budget medium
-
-# Low reasoning for fast responses
-leann ask my-index --llm ollama --model gpt-oss:20b --thinking-budget low
-```
-
-### RAG Examples
-```bash
-# Email RAG with high reasoning
-python apps/email_rag.py --llm ollama --llm-model gpt-oss:20b --thinking-budget high
-
-# Document RAG with medium reasoning
-python apps/document_rag.py --llm openai --llm-model gpt-4o --thinking-budget medium
-```
-
-## Supported Models
-
-### Ollama Models
-- **GPT-Oss:20b**: Primary target model with reasoning capabilities
-- **Other reasoning models**: Any Ollama model that supports the `reasoning` parameter
-
-### OpenAI Models
-- **o3, o3-mini, o4-mini, o1**: o-series reasoning models with `reasoning_effort` parameter
-- **GPT-OSS models**: Models that support reasoning capabilities
-
-## Testing
-
-The implementation includes comprehensive testing:
-- Parameter handling verification
-- Backend-specific API format validation
-- CLI argument parsing tests
-- Integration with existing LEANN architecture

docs/code/embedding_model_compare.py

@@ -1,98 +0,0 @@
-"""
-Comparison between Sentence Transformers and OpenAI embeddings
-
-This example shows how different embedding models handle complex queries
-and demonstrates the differences between local and API-based embeddings.
-"""
-
-import numpy as np
-from leann.embedding_compute import compute_embeddings
-
-# OpenAI API key should be set as environment variable
-# export OPENAI_API_KEY="your-api-key-here"
-
-# Test data
-conference_text = "[Title]: COLING 2025 Conference\n[URL]: https://coling2025.org/"
-browser_text = "[Title]: Browser Use Tool\n[URL]: https://github.com/browser-use"
-
-# Two queries with same intent but different wording
-query1 = "Tell me my browser history about some conference i often visit"
-query2 = "browser history about conference I often visit"
-
-texts = [query1, query2, conference_text, browser_text]
-
-
-def cosine_similarity(a, b):
-    return np.dot(a, b)  # Already normalized
-
-
-def analyze_embeddings(embeddings, model_name):
-    print(f"\n=== {model_name} Results ===")
-
-    # Results for Query 1
-    sim1_conf = cosine_similarity(embeddings[0], embeddings[2])
-    sim1_browser = cosine_similarity(embeddings[0], embeddings[3])
-
-    print(f"Query 1: '{query1}'")
-    print(f"  → Conference similarity: {sim1_conf:.4f} {'✓' if sim1_conf > sim1_browser else ''}")
-    print(
-        f"  → Browser similarity:    {sim1_browser:.4f} {'✓' if sim1_browser > sim1_conf else ''}"
-    )
-    print(f"  Winner: {'Conference' if sim1_conf > sim1_browser else 'Browser'}")
-
-    # Results for Query 2
-    sim2_conf = cosine_similarity(embeddings[1], embeddings[2])
-    sim2_browser = cosine_similarity(embeddings[1], embeddings[3])
-
-    print(f"\nQuery 2: '{query2}'")
-    print(f"  → Conference similarity: {sim2_conf:.4f} {'✓' if sim2_conf > sim2_browser else ''}")
-    print(
-        f"  → Browser similarity:    {sim2_browser:.4f} {'✓' if sim2_browser > sim2_conf else ''}"
-    )
-    print(f"  Winner: {'Conference' if sim2_conf > sim2_browser else 'Browser'}")
-
-    # Show the impact
-    print("\n=== Impact Analysis ===")
-    print(f"Conference similarity change: {sim2_conf - sim1_conf:+.4f}")
-    print(f"Browser similarity change:    {sim2_browser - sim1_browser:+.4f}")
-
-    if sim1_conf > sim1_browser and sim2_browser > sim2_conf:
-        print("❌ FLIP: Adding 'browser history' flips winner from Conference to Browser!")
-    elif sim1_conf > sim1_browser and sim2_conf > sim2_browser:
-        print("✅ STABLE: Conference remains winner in both queries")
-    elif sim1_browser > sim1_conf and sim2_browser > sim2_conf:
-        print("✅ STABLE: Browser remains winner in both queries")
-    else:
-        print("🔄 MIXED: Results vary between queries")
-
-    return {
-        "query1_conf": sim1_conf,
-        "query1_browser": sim1_browser,
-        "query2_conf": sim2_conf,
-        "query2_browser": sim2_browser,
-    }
-
-
-# Test Sentence Transformers
-print("Testing Sentence Transformers (facebook/contriever)...")
-try:
-    st_embeddings = compute_embeddings(texts, "facebook/contriever", mode="sentence-transformers")
-    st_results = analyze_embeddings(st_embeddings, "Sentence Transformers (facebook/contriever)")
-except Exception as e:
-    print(f"❌ Sentence Transformers failed: {e}")
-    st_results = None
-
-# Test OpenAI
-print("\n" + "=" * 60)
-print("Testing OpenAI (text-embedding-3-small)...")
-try:
-    openai_embeddings = compute_embeddings(texts, "text-embedding-3-small", mode="openai")
-    openai_results = analyze_embeddings(openai_embeddings, "OpenAI (text-embedding-3-small)")
-except Exception as e:
-    print(f"❌ OpenAI failed: {e}")
-    openai_results = None
-
-# Compare results
-if st_results and openai_results:
-    print("\n" + "=" * 60)
-    print("=== COMPARISON SUMMARY ===")

docs/configuration-guide.md

@@ -1,285 +0,0 @@
-# LEANN Configuration Guide
-
-This guide helps you optimize LEANN for different use cases and understand the trade-offs between various configuration options.
-
-## Getting Started: Simple is Better
-
-When first trying LEANN, start with a small dataset to quickly validate your approach:
-
-**For document RAG**: The default `data/` directory works perfectly - includes 2 AI research papers, Pride and Prejudice literature, and a technical report
-```bash
-python -m apps.document_rag --query "What techniques does LEANN use?"
-```
-
-**For other data sources**: Limit the dataset size for quick testing
-```bash
-# WeChat: Test with recent messages only
-python -m apps.wechat_rag --max-items 100 --query "What did we discuss about the project timeline?"
-
-# Browser history: Last few days
-python -m apps.browser_rag --max-items 500 --query "Find documentation about vector databases"
-
-# Email: Recent inbox
-python -m apps.email_rag --max-items 200 --query "Who sent updates about the deployment status?"
-```
-
-Once validated, scale up gradually:
-- 100 documents → 1,000 → 10,000 → full dataset (`--max-items -1`)
-- This helps identify issues early before committing to long processing times
-
-## Embedding Model Selection: Understanding the Trade-offs
-
-Based on our experience developing LEANN, embedding models fall into three categories:
-
-### Small Models (< 100M parameters)
-**Example**: `sentence-transformers/all-MiniLM-L6-v2` (22M params)
-- **Pros**: Lightweight, fast for both indexing and inference
-- **Cons**: Lower semantic understanding, may miss nuanced relationships
-- **Use when**: Speed is critical, handling simple queries, interactive mode, or just experimenting with LEANN. If time is not a constraint, consider using a larger/better embedding model
-
-### Medium Models (100M-500M parameters)
-**Example**: `facebook/contriever` (110M params), `BAAI/bge-base-en-v1.5` (110M params)
-- **Pros**: Balanced performance, good multilingual support, reasonable speed
-- **Cons**: Requires more compute than small models
-- **Use when**: Need quality results without extreme compute requirements, general-purpose RAG applications
-
-### Large Models (500M+ parameters)
-**Example**: `Qwen/Qwen3-Embedding-0.6B` (600M params), `intfloat/multilingual-e5-large` (560M params)
-- **Pros**: Best semantic understanding, captures complex relationships, excellent multilingual support. **Qwen3-Embedding-0.6B achieves nearly OpenAI API performance!**
-- **Cons**: Slower inference, longer index build times
-- **Use when**: Quality is paramount and you have sufficient compute resources. **Highly recommended** for production use
-
-### Quick Start: Cloud and Local Embedding Options
-
-**OpenAI Embeddings (Fastest Setup)**
-For immediate testing without local model downloads:
-```bash
-# Set OpenAI embeddings (requires OPENAI_API_KEY)
---embedding-mode openai --embedding-model text-embedding-3-small
-```
-
-**Ollama Embeddings (Privacy-Focused)**
-For local embeddings with complete privacy:
-```bash
-# First, pull an embedding model
-ollama pull nomic-embed-text
-
-# Use Ollama embeddings
---embedding-mode ollama --embedding-model nomic-embed-text
-```
-
-<details>
-<summary><strong>Cloud vs Local Trade-offs</strong></summary>
-
-**OpenAI Embeddings** (`text-embedding-3-small/large`)
-- **Pros**: No local compute needed, consistently fast, high quality
-- **Cons**: Requires API key, costs money, data leaves your system, [known limitations with certain languages](https://yichuan-w.github.io/blog/lessons_learned_in_dev_leann/)
-- **When to use**: Prototyping, non-sensitive data, need immediate results
-
-**Local Embeddings**
-- **Pros**: Complete privacy, no ongoing costs, full control, can sometimes outperform OpenAI embeddings
-- **Cons**: Slower than cloud APIs, requires local compute resources
-- **When to use**: Production systems, sensitive data, cost-sensitive applications
-
-</details>
-
-## Index Selection: Matching Your Scale
-
-### HNSW (Hierarchical Navigable Small World)
-**Best for**: Small to medium datasets (< 10M vectors) - **Default and recommended for extreme low storage**
-- Full recomputation required
-- High memory usage during build phase
-- Excellent recall (95%+)
-
-```bash
-# Optimal for most use cases
---backend-name hnsw --graph-degree 32 --build-complexity 64
-```
-
-### DiskANN
-**Best for**: Large datasets (> 10M vectors, 10GB+ index size) - **⚠️ Beta version, still in active development**
-- Uses Product Quantization (PQ) for coarse filtering during graph traversal
-- Novel approach: stores only PQ codes, performs rerank with exact computation in final step
-- Implements a corner case of double-queue: prunes all neighbors and recomputes at the end
-
-```bash
-# For billion-scale deployments
---backend-name diskann --graph-degree 64 --build-complexity 128
-```
-
-## LLM Selection: Engine and Model Comparison
-
-### LLM Engines
-
-**OpenAI** (`--llm openai`)
-- **Pros**: Best quality, consistent performance, no local resources needed
-- **Cons**: Costs money ($0.15-2.5 per million tokens), requires internet, data privacy concerns
-- **Models**: `gpt-4o-mini` (fast, cheap), `gpt-4o` (best quality), `o3` (reasoning), `o3-mini` (reasoning, cheaper)
-- **Thinking Budget**: Use `--thinking-budget low/medium/high` for o-series reasoning models (o3, o3-mini, o4-mini)
-- **Note**: Our current default, but we recommend switching to Ollama for most use cases
-
-**Ollama** (`--llm ollama`)
-- **Pros**: Fully local, free, privacy-preserving, good model variety
-- **Cons**: Requires local GPU/CPU resources, slower than cloud APIs, need to install extra [ollama app](https://github.com/ollama/ollama?tab=readme-ov-file#ollama) and pre-download models by `ollama pull`
-- **Models**: `qwen3:0.6b` (ultra-fast), `qwen3:1.7b` (balanced), `qwen3:4b` (good quality), `qwen3:7b` (high quality), `deepseek-r1:1.5b` (reasoning)
-- **Thinking Budget**: Use `--thinking-budget low/medium/high` for reasoning models like GPT-Oss:20b
-
-**HuggingFace** (`--llm hf`)
-- **Pros**: Free tier available, huge model selection, direct model loading (vs Ollama's server-based approach)
-- **Cons**: More complex initial setup
-- **Models**: `Qwen/Qwen3-1.7B-FP8`
-
-## Parameter Tuning Guide
-
-### Search Complexity Parameters
-
-**`--build-complexity`** (index building)
-- Controls thoroughness during index construction
-- Higher = better recall but slower build
-- Recommendations:
-  - 32: Quick prototyping
-  - 64: Balanced (default)
-  - 128: Production systems
-  - 256: Maximum quality
-
-**`--search-complexity`** (query time)
-- Controls search thoroughness
-- Higher = better results but slower
-- Recommendations:
-  - 16: Fast/Interactive search
-  - 32: High quality with diversity
-  - 64+: Maximum accuracy
-
-### Top-K Selection
-
-**`--top-k`** (number of retrieved chunks)
-- More chunks = better context but slower LLM processing
-- Should be always smaller than `--search-complexity`
-- Guidelines:
-  - 10-20: General questions (default: 20)
-  - 30+: Complex multi-hop reasoning requiring comprehensive context
-
-**Trade-off formula**:
-- Retrieval time ∝ log(n) × search_complexity
-- LLM processing time ∝ top_k × chunk_size
-- Total context = top_k × chunk_size tokens
-
-### Thinking Budget for Reasoning Models
-
-**`--thinking-budget`** (reasoning effort level)
-- Controls the computational effort for reasoning models
-- Options: `low`, `medium`, `high`
-- Guidelines:
-  - `low`: Fast responses, basic reasoning (default for simple queries)
-  - `medium`: Balanced speed and reasoning depth
-  - `high`: Maximum reasoning effort, best for complex analytical questions
-- **Supported Models**:
-  - **Ollama**: `gpt-oss:20b`, `gpt-oss:120b`
-  - **OpenAI**: `o3`, `o3-mini`, `o4-mini`, `o1` (o-series reasoning models)
-- **Note**: Models without reasoning support will show a warning and proceed without reasoning parameters
-- **Example**: `--thinking-budget high` for complex analytical questions
-
-**📖 For detailed usage examples and implementation details, check out [Thinking Budget Documentation](THINKING_BUDGET_FEATURE.md)**
-
-**💡 Quick Examples:**
-```bash
-# OpenAI o-series reasoning model
-python apps/document_rag.py --query "What are the main techniques LEANN explores?" \
-  --index-dir hnswbuild --backend hnsw \
-  --llm openai --llm-model o3 --thinking-budget medium
-
-# Ollama reasoning model
-python apps/document_rag.py --query "What are the main techniques LEANN explores?" \
-  --index-dir hnswbuild --backend hnsw \
-  --llm ollama --llm-model gpt-oss:20b --thinking-budget high
-```
-
-### Graph Degree (HNSW/DiskANN)
-
-**`--graph-degree`**
-- Number of connections per node in the graph
-- Higher = better recall but more memory
-- HNSW: 16-32 (default: 32)
-- DiskANN: 32-128 (default: 64)
-
-
-## Performance Optimization Checklist
-
-### If Embedding is Too Slow
-
-1. **Switch to smaller model**:
-   ```bash
-   # From large model
-   --embedding-model Qwen/Qwen3-Embedding-0.6B
-   # To small model
-   --embedding-model sentence-transformers/all-MiniLM-L6-v2
-   ```
-
-2. **Limit dataset size for testing**:
-   ```bash
-   --max-items 1000  # Process first 1k items only
-   ```
-
-3. **Use MLX on Apple Silicon** (optional optimization):
-   ```bash
-   --embedding-mode mlx --embedding-model mlx-community/Qwen3-Embedding-0.6B-8bit
-   ```
-    MLX might not be the best choice, as we tested and found that it only offers 1.3x acceleration compared to HF, so maybe using ollama is a better choice for embedding generation
-
-4. **Use Ollama**
-   ```bash
-   --embedding-mode ollama --embedding-model nomic-embed-text
-   ```
-   To discover additional embedding models in ollama, check out https://ollama.com/search?c=embedding or read more about embedding models at https://ollama.com/blog/embedding-models, please do check the model size that works best for you
-### If Search Quality is Poor
-
-1. **Increase retrieval count**:
-   ```bash
-   --top-k 30  # Retrieve more candidates
-   ```
-
-2. **Upgrade embedding model**:
-   ```bash
-   # For English
-   --embedding-model BAAI/bge-base-en-v1.5
-   # For multilingual
-   --embedding-model intfloat/multilingual-e5-large
-   ```
-
-## Understanding the Trade-offs
-
-Every configuration choice involves trade-offs:
-
-| Factor | Small/Fast | Large/Quality |
-|--------|------------|---------------|
-| Embedding Model | `all-MiniLM-L6-v2` | `Qwen/Qwen3-Embedding-0.6B` |
-| Chunk Size | 512 tokens | 128 tokens |
-| Index Type | HNSW | DiskANN |
-| LLM | `qwen3:1.7b` | `gpt-4o` |
-
-The key is finding the right balance for your specific use case. Start small and simple, measure performance, then scale up only where needed.
-
-## Deep Dive: Critical Configuration Decisions
-
-### When to Disable Recomputation
-
-LEANN's recomputation feature provides exact distance calculations but can be disabled for extreme QPS requirements:
-
-```bash
---no-recompute  # Disable selective recomputation
-```
-
-**Trade-offs**:
-- **With recomputation** (default): Exact distances, best quality, higher latency, minimal storage (only stores metadata, recomputes embeddings on-demand)
-- **Without recomputation**: Must store full embeddings, significantly higher memory and storage usage (10-100x more), but faster search
-
-**Disable when**:
-- You have abundant storage and memory
-- Need extremely low latency (< 100ms)
-- Running a read-heavy workload where storage cost is acceptable
-
-## Further Reading
-
-- [Lessons Learned Developing LEANN](https://yichuan-w.github.io/blog/lessons_learned_in_dev_leann/)
-- [LEANN Technical Paper](https://arxiv.org/abs/2506.08276)
-- [DiskANN Original Paper](https://papers.nips.cc/paper/2019/file/09853c7fb1d3f8ee67a61b6bf4a7f8e6-Paper.pdf)

docs/features.md

@@ -5,7 +5,7 @@
 - **🔄 Real-time Embeddings** - Eliminate heavy embedding storage with dynamic computation using optimized ZMQ servers and highly optimized search paradigm (overlapping and batching) with highly optimized embedding engine
 - **📈 Scalable Architecture** - Handles millions of documents on consumer hardware; the larger your dataset, the more LEANN can save
 - **🎯 Graph Pruning** - Advanced techniques to minimize the storage overhead of vector search to a limited footprint
-- **🏗️ Pluggable Backends** - HNSW/FAISS (default), with optional DiskANN for large-scale deployments
+- **🏗️ Pluggable Backends** - DiskANN, HNSW/FAISS with unified API
 
 ## 🛠️ Technical Highlights
 - **🔄 Recompute Mode** - Highest accuracy scenarios while eliminating vector storage overhead
@@ -13,7 +13,7 @@
 - **🚀 High-throughput Embedding Pipeline** - Optimized batched processing for maximum efficiency
 - **🎯 Two-level Search** - Novel coarse-to-fine search overlap for accelerated query processing (optional)
 - **💾 Memory-mapped Indices** - Fast startup with raw text mapping to reduce memory overhead
-- **🚀 MLX Support** - Ultra-fast recompute/build with quantized embedding models, accelerating building and search ([minimal example](../examples/mlx_demo.py))
+- **🚀 MLX Support** - Ultra-fast recompute/build with quantized embedding models, accelerating building and search ([minimal example](test/build_mlx_index.py))
 
 ## 🎨 Developer Experience
 

docs/normalized_embeddings.md

@@ -72,4 +72,4 @@ Using the wrong distance metric with normalized embeddings can lead to:
 - **Incorrect ranking** of search results
 - **Suboptimal performance** compared to using the correct metric
 
-For more details on why this happens, see our analysis in the [embedding detection code](../packages/leann-core/src/leann/api.py) which automatically handles normalized embeddings and MIPS distance metric issues.
+For more details on why this happens, see our analysis of [OpenAI embeddings with MIPS](../examples/main_cli_example.py).

docs/roadmap.md

@@ -2,8 +2,8 @@
 
 ## 🎯 Q2 2025
 
-- [X] HNSW backend integration
 - [X] DiskANN backend with MIPS/L2/Cosine support
+- [X] HNSW backend integration
 - [X] Real-time embedding pipeline
 - [X] Memory-efficient graph pruning
 

examples/compare_faiss_vs_leann.py

@@ -62,7 +62,7 @@ def test_faiss_hnsw():
 
     try:
         result = subprocess.run(
-            [sys.executable, "benchmarks/faiss_only.py"],
+            [sys.executable, "examples/faiss_only.py"],
             capture_output=True,
             text=True,
             timeout=300,
@@ -115,7 +115,7 @@ def test_leann_hnsw():
 
     # Load and parse documents
     documents = SimpleDirectoryReader(
-        "data",
+        "examples/data",
         recursive=True,
         encoding="utf-8",
         required_exts=[".pdf", ".txt", ".md"],

examples/document_search.py

@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+"""
+Document search demo with recompute mode
+"""
+
+import shutil
+import time
+from pathlib import Path
+
+# Import backend packages to trigger plugin registration
+try:
+    import leann_backend_diskann  # noqa: F401
+    import leann_backend_hnsw  # noqa: F401
+
+    print("INFO: Backend packages imported successfully.")
+except ImportError as e:
+    print(f"WARNING: Could not import backend packages. Error: {e}")
+
+# Import upper-level API from leann-core
+from leann.api import LeannBuilder, LeannChat, LeannSearcher
+
+
+def load_sample_documents():
+    """Create sample documents for demonstration"""
+    docs = [
+        {
+            "title": "Intro to Python",
+            "content": "Python is a high-level, interpreted language known for simplicity.",
+        },
+        {
+            "title": "ML Basics",
+            "content": "Machine learning builds systems that learn from data.",
+        },
+        {
+            "title": "Data Structures",
+            "content": "Data structures like arrays, lists, and graphs organize data.",
+        },
+    ]
+    return docs
+
+
+def main():
+    print("==========================================================")
+    print("=== Leann Document Search Demo (DiskANN + Recompute) ===")
+    print("==========================================================")
+
+    INDEX_DIR = Path("./test_indices")
+    INDEX_PATH = str(INDEX_DIR / "documents.diskann")
+    BACKEND_TO_TEST = "diskann"
+
+    if INDEX_DIR.exists():
+        print(f"--- Cleaning up old index directory: {INDEX_DIR} ---")
+        shutil.rmtree(INDEX_DIR)
+
+    # --- 1. Build index ---
+    print(f"\n[PHASE 1] Building index using '{BACKEND_TO_TEST}' backend...")
+
+    builder = LeannBuilder(backend_name=BACKEND_TO_TEST, graph_degree=32, complexity=64)
+
+    documents = load_sample_documents()
+    print(f"Loaded {len(documents)} sample documents.")
+    for doc in documents:
+        builder.add_text(doc["content"], metadata={"title": doc["title"]})
+
+    builder.build_index(INDEX_PATH)
+    print("\nIndex built!")
+
+    # --- 2. Basic search demo ---
+    print(f"\n[PHASE 2] Basic search using '{BACKEND_TO_TEST}' backend...")
+    searcher = LeannSearcher(index_path=INDEX_PATH)
+
+    query = "What is machine learning?"
+    print(f"\nQuery: '{query}'")
+
+    print("\n--- Basic search mode (PQ computation) ---")
+    start_time = time.time()
+    results = searcher.search(query, top_k=2)
+    basic_time = time.time() - start_time
+
+    print(f"⏱️  Basic search time: {basic_time:.3f} seconds")
+    print(">>> Basic search results <<<")
+    for i, res in enumerate(results, 1):
+        print(
+            f"  {i}. ID: {res.id}, Score: {res.score:.4f}, Text: '{res.text}', Metadata: {res.metadata}"
+        )
+
+    # --- 3. Recompute search demo ---
+    print("\n[PHASE 3] Recompute search using embedding server...")
+
+    print("\n--- Recompute search mode (get real embeddings via network) ---")
+
+    # Configure recompute parameters
+    recompute_params = {
+        "recompute_beighbor_embeddings": True,  # Enable network recomputation
+        "USE_DEFERRED_FETCH": False,  # Don't use deferred fetch
+        "skip_search_reorder": True,  # Skip search reordering
+        "dedup_node_dis": True,  # Enable node distance deduplication
+        "prune_ratio": 0.1,  # Pruning ratio 10%
+        "batch_recompute": False,  # Don't use batch recomputation
+        "global_pruning": False,  # Don't use global pruning
+        "zmq_port": 5555,  # ZMQ port
+        "embedding_model": "sentence-transformers/all-mpnet-base-v2",
+    }
+
+    print("Recompute parameter configuration:")
+    for key, value in recompute_params.items():
+        print(f"  {key}: {value}")
+
+    print("\n🔄 Executing Recompute search...")
+    try:
+        start_time = time.time()
+        recompute_results = searcher.search(query, top_k=2, **recompute_params)
+        recompute_time = time.time() - start_time
+
+        print(f"⏱️  Recompute search time: {recompute_time:.3f} seconds")
+        print(">>> Recompute search results <<<")
+        for i, res in enumerate(recompute_results, 1):
+            print(
+                f"  {i}. ID: {res.id}, Score: {res.score:.4f}, Text: '{res.text}', Metadata: {res.metadata}"
+            )
+
+        # Compare results
+        print("\n--- Result comparison ---")
+        print(f"Basic search time: {basic_time:.3f} seconds")
+        print(f"Recompute time: {recompute_time:.3f} seconds")
+
+        print("\nBasic search vs Recompute results:")
+        for i in range(min(len(results), len(recompute_results))):
+            basic_score = results[i].score
+            recompute_score = recompute_results[i].score
+            score_diff = abs(basic_score - recompute_score)
+            print(
+                f"  Position {i + 1}: PQ={basic_score:.4f}, Recompute={recompute_score:.4f}, Difference={score_diff:.4f}"
+            )
+
+        if recompute_time > basic_time:
+            print("✅ Recompute mode working correctly (more accurate but slower)")
+        else:
+            print("i️  Recompute time is unusually fast, network recomputation may not be enabled")
+
+    except Exception as e:
+        print(f"❌ Recompute search failed: {e}")
+        print("This usually indicates an embedding server connection issue")
+
+    # --- 4. Chat demo ---
+    print("\n[PHASE 4] Starting chat session...")
+    chat = LeannChat(index_path=INDEX_PATH)
+    chat_response = chat.ask(query)
+    print(f"You: {query}")
+    print(f"Leann: {chat_response}")
+
+    print("\n==========================================================")
+    print("✅ Demo finished successfully!")
+    print("==========================================================")
+
+
+if __name__ == "__main__":
+    main()

examples/email_data/LEANN_email_reader.py

@@ -52,11 +52,6 @@ class EmlxReader(BaseReader):
         docs: list[Document] = []
         max_count = load_kwargs.get("max_count", 1000)
         count = 0
-        total_files = 0
-        successful_files = 0
-        failed_files = 0
-
-        print(f"Starting to process directory: {input_dir}")
 
         # Walk through the directory recursively
         for dirpath, dirnames, filenames in os.walk(input_dir):
@@ -64,12 +59,10 @@ class EmlxReader(BaseReader):
             dirnames[:] = [d for d in dirnames if not d.startswith(".")]
 
             for filename in filenames:
-                # Check if we've reached the max count (skip if max_count == -1)
-                if max_count > 0 and count >= max_count:
+                if count >= max_count:
                     break
 
                 if filename.endswith(".emlx"):
-                    total_files += 1
                     filepath = os.path.join(dirpath, filename)
                     try:
                         # Read the .emlx file
@@ -105,26 +98,17 @@ class EmlxReader(BaseReader):
                                                 and not self.include_html
                                             ):
                                                 continue
-                                            try:
-                                                payload = part.get_payload(decode=True)
-                                                if payload:
-                                                    body += payload.decode("utf-8", errors="ignore")
-                                            except Exception as e:
-                                                print(f"Error decoding payload: {e}")
-                                                continue
+                                            body += part.get_payload(decode=True).decode(
+                                                "utf-8", errors="ignore"
+                                            )
+                                            # break
                                 else:
-                                    try:
-                                        payload = msg.get_payload(decode=True)
-                                        if payload:
-                                            body = payload.decode("utf-8", errors="ignore")
-                                    except Exception as e:
-                                        print(f"Error decoding single part payload: {e}")
-                                        body = ""
+                                    body = msg.get_payload(decode=True).decode(
+                                        "utf-8", errors="ignore"
+                                    )
 
-                                # Only create document if we have some content
-                                if body.strip() or subject != "No Subject":
-                                    # Create document content with metadata embedded in text
-                                    doc_content = f"""
+                                # Create document content with metadata embedded in text
+                                doc_content = f"""
 [File]: {filename}
 [From]: {from_addr}
 [To]: {to_addr}
@@ -134,34 +118,18 @@ class EmlxReader(BaseReader):
 {body}
 """
 
-                                    # No separate metadata - everything is in the text
-                                    doc = Document(text=doc_content, metadata={})
-                                    docs.append(doc)
-                                    count += 1
-                                    successful_files += 1
-
-                                    # Print first few successful files for debugging
-                                    if successful_files <= 3:
-                                        print(
-                                            f"Successfully loaded: {filename} - Subject: {subject[:50]}..."
-                                        )
+                                # No separate metadata - everything is in the text
+                                doc = Document(text=doc_content, metadata={})
+                                docs.append(doc)
+                                count += 1
 
                             except Exception as e:
-                                failed_files += 1
-                                if failed_files <= 5:  # Only print first few errors
-                                    print(f"Error parsing email from {filepath}: {e}")
+                                print(f"Error parsing email from {filepath}: {e}")
                                 continue
 
                     except Exception as e:
-                        failed_files += 1
-                        if failed_files <= 5:  # Only print first few errors
-                            print(f"Error reading file {filepath}: {e}")
+                        print(f"Error reading file {filepath}: {e}")
                         continue
 
-        print("Processing summary:")
-        print(f"  Total .emlx files found: {total_files}")
-        print(f"  Successfully loaded: {successful_files}")
-        print(f"  Failed to load: {failed_files}")
-        print(f"  Final documents: {len(docs)}")
-
+        print(f"Loaded {len(docs)} email documents")
         return docs

examples/faiss_only.py

@@ -65,7 +65,7 @@ def main():
     tracker.checkpoint("After Faiss index creation")
 
     documents = SimpleDirectoryReader(
-        "data",
+        "examples/data",
         recursive=True,
         encoding="utf-8",
         required_exts=[".pdf", ".txt", ".md"],

examples/google_history_reader_leann.py

@@ -0,0 +1,362 @@
+import argparse
+import asyncio
+import os
+
+try:
+    import dotenv
+
+    dotenv.load_dotenv()
+except ModuleNotFoundError:
+    # python-dotenv is not installed; skip loading environment variables
+    dotenv = None
+from pathlib import Path
+
+from leann.api import LeannBuilder, LeannChat
+from llama_index.core.node_parser import SentenceSplitter
+
+# dotenv.load_dotenv()  # handled above if python-dotenv is available
+
+# Default Chrome profile path
+DEFAULT_CHROME_PROFILE = os.path.expanduser("~/Library/Application Support/Google/Chrome/Default")
+
+
+def create_leann_index_from_multiple_chrome_profiles(
+    profile_dirs: list[Path],
+    index_path: str = "chrome_history_index.leann",
+    max_count: int = -1,
+    embedding_model: str = "facebook/contriever",
+    embedding_mode: str = "sentence-transformers",
+):
+    """
+    Create LEANN index from multiple Chrome profile data sources.
+
+    Args:
+        profile_dirs: List of Path objects pointing to Chrome profile directories
+        index_path: Path to save the LEANN index
+        max_count: Maximum number of history entries to process per profile
+        embedding_model: The embedding model to use
+        embedding_mode: The embedding backend mode
+    """
+    print("Creating LEANN index from multiple Chrome profile data sources...")
+
+    # Load documents using ChromeHistoryReader from history_data
+    from history_data.history import ChromeHistoryReader
+
+    reader = ChromeHistoryReader()
+
+    INDEX_DIR = Path(index_path).parent
+
+    if not INDEX_DIR.exists():
+        print("--- Index directory not found, building new index ---")
+        all_documents = []
+        total_processed = 0
+
+        # Process each Chrome profile directory
+        for i, profile_dir in enumerate(profile_dirs):
+            print(f"\nProcessing Chrome profile {i + 1}/{len(profile_dirs)}: {profile_dir}")
+
+            try:
+                documents = reader.load_data(
+                    chrome_profile_path=str(profile_dir), max_count=max_count
+                )
+                if documents:
+                    print(f"Loaded {len(documents)} history documents from {profile_dir}")
+                    all_documents.extend(documents)
+                    total_processed += len(documents)
+
+                    # Check if we've reached the max count
+                    if max_count > 0 and total_processed >= max_count:
+                        print(f"Reached max count of {max_count} documents")
+                        break
+                else:
+                    print(f"No documents loaded from {profile_dir}")
+            except Exception as e:
+                print(f"Error processing {profile_dir}: {e}")
+                continue
+
+        if not all_documents:
+            print("No documents loaded from any source. Exiting.")
+            # highlight info that you need to close all chrome browser before running this script and high light the instruction!!
+            print(
+                "\033[91mYou need to close or quit all chrome browser before running this script\033[0m"
+            )
+            return None
+
+        print(
+            f"\nTotal loaded {len(all_documents)} history documents from {len(profile_dirs)} profiles"
+        )
+
+        # Create text splitter with 256 chunk size
+        text_splitter = SentenceSplitter(chunk_size=256, chunk_overlap=128)
+
+        # Convert Documents to text strings and chunk them
+        all_texts = []
+        for doc in all_documents:
+            # Split the document into chunks
+            nodes = text_splitter.get_nodes_from_documents([doc])
+            for node in nodes:
+                text = node.get_content()
+                # text = '[Title] ' + doc.metadata["title"] + '\n' + text
+                all_texts.append(text)
+
+        print(f"Created {len(all_texts)} text chunks from {len(all_documents)} documents")
+
+        # Create LEANN index directory
+        print("--- Index directory not found, building new index ---")
+        INDEX_DIR.mkdir(exist_ok=True)
+
+        print("--- Building new LEANN index ---")
+
+        print("\n[PHASE 1] Building Leann index...")
+
+        # Use HNSW backend for better macOS compatibility
+        # LeannBuilder will automatically detect normalized embeddings and set appropriate distance metric
+        builder = LeannBuilder(
+            backend_name="hnsw",
+            embedding_model=embedding_model,
+            embedding_mode=embedding_mode,
+            graph_degree=32,
+            complexity=64,
+            is_compact=True,
+            is_recompute=True,
+            num_threads=1,  # Force single-threaded mode
+        )
+
+        print(f"Adding {len(all_texts)} history chunks to index...")
+        for chunk_text in all_texts:
+            builder.add_text(chunk_text)
+
+        builder.build_index(index_path)
+        print(f"\nLEANN index built at {index_path}!")
+    else:
+        print(f"--- Using existing index at {INDEX_DIR} ---")
+
+    return index_path
+
+
+def create_leann_index(
+    profile_path: str | None = None,
+    index_path: str = "chrome_history_index.leann",
+    max_count: int = 1000,
+    embedding_model: str = "facebook/contriever",
+    embedding_mode: str = "sentence-transformers",
+):
+    """
+    Create LEANN index from Chrome history data.
+
+    Args:
+        profile_path: Path to the Chrome profile directory (optional, uses default if None)
+        index_path: Path to save the LEANN index
+        max_count: Maximum number of history entries to process
+        embedding_model: The embedding model to use
+        embedding_mode: The embedding backend mode
+    """
+    print("Creating LEANN index from Chrome history data...")
+    INDEX_DIR = Path(index_path).parent
+
+    if not INDEX_DIR.exists():
+        print("--- Index directory not found, building new index ---")
+        INDEX_DIR.mkdir(exist_ok=True)
+
+        print("--- Building new LEANN index ---")
+
+        print("\n[PHASE 1] Building Leann index...")
+
+        # Load documents using ChromeHistoryReader from history_data
+        from history_data.history import ChromeHistoryReader
+
+        reader = ChromeHistoryReader()
+
+        documents = reader.load_data(chrome_profile_path=profile_path, max_count=max_count)
+
+        if not documents:
+            print("No documents loaded. Exiting.")
+            return None
+
+        print(f"Loaded {len(documents)} history documents")
+
+        # Create text splitter with 256 chunk size
+        text_splitter = SentenceSplitter(chunk_size=256, chunk_overlap=25)
+
+        # Convert Documents to text strings and chunk them
+        all_texts = []
+        for doc in documents:
+            # Split the document into chunks
+            nodes = text_splitter.get_nodes_from_documents([doc])
+            for node in nodes:
+                all_texts.append(node.get_content())
+
+        print(f"Created {len(all_texts)} text chunks from {len(documents)} documents")
+
+        # Create LEANN index directory
+        print("--- Index directory not found, building new index ---")
+        INDEX_DIR.mkdir(exist_ok=True)
+
+        print("--- Building new LEANN index ---")
+
+        print("\n[PHASE 1] Building Leann index...")
+
+        # Use HNSW backend for better macOS compatibility
+        # LeannBuilder will automatically detect normalized embeddings and set appropriate distance metric
+        builder = LeannBuilder(
+            backend_name="hnsw",
+            embedding_model=embedding_model,
+            embedding_mode=embedding_mode,
+            graph_degree=32,
+            complexity=64,
+            is_compact=True,
+            is_recompute=True,
+            num_threads=1,  # Force single-threaded mode
+        )
+
+        print(f"Adding {len(all_texts)} history chunks to index...")
+        for chunk_text in all_texts:
+            builder.add_text(chunk_text)
+
+        builder.build_index(index_path)
+        print(f"\nLEANN index built at {index_path}!")
+    else:
+        print(f"--- Using existing index at {INDEX_DIR} ---")
+
+    return index_path
+
+
+async def query_leann_index(index_path: str, query: str):
+    """
+    Query the LEANN index.
+
+    Args:
+        index_path: Path to the LEANN index
+        query: The query string
+    """
+    print("\n[PHASE 2] Starting Leann chat session...")
+    chat = LeannChat(index_path=index_path)
+
+    print(f"You: {query}")
+    chat_response = chat.ask(
+        query,
+        top_k=10,
+        recompute_beighbor_embeddings=True,
+        complexity=32,
+        beam_width=1,
+        llm_config={
+            "type": "openai",
+            "model": "gpt-4o",
+            "api_key": os.getenv("OPENAI_API_KEY"),
+        },
+        llm_kwargs={"temperature": 0.0, "max_tokens": 1000},
+    )
+
+    print(f"Leann chat response: \033[36m{chat_response}\033[0m")
+
+
+async def main():
+    # Parse command line arguments
+    parser = argparse.ArgumentParser(
+        description="LEANN Chrome History Reader - Create and query browser history index"
+    )
+    parser.add_argument(
+        "--chrome-profile",
+        type=str,
+        default=DEFAULT_CHROME_PROFILE,
+        help=f"Path to Chrome profile directory (default: {DEFAULT_CHROME_PROFILE}), usually you dont need to change this",
+    )
+    parser.add_argument(
+        "--index-dir",
+        type=str,
+        default="./google_history_index",
+        help="Directory to store the LEANN index (default: ./chrome_history_index_leann_test)",
+    )
+    parser.add_argument(
+        "--max-entries",
+        type=int,
+        default=1000,
+        help="Maximum number of history entries to process (default: 1000)",
+    )
+    parser.add_argument(
+        "--query",
+        type=str,
+        default=None,
+        help="Single query to run (default: runs example queries)",
+    )
+    parser.add_argument(
+        "--auto-find-profiles",
+        action="store_true",
+        default=True,
+        help="Automatically find all Chrome profiles (default: True)",
+    )
+    parser.add_argument(
+        "--embedding-model",
+        type=str,
+        default="facebook/contriever",
+        help="The embedding model to use (e.g., 'facebook/contriever', 'text-embedding-3-small')",
+    )
+    parser.add_argument(
+        "--embedding-mode",
+        type=str,
+        default="sentence-transformers",
+        choices=["sentence-transformers", "openai", "mlx"],
+        help="The embedding backend mode",
+    )
+    parser.add_argument(
+        "--use-existing-index",
+        action="store_true",
+        help="Use existing index without rebuilding",
+    )
+
+    args = parser.parse_args()
+
+    INDEX_DIR = Path(args.index_dir)
+    INDEX_PATH = str(INDEX_DIR / "chrome_history.leann")
+
+    print(f"Using Chrome profile: {args.chrome_profile}")
+    print(f"Index directory: {INDEX_DIR}")
+    print(f"Max entries: {args.max_entries}")
+
+    if args.use_existing_index:
+        # Use existing index without rebuilding
+        if not Path(INDEX_PATH).exists():
+            print(f"Error: Index file not found at {INDEX_PATH}")
+            return
+        print(f"Using existing index at {INDEX_PATH}")
+        index_path = INDEX_PATH
+    else:
+        # Find Chrome profile directories
+        from history_data.history import ChromeHistoryReader
+
+        if args.auto_find_profiles:
+            profile_dirs = ChromeHistoryReader.find_chrome_profiles()
+            if not profile_dirs:
+                print("No Chrome profiles found automatically. Exiting.")
+                return
+        else:
+            # Use single specified profile
+            profile_path = Path(args.chrome_profile)
+            if not profile_path.exists():
+                print(f"Chrome profile not found: {profile_path}")
+                return
+            profile_dirs = [profile_path]
+
+        # Create or load the LEANN index from all sources
+        index_path = create_leann_index_from_multiple_chrome_profiles(
+            profile_dirs, INDEX_PATH, args.max_entries, args.embedding_model, args.embedding_mode
+        )
+
+    if index_path:
+        if args.query:
+            # Run single query
+            await query_leann_index(index_path, args.query)
+        else:
+            # Example queries
+            queries = [
+                "What websites did I visit about machine learning?",
+                "Find my search history about programming",
+            ]
+
+            for query in queries:
+                print("\n" + "=" * 60)
+                await query_leann_index(index_path, query)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/history_data/history.py

@@ -97,11 +97,6 @@ class ChromeHistoryReader(BaseReader):
 
         except Exception as e:
             print(f"Error reading Chrome history: {e}")
-            # add you may need to close your browser to make the database file available
-            # also highlight in red
-            print(
-                "\033[91mYou may need to close your browser to make the database file available\033[0m"
-            )
             return docs
 
         return docs

examples/history_data/wechat_history.py

@@ -411,8 +411,8 @@ Messages ({len(messages)} messages, {message_group["total_length"]} chars):
         wechat_export_dir = load_kwargs.get("wechat_export_dir", None)
         include_non_text = load_kwargs.get("include_non_text", False)
         concatenate_messages = load_kwargs.get("concatenate_messages", False)
-        max_length = load_kwargs.get("max_length", 1000)
-        time_window_minutes = load_kwargs.get("time_window_minutes", 30)
+        load_kwargs.get("max_length", 1000)
+        load_kwargs.get("time_window_minutes", 30)
 
         # Default WeChat export path
         if wechat_export_dir is None:
@@ -460,9 +460,9 @@ Messages ({len(messages)} messages, {message_group["total_length"]} chars):
                         # Concatenate messages based on rules
                         message_groups = self._concatenate_messages(
                             readable_messages,
-                            max_length=max_length,
-                            time_window_minutes=time_window_minutes,
-                            overlap_messages=0,  # No overlap between groups
+                            max_length=-1,
+                            time_window_minutes=-1,
+                            overlap_messages=0,  # Keep 2 messages overlap between groups
                         )
 
                         # Create documents from concatenated groups
@@ -532,9 +532,7 @@ Message: {readable_text if readable_text else message_text}
 """
 
                             # Create document with embedded metadata
-                            doc = Document(
-                                text=doc_content, metadata={"contact_name": contact_name}
-                            )
+                            doc = Document(text=doc_content, metadata={})
                             docs.append(doc)
                             count += 1
 
@@ -562,8 +560,8 @@ Message: {readable_text if readable_text else message_text}
 
         # Look for common export directory names
         possible_dirs = [
+            Path("./wechat_export_test"),
             Path("./wechat_export"),
-            Path("./wechat_export_direct"),
             Path("./wechat_chat_history"),
             Path("./chat_export"),
         ]

examples/mail_reader_leann.py

@@ -0,0 +1,342 @@
+import argparse
+import asyncio
+import os
+import sys
+from pathlib import Path
+
+import dotenv
+
+# Add the project root to Python path so we can import from examples
+project_root = Path(__file__).parent.parent
+sys.path.insert(0, str(project_root))
+
+from leann.api import LeannBuilder, LeannChat
+from llama_index.core.node_parser import SentenceSplitter
+
+dotenv.load_dotenv()
+
+
+# Auto-detect user's mail path
+def get_mail_path():
+    """Get the mail path for the current user"""
+    home_dir = os.path.expanduser("~")
+    return os.path.join(home_dir, "Library", "Mail")
+
+
+# Default mail path for macOS
+DEFAULT_MAIL_PATH = "/Users/yichuan/Library/Mail/V10/0FCA0879-FD8C-4B7E-83BF-FDDA930791C5/[Gmail].mbox/All Mail.mbox/78BA5BE1-8819-4F9A-9613-EB63772F1DD0/Data"
+
+
+def create_leann_index_from_multiple_sources(
+    messages_dirs: list[Path],
+    index_path: str = "mail_index.leann",
+    max_count: int = -1,
+    include_html: bool = False,
+    embedding_model: str = "facebook/contriever",
+):
+    """
+    Create LEANN index from multiple mail data sources.
+
+    Args:
+        messages_dirs: List of Path objects pointing to Messages directories
+        index_path: Path to save the LEANN index
+        max_count: Maximum number of emails to process per directory
+        include_html: Whether to include HTML content in email processing
+    """
+    print("Creating LEANN index from multiple mail data sources...")
+
+    # Load documents using EmlxReader from LEANN_email_reader
+    from examples.email_data.LEANN_email_reader import EmlxReader
+
+    reader = EmlxReader(include_html=include_html)
+    # from email_data.email import EmlxMboxReader
+    # from pathlib import Path
+    # reader = EmlxMboxReader()
+    INDEX_DIR = Path(index_path).parent
+
+    if not INDEX_DIR.exists():
+        print("--- Index directory not found, building new index ---")
+        all_documents = []
+        total_processed = 0
+
+        # Process each Messages directory
+        for i, messages_dir in enumerate(messages_dirs):
+            print(f"\nProcessing Messages directory {i + 1}/{len(messages_dirs)}: {messages_dir}")
+
+            try:
+                documents = reader.load_data(messages_dir)
+                if documents:
+                    print(f"Loaded {len(documents)} email documents from {messages_dir}")
+                    all_documents.extend(documents)
+                    total_processed += len(documents)
+
+                    # Check if we've reached the max count
+                    if max_count > 0 and total_processed >= max_count:
+                        print(f"Reached max count of {max_count} documents")
+                        break
+                else:
+                    print(f"No documents loaded from {messages_dir}")
+            except Exception as e:
+                print(f"Error processing {messages_dir}: {e}")
+                continue
+
+        if not all_documents:
+            print("No documents loaded from any source. Exiting.")
+            return None
+
+        print(
+            f"\nTotal loaded {len(all_documents)} email documents from {len(messages_dirs)} directories and starting to split them into chunks"
+        )
+
+        # Create text splitter with 256 chunk size
+        text_splitter = SentenceSplitter(chunk_size=256, chunk_overlap=25)
+
+        # Convert Documents to text strings and chunk them
+        all_texts = []
+        for doc in all_documents:
+            # Split the document into chunks
+            nodes = text_splitter.get_nodes_from_documents([doc])
+            for node in nodes:
+                text = node.get_content()
+                # text = '[subject] ' + doc.metadata["subject"] + '\n' + text
+                all_texts.append(text)
+
+        print(
+            f"Finished splitting {len(all_documents)} documents into {len(all_texts)} text chunks"
+        )
+
+        # Create LEANN index directory
+
+        print("--- Index directory not found, building new index ---")
+        INDEX_DIR.mkdir(exist_ok=True)
+
+        print("--- Building new LEANN index ---")
+
+        print("\n[PHASE 1] Building Leann index...")
+
+        # Use HNSW backend for better macOS compatibility
+        builder = LeannBuilder(
+            backend_name="hnsw",
+            embedding_model=embedding_model,
+            graph_degree=32,
+            complexity=64,
+            is_compact=True,
+            is_recompute=True,
+            num_threads=1,  # Force single-threaded mode
+        )
+
+        print(f"Adding {len(all_texts)} email chunks to index...")
+        for chunk_text in all_texts:
+            builder.add_text(chunk_text)
+
+        builder.build_index(index_path)
+        print(f"\nLEANN index built at {index_path}!")
+    else:
+        print(f"--- Using existing index at {INDEX_DIR} ---")
+
+    return index_path
+
+
+def create_leann_index(
+    mail_path: str,
+    index_path: str = "mail_index.leann",
+    max_count: int = 1000,
+    include_html: bool = False,
+    embedding_model: str = "facebook/contriever",
+):
+    """
+    Create LEANN index from mail data.
+
+    Args:
+        mail_path: Path to the mail directory
+        index_path: Path to save the LEANN index
+        max_count: Maximum number of emails to process
+        include_html: Whether to include HTML content in email processing
+    """
+    print("Creating LEANN index from mail data...")
+    INDEX_DIR = Path(index_path).parent
+
+    if not INDEX_DIR.exists():
+        print("--- Index directory not found, building new index ---")
+        INDEX_DIR.mkdir(exist_ok=True)
+
+        print("--- Building new LEANN index ---")
+
+        print("\n[PHASE 1] Building Leann index...")
+
+        # Load documents using EmlxReader from LEANN_email_reader
+        from examples.email_data.LEANN_email_reader import EmlxReader
+
+        reader = EmlxReader(include_html=include_html)
+        # from email_data.email import EmlxMboxReader
+        # from pathlib import Path
+        # reader = EmlxMboxReader()
+        documents = reader.load_data(Path(mail_path))
+
+        if not documents:
+            print("No documents loaded. Exiting.")
+            return None
+
+        print(f"Loaded {len(documents)} email documents")
+
+        # Create text splitter with 256 chunk size
+        text_splitter = SentenceSplitter(chunk_size=256, chunk_overlap=128)
+
+        # Convert Documents to text strings and chunk them
+        all_texts = []
+        for doc in documents:
+            # Split the document into chunks
+            nodes = text_splitter.get_nodes_from_documents([doc])
+            for node in nodes:
+                all_texts.append(node.get_content())
+
+        print(f"Created {len(all_texts)} text chunks from {len(documents)} documents")
+
+        # Create LEANN index directory
+
+        print("--- Index directory not found, building new index ---")
+        INDEX_DIR.mkdir(exist_ok=True)
+
+        print("--- Building new LEANN index ---")
+
+        print("\n[PHASE 1] Building Leann index...")
+
+        # Use HNSW backend for better macOS compatibility
+        builder = LeannBuilder(
+            backend_name="hnsw",
+            embedding_model=embedding_model,
+            graph_degree=32,
+            complexity=64,
+            is_compact=True,
+            is_recompute=True,
+            num_threads=1,  # Force single-threaded mode
+        )
+
+        print(f"Adding {len(all_texts)} email chunks to index...")
+        for chunk_text in all_texts:
+            builder.add_text(chunk_text)
+
+        builder.build_index(index_path)
+        print(f"\nLEANN index built at {index_path}!")
+    else:
+        print(f"--- Using existing index at {INDEX_DIR} ---")
+
+    return index_path
+
+
+async def query_leann_index(index_path: str, query: str):
+    """
+    Query the LEANN index.
+
+    Args:
+        index_path: Path to the LEANN index
+        query: The query string
+    """
+    print("\n[PHASE 2] Starting Leann chat session...")
+    chat = LeannChat(index_path=index_path, llm_config={"type": "openai", "model": "gpt-4o"})
+
+    print(f"You: {query}")
+    import time
+
+    time.time()
+    chat_response = chat.ask(
+        query,
+        top_k=20,
+        recompute_beighbor_embeddings=True,
+        complexity=32,
+        beam_width=1,
+    )
+    time.time()
+    # print(f"Time taken: {end_time - start_time} seconds")
+    # highlight the answer
+    print(f"Leann chat response: \033[36m{chat_response}\033[0m")
+
+
+async def main():
+    # Parse command line arguments
+    parser = argparse.ArgumentParser(description="LEANN Mail Reader - Create and query email index")
+    # Remove --mail-path argument and auto-detect all Messages directories
+    # Remove DEFAULT_MAIL_PATH
+    parser.add_argument(
+        "--index-dir",
+        type=str,
+        default="./mail_index",
+        help="Directory to store the LEANN index (default: ./mail_index_leann_raw_text_all_dicts)",
+    )
+    parser.add_argument(
+        "--max-emails",
+        type=int,
+        default=1000,
+        help="Maximum number of emails to process (-1 means all)",
+    )
+    parser.add_argument(
+        "--query",
+        type=str,
+        default="Give me some funny advertisement about apple or other companies",
+        help="Single query to run (default: runs example queries)",
+    )
+    parser.add_argument(
+        "--include-html",
+        action="store_true",
+        default=False,
+        help="Include HTML content in email processing (default: False)",
+    )
+    parser.add_argument(
+        "--embedding-model",
+        type=str,
+        default="facebook/contriever",
+        help="Embedding model to use (default: facebook/contriever)",
+    )
+
+    args = parser.parse_args()
+
+    print(f"args: {args}")
+
+    # Automatically find all Messages directories under the current user's Mail directory
+    from examples.email_data.LEANN_email_reader import find_all_messages_directories
+
+    mail_path = get_mail_path()
+    print(f"Searching for email data in: {mail_path}")
+    messages_dirs = find_all_messages_directories(mail_path)
+    # messages_dirs = find_all_messages_directories(DEFAULT_MAIL_PATH)
+    # messages_dirs = [DEFAULT_MAIL_PATH]
+    # messages_dirs = messages_dirs[:1]
+
+    print("len(messages_dirs): ", len(messages_dirs))
+
+    if not messages_dirs:
+        print("No Messages directories found. Exiting.")
+        return
+
+    INDEX_DIR = Path(args.index_dir)
+    INDEX_PATH = str(INDEX_DIR / "mail_documents.leann")
+    print(f"Index directory: {INDEX_DIR}")
+    print(f"Found {len(messages_dirs)} Messages directories.")
+
+    # Create or load the LEANN index from all sources
+    index_path = create_leann_index_from_multiple_sources(
+        messages_dirs,
+        INDEX_PATH,
+        args.max_emails,
+        args.include_html,
+        args.embedding_model,
+    )
+
+    if index_path:
+        if args.query:
+            # Run single query
+            await query_leann_index(index_path, args.query)
+        else:
+            # Example queries
+            queries = [
+                "Hows Berkeley Graduate Student Instructor",
+                "how's the icloud related advertisement saying",
+                "Whats the number of class recommend to take per semester for incoming EECS students",
+            ]
+            for query in queries:
+                print("\n" + "=" * 60)
+                await query_leann_index(index_path, query)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/mail_reader_llamaindex.py

@@ -0,0 +1,135 @@
+import argparse
+import os
+import sys
+from pathlib import Path
+
+# Add the project root to Python path so we can import from examples
+project_root = Path(__file__).parent.parent
+sys.path.insert(0, str(project_root))
+
+import torch
+from llama_index.core import StorageContext, VectorStoreIndex
+from llama_index.core.node_parser import SentenceSplitter
+
+# --- EMBEDDING MODEL ---
+from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+
+# --- END EMBEDDING MODEL ---
+# Import EmlxReader from the new module
+from examples.email_data.LEANN_email_reader import EmlxReader
+
+
+def create_and_save_index(
+    mail_path: str,
+    save_dir: str = "mail_index_embedded",
+    max_count: int = 1000,
+    include_html: bool = False,
+):
+    print("Creating index from mail data with embedded metadata...")
+    documents = EmlxReader(include_html=include_html).load_data(mail_path, max_count=max_count)
+    if not documents:
+        print("No documents loaded. Exiting.")
+        return None
+    text_splitter = SentenceSplitter(chunk_size=256, chunk_overlap=25)
+    # Use facebook/contriever as the embedder
+    embed_model = HuggingFaceEmbedding(model_name="facebook/contriever")
+    # set on device
+
+    if torch.cuda.is_available():
+        embed_model._model.to("cuda")
+    # set mps
+    elif torch.backends.mps.is_available():
+        embed_model._model.to("mps")
+    else:
+        embed_model._model.to("cpu")
+    index = VectorStoreIndex.from_documents(
+        documents, transformations=[text_splitter], embed_model=embed_model
+    )
+    os.makedirs(save_dir, exist_ok=True)
+    index.storage_context.persist(persist_dir=save_dir)
+    print(f"Index saved to {save_dir}")
+    return index
+
+
+def load_index(save_dir: str = "mail_index_embedded"):
+    try:
+        storage_context = StorageContext.from_defaults(persist_dir=save_dir)
+        index = VectorStoreIndex.from_vector_store(
+            storage_context.vector_store, storage_context=storage_context
+        )
+        print(f"Index loaded from {save_dir}")
+        return index
+    except Exception as e:
+        print(f"Error loading index: {e}")
+        return None
+
+
+def query_index(index, query: str):
+    if index is None:
+        print("No index available for querying.")
+        return
+    query_engine = index.as_query_engine()
+    response = query_engine.query(query)
+    print(f"Query: {query}")
+    print(f"Response: {response}")
+
+
+def main():
+    # Parse command line arguments
+    parser = argparse.ArgumentParser(
+        description="LlamaIndex Mail Reader - Create and query email index"
+    )
+    parser.add_argument(
+        "--mail-path",
+        type=str,
+        default="/Users/yichuan/Library/Mail/V10/0FCA0879-FD8C-4B7E-83BF-FDDA930791C5/[Gmail].mbox/All Mail.mbox/78BA5BE1-8819-4F9A-9613-EB63772F1DD0/Data/9/Messages",
+        help="Path to mail data directory",
+    )
+    parser.add_argument(
+        "--save-dir",
+        type=str,
+        default="mail_index_embedded",
+        help="Directory to store the index (default: mail_index_embedded)",
+    )
+    parser.add_argument(
+        "--max-emails",
+        type=int,
+        default=10000,
+        help="Maximum number of emails to process",
+    )
+    parser.add_argument(
+        "--include-html",
+        action="store_true",
+        default=False,
+        help="Include HTML content in email processing (default: False)",
+    )
+
+    args = parser.parse_args()
+
+    mail_path = args.mail_path
+    save_dir = args.save_dir
+
+    if os.path.exists(save_dir) and os.path.exists(os.path.join(save_dir, "vector_store.json")):
+        print("Loading existing index...")
+        index = load_index(save_dir)
+    else:
+        print("Creating new index...")
+        index = create_and_save_index(
+            mail_path,
+            save_dir,
+            max_count=args.max_emails,
+            include_html=args.include_html,
+        )
+    if index:
+        queries = [
+            "Hows Berkeley Graduate Student Instructor",
+            "how's the icloud related advertisement saying",
+            "Whats the number of class recommend to take per semester for incoming EECS students",
+        ]
+        for query in queries:
+            print("\n" + "=" * 50)
+            query_index(index, query)
+
+
+if __name__ == "__main__":
+    main()

examples/main_cli_example.py

@@ -0,0 +1,136 @@
+import argparse
+import asyncio
+from pathlib import Path
+
+import dotenv
+from leann.api import LeannBuilder, LeannChat
+from llama_index.core import SimpleDirectoryReader
+from llama_index.core.node_parser import SentenceSplitter
+
+dotenv.load_dotenv()
+
+
+async def main(args):
+    INDEX_DIR = Path(args.index_dir)
+    INDEX_PATH = str(INDEX_DIR / "pdf_documents.leann")
+
+    if not INDEX_DIR.exists():
+        node_parser = SentenceSplitter(
+            chunk_size=256, chunk_overlap=128, separator=" ", paragraph_separator="\n\n"
+        )
+
+        print("Loading documents...")
+        documents = SimpleDirectoryReader(
+            args.data_dir,
+            recursive=True,
+            encoding="utf-8",
+            required_exts=[".pdf", ".txt", ".md"],
+        ).load_data(show_progress=True)
+        print("Documents loaded.")
+        all_texts = []
+        for doc in documents:
+            nodes = node_parser.get_nodes_from_documents([doc])
+            if nodes:
+                all_texts.extend(node.get_content() for node in nodes)
+
+        print("--- Index directory not found, building new index ---")
+
+        print("\n[PHASE 1] Building Leann index...")
+
+        # LeannBuilder now automatically detects normalized embeddings and sets appropriate distance metric
+        print(f"Using {args.embedding_model} with {args.embedding_mode} mode")
+
+        # Use HNSW backend for better macOS compatibility
+        builder = LeannBuilder(
+            backend_name="hnsw",
+            embedding_model=args.embedding_model,
+            embedding_mode=args.embedding_mode,
+            # distance_metric is automatically set based on embedding model
+            graph_degree=32,
+            complexity=64,
+            is_compact=True,
+            is_recompute=True,
+            num_threads=1,  # Force single-threaded mode
+        )
+
+        print(f"Loaded {len(all_texts)} text chunks from documents.")
+        for chunk_text in all_texts:
+            builder.add_text(chunk_text)
+
+        builder.build_index(INDEX_PATH)
+        print(f"\nLeann index built at {INDEX_PATH}!")
+    else:
+        print(f"--- Using existing index at {INDEX_DIR} ---")
+
+    print("\n[PHASE 2] Starting Leann chat session...")
+
+    llm_config = {"type": "hf", "model": "Qwen/Qwen3-4B"}
+    llm_config = {"type": "ollama", "model": "qwen3:8b"}
+    llm_config = {"type": "openai", "model": "gpt-4o"}
+
+    chat = LeannChat(index_path=INDEX_PATH, llm_config=llm_config)
+    # query = (
+    #     "什么是盘古大模型以及盘古开发过程中遇到了什么阴暗面,任务令一般在什么城市颁发"
+    # )
+    query = args.query
+
+    print(f"You: {query}")
+    chat_response = chat.ask(query, top_k=20, recompute_embeddings=True, complexity=32)
+    print(f"Leann chat response: \033[36m{chat_response}\033[0m")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Run Leann Chat with various LLM backends.")
+    parser.add_argument(
+        "--llm",
+        type=str,
+        default="hf",
+        choices=["simulated", "ollama", "hf", "openai"],
+        help="The LLM backend to use.",
+    )
+    parser.add_argument(
+        "--model",
+        type=str,
+        default="Qwen/Qwen3-0.6B",
+        help="The model name to use (e.g., 'llama3:8b' for ollama, 'deepseek-ai/deepseek-llm-7b-chat' for hf, 'gpt-4o' for openai).",
+    )
+    parser.add_argument(
+        "--embedding-model",
+        type=str,
+        default="facebook/contriever",
+        help="The embedding model to use (e.g., 'facebook/contriever', 'text-embedding-3-small').",
+    )
+    parser.add_argument(
+        "--embedding-mode",
+        type=str,
+        default="sentence-transformers",
+        choices=["sentence-transformers", "openai", "mlx"],
+        help="The embedding backend mode.",
+    )
+    parser.add_argument(
+        "--host",
+        type=str,
+        default="http://localhost:11434",
+        help="The host for the Ollama API.",
+    )
+    parser.add_argument(
+        "--index-dir",
+        type=str,
+        default="./test_doc_files",
+        help="Directory where the Leann index will be stored.",
+    )
+    parser.add_argument(
+        "--data-dir",
+        type=str,
+        default="examples/data",
+        help="Directory containing documents to index (PDF, TXT, MD files).",
+    )
+    parser.add_argument(
+        "--query",
+        type=str,
+        default="Based on the paper, what are the main techniques LEANN explores to reduce the storage overhead and DLPM explore to achieve Fairness and Efiiciency trade-off?",
+        help="The query to ask the Leann chat system.",
+    )
+    args = parser.parse_args()
+
+    asyncio.run(main(args))

examples/multi_vector_aggregator.py

@@ -0,0 +1,360 @@
+#!/usr/bin/env python3
+"""
+Multi-Vector Aggregator for Fat Embeddings
+==========================================
+
+This module implements aggregation strategies for multi-vector embeddings,
+similar to ColPali's approach where multiple patch vectors represent a single document.
+
+Key features:
+- MaxSim aggregation (take maximum similarity across patches)
+- Voting-based aggregation (count patch matches)
+- Weighted aggregation (attention-score weighted)
+- Spatial clustering of matching patches
+- Document-level result consolidation
+"""
+
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import Any
+
+import numpy as np
+
+
+@dataclass
+class PatchResult:
+    """Represents a single patch search result."""
+
+    patch_id: int
+    image_name: str
+    image_path: str
+    coordinates: tuple[int, int, int, int]  # (x1, y1, x2, y2)
+    score: float
+    attention_score: float
+    scale: float
+    metadata: dict[str, Any]
+
+
+@dataclass
+class AggregatedResult:
+    """Represents an aggregated document-level result."""
+
+    image_name: str
+    image_path: str
+    doc_score: float
+    patch_count: int
+    best_patch: PatchResult
+    all_patches: list[PatchResult]
+    aggregation_method: str
+    spatial_clusters: list[list[PatchResult]] | None = None
+
+
+class MultiVectorAggregator:
+    """
+    Aggregates multiple patch-level results into document-level results.
+    """
+
+    def __init__(
+        self,
+        aggregation_method: str = "maxsim",
+        spatial_clustering: bool = True,
+        cluster_distance_threshold: float = 100.0,
+    ):
+        """
+        Initialize the aggregator.
+
+        Args:
+            aggregation_method: "maxsim", "voting", "weighted", or "mean"
+            spatial_clustering: Whether to cluster spatially close patches
+            cluster_distance_threshold: Distance threshold for spatial clustering
+        """
+        self.aggregation_method = aggregation_method
+        self.spatial_clustering = spatial_clustering
+        self.cluster_distance_threshold = cluster_distance_threshold
+
+    def aggregate_results(
+        self, search_results: list[dict[str, Any]], top_k: int = 10
+    ) -> list[AggregatedResult]:
+        """
+        Aggregate patch-level search results into document-level results.
+
+        Args:
+            search_results: List of search results from LeannSearcher
+            top_k: Number of top documents to return
+
+        Returns:
+            List of aggregated document results
+        """
+        # Group results by image
+        image_groups = defaultdict(list)
+
+        for result in search_results:
+            metadata = result.metadata
+            if "image_name" in metadata and "patch_id" in metadata:
+                patch_result = PatchResult(
+                    patch_id=metadata["patch_id"],
+                    image_name=metadata["image_name"],
+                    image_path=metadata["image_path"],
+                    coordinates=tuple(metadata["coordinates"]),
+                    score=result.score,
+                    attention_score=metadata.get("attention_score", 0.0),
+                    scale=metadata.get("scale", 1.0),
+                    metadata=metadata,
+                )
+                image_groups[metadata["image_name"]].append(patch_result)
+
+        # Aggregate each image group
+        aggregated_results = []
+        for image_name, patches in image_groups.items():
+            if len(patches) == 0:
+                continue
+
+            agg_result = self._aggregate_image_patches(image_name, patches)
+            aggregated_results.append(agg_result)
+
+        # Sort by aggregated score and return top-k
+        aggregated_results.sort(key=lambda x: x.doc_score, reverse=True)
+        return aggregated_results[:top_k]
+
+    def _aggregate_image_patches(
+        self, image_name: str, patches: list[PatchResult]
+    ) -> AggregatedResult:
+        """Aggregate patches for a single image."""
+
+        if self.aggregation_method == "maxsim":
+            doc_score = max(patch.score for patch in patches)
+            best_patch = max(patches, key=lambda p: p.score)
+
+        elif self.aggregation_method == "voting":
+            # Count patches above threshold
+            threshold = np.percentile([p.score for p in patches], 75)
+            doc_score = sum(1 for patch in patches if patch.score >= threshold)
+            best_patch = max(patches, key=lambda p: p.score)
+
+        elif self.aggregation_method == "weighted":
+            # Weight by attention scores
+            total_weighted_score = sum(p.score * p.attention_score for p in patches)
+            total_weights = sum(p.attention_score for p in patches)
+            doc_score = total_weighted_score / max(total_weights, 1e-8)
+            best_patch = max(patches, key=lambda p: p.score * p.attention_score)
+
+        elif self.aggregation_method == "mean":
+            doc_score = np.mean([patch.score for patch in patches])
+            best_patch = max(patches, key=lambda p: p.score)
+
+        else:
+            raise ValueError(f"Unknown aggregation method: {self.aggregation_method}")
+
+        # Spatial clustering if enabled
+        spatial_clusters = None
+        if self.spatial_clustering:
+            spatial_clusters = self._cluster_patches_spatially(patches)
+
+        return AggregatedResult(
+            image_name=image_name,
+            image_path=patches[0].image_path,
+            doc_score=float(doc_score),
+            patch_count=len(patches),
+            best_patch=best_patch,
+            all_patches=sorted(patches, key=lambda p: p.score, reverse=True),
+            aggregation_method=self.aggregation_method,
+            spatial_clusters=spatial_clusters,
+        )
+
+    def _cluster_patches_spatially(self, patches: list[PatchResult]) -> list[list[PatchResult]]:
+        """Cluster patches that are spatially close to each other."""
+        if len(patches) <= 1:
+            return [patches]
+
+        clusters = []
+        remaining_patches = patches.copy()
+
+        while remaining_patches:
+            # Start new cluster with highest scoring remaining patch
+            seed_patch = max(remaining_patches, key=lambda p: p.score)
+            current_cluster = [seed_patch]
+            remaining_patches.remove(seed_patch)
+
+            # Add nearby patches to cluster
+            added_to_cluster = True
+            while added_to_cluster:
+                added_to_cluster = False
+                for patch in remaining_patches.copy():
+                    if self._is_patch_nearby(patch, current_cluster):
+                        current_cluster.append(patch)
+                        remaining_patches.remove(patch)
+                        added_to_cluster = True
+
+            clusters.append(current_cluster)
+
+        return sorted(clusters, key=lambda cluster: max(p.score for p in cluster), reverse=True)
+
+    def _is_patch_nearby(self, patch: PatchResult, cluster: list[PatchResult]) -> bool:
+        """Check if a patch is spatially close to any patch in the cluster."""
+        patch_center = self._get_patch_center(patch.coordinates)
+
+        for cluster_patch in cluster:
+            cluster_center = self._get_patch_center(cluster_patch.coordinates)
+            distance = np.sqrt(
+                (patch_center[0] - cluster_center[0]) ** 2
+                + (patch_center[1] - cluster_center[1]) ** 2
+            )
+
+            if distance <= self.cluster_distance_threshold:
+                return True
+
+        return False
+
+    def _get_patch_center(self, coordinates: tuple[int, int, int, int]) -> tuple[float, float]:
+        """Get center point of a patch."""
+        x1, y1, x2, y2 = coordinates
+        return ((x1 + x2) / 2, (y1 + y2) / 2)
+
+    def print_aggregated_results(
+        self, results: list[AggregatedResult], max_patches_per_doc: int = 3
+    ):
+        """Pretty print aggregated results."""
+        print(f"\n🔍 Aggregated Results (method: {self.aggregation_method})")
+        print("=" * 80)
+
+        for i, result in enumerate(results):
+            print(f"\n{i + 1}. {result.image_name}")
+            print(f"   Doc Score: {result.doc_score:.4f} | Patches: {result.patch_count}")
+            print(f"   Path: {result.image_path}")
+
+            # Show best patch
+            best = result.best_patch
+            print(
+                f"   🌟 Best Patch: #{best.patch_id} at {best.coordinates} (score: {best.score:.4f})"
+            )
+
+            # Show top patches
+            print("   📍 Top Patches:")
+            for j, patch in enumerate(result.all_patches[:max_patches_per_doc]):
+                print(
+                    f"      {j + 1}. Patch #{patch.patch_id}: {patch.score:.4f} at {patch.coordinates}"
+                )
+
+            # Show spatial clusters if available
+            if result.spatial_clusters and len(result.spatial_clusters) > 1:
+                print(f"   🗂️ Spatial Clusters: {len(result.spatial_clusters)}")
+                for j, cluster in enumerate(result.spatial_clusters[:2]):  # Show top 2 clusters
+                    cluster_score = max(p.score for p in cluster)
+                    print(
+                        f"      Cluster {j + 1}: {len(cluster)} patches (best: {cluster_score:.4f})"
+                    )
+
+
+def demo_aggregation():
+    """Demonstrate the multi-vector aggregation functionality."""
+    print("=== Multi-Vector Aggregation Demo ===")
+
+    # Simulate some patch-level search results
+    # In real usage, these would come from LeannSearcher.search()
+
+    class MockResult:
+        def __init__(self, score, metadata):
+            self.score = score
+            self.metadata = metadata
+
+    # Simulate results for 2 images with multiple patches each
+    mock_results = [
+        # Image 1: cats_and_kitchen.jpg - 4 patches
+        MockResult(
+            0.85,
+            {
+                "image_name": "cats_and_kitchen.jpg",
+                "image_path": "/path/to/cats_and_kitchen.jpg",
+                "patch_id": 3,
+                "coordinates": [100, 50, 224, 174],  # Kitchen area
+                "attention_score": 0.92,
+                "scale": 1.0,
+            },
+        ),
+        MockResult(
+            0.78,
+            {
+                "image_name": "cats_and_kitchen.jpg",
+                "image_path": "/path/to/cats_and_kitchen.jpg",
+                "patch_id": 7,
+                "coordinates": [200, 300, 324, 424],  # Cat area
+                "attention_score": 0.88,
+                "scale": 1.0,
+            },
+        ),
+        MockResult(
+            0.72,
+            {
+                "image_name": "cats_and_kitchen.jpg",
+                "image_path": "/path/to/cats_and_kitchen.jpg",
+                "patch_id": 12,
+                "coordinates": [150, 100, 274, 224],  # Appliances
+                "attention_score": 0.75,
+                "scale": 1.0,
+            },
+        ),
+        MockResult(
+            0.65,
+            {
+                "image_name": "cats_and_kitchen.jpg",
+                "image_path": "/path/to/cats_and_kitchen.jpg",
+                "patch_id": 15,
+                "coordinates": [50, 250, 174, 374],  # Furniture
+                "attention_score": 0.70,
+                "scale": 1.0,
+            },
+        ),
+        # Image 2: city_street.jpg - 3 patches
+        MockResult(
+            0.68,
+            {
+                "image_name": "city_street.jpg",
+                "image_path": "/path/to/city_street.jpg",
+                "patch_id": 2,
+                "coordinates": [300, 100, 424, 224],  # Buildings
+                "attention_score": 0.80,
+                "scale": 1.0,
+            },
+        ),
+        MockResult(
+            0.62,
+            {
+                "image_name": "city_street.jpg",
+                "image_path": "/path/to/city_street.jpg",
+                "patch_id": 8,
+                "coordinates": [100, 350, 224, 474],  # Street level
+                "attention_score": 0.75,
+                "scale": 1.0,
+            },
+        ),
+        MockResult(
+            0.55,
+            {
+                "image_name": "city_street.jpg",
+                "image_path": "/path/to/city_street.jpg",
+                "patch_id": 11,
+                "coordinates": [400, 200, 524, 324],  # Sky area
+                "attention_score": 0.60,
+                "scale": 1.0,
+            },
+        ),
+    ]
+
+    # Test different aggregation methods
+    methods = ["maxsim", "voting", "weighted", "mean"]
+
+    for method in methods:
+        print(f"\n{'=' * 20} {method.upper()} AGGREGATION {'=' * 20}")
+
+        aggregator = MultiVectorAggregator(
+            aggregation_method=method,
+            spatial_clustering=True,
+            cluster_distance_threshold=100.0,
+        )
+
+        aggregated = aggregator.aggregate_results(mock_results, top_k=5)
+        aggregator.print_aggregated_results(aggregated)
+
+
+if __name__ == "__main__":
+    demo_aggregation()

examples/openai_hnsw_example.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+"""
+OpenAI Embedding Example
+
+Complete example showing how to build and search with OpenAI embeddings using HNSW backend.
+"""
+
+import os
+from pathlib import Path
+
+import dotenv
+from leann.api import LeannBuilder, LeannSearcher
+
+# Load environment variables
+dotenv.load_dotenv()
+
+
+def main():
+    # Check if OpenAI API key is available
+    api_key = os.getenv("OPENAI_API_KEY")
+    if not api_key:
+        print("ERROR: OPENAI_API_KEY environment variable not set")
+        return False
+
+    print(f"✅ OpenAI API key found: {api_key[:10]}...")
+
+    # Sample texts
+    sample_texts = [
+        "Machine learning is a powerful technology that enables computers to learn from data.",
+        "Natural language processing helps computers understand and generate human language.",
+        "Deep learning uses neural networks with multiple layers to solve complex problems.",
+        "Computer vision allows machines to interpret and understand visual information.",
+        "Reinforcement learning trains agents to make decisions through trial and error.",
+        "Data science combines statistics, math, and programming to extract insights from data.",
+        "Artificial intelligence aims to create machines that can perform human-like tasks.",
+        "Python is a popular programming language used extensively in data science and AI.",
+        "Neural networks are inspired by the structure and function of the human brain.",
+        "Big data refers to extremely large datasets that require special tools to process.",
+    ]
+
+    INDEX_DIR = Path("./simple_openai_test_index")
+    INDEX_PATH = str(INDEX_DIR / "simple_test.leann")
+
+    print("\n=== Building Index with OpenAI Embeddings ===")
+    print(f"Index path: {INDEX_PATH}")
+
+    try:
+        # Use proper configuration for OpenAI embeddings
+        builder = LeannBuilder(
+            backend_name="hnsw",
+            embedding_model="text-embedding-3-small",
+            embedding_mode="openai",
+            # HNSW settings for OpenAI embeddings
+            M=16,  # Smaller graph degree
+            efConstruction=64,  # Smaller construction complexity
+            is_compact=True,  # Enable compact storage for recompute
+            is_recompute=True,  # MUST enable for OpenAI embeddings
+            num_threads=1,
+        )
+
+        print(f"Adding {len(sample_texts)} texts to the index...")
+        for i, text in enumerate(sample_texts):
+            metadata = {"id": f"doc_{i}", "topic": "AI"}
+            builder.add_text(text, metadata)
+
+        print("Building index...")
+        builder.build_index(INDEX_PATH)
+        print("✅ Index built successfully!")
+
+    except Exception as e:
+        print(f"❌ Error building index: {e}")
+        import traceback
+
+        traceback.print_exc()
+        return False
+
+    print("\n=== Testing Search ===")
+
+    try:
+        searcher = LeannSearcher(INDEX_PATH)
+
+        test_queries = [
+            "What is machine learning?",
+            "How do neural networks work?",
+            "Programming languages for data science",
+        ]
+
+        for query in test_queries:
+            print(f"\n🔍 Query: '{query}'")
+            results = searcher.search(query, top_k=3)
+
+            print(f"   Found {len(results)} results:")
+            for i, result in enumerate(results):
+                print(f"   {i + 1}. Score: {result.score:.4f}")
+                print(f"      Text: {result.text[:80]}...")
+
+        print("\n✅ Search test completed successfully!")
+        return True
+
+    except Exception as e:
+        print(f"❌ Error during search: {e}")
+        import traceback
+
+        traceback.print_exc()
+        return False
+
+
+if __name__ == "__main__":
+    success = main()
+    if success:
+        print("\n🎉 Simple OpenAI index test completed successfully!")
+    else:
+        print("\n💥 Simple OpenAI index test failed!")

examples/resue_index.py

@@ -0,0 +1,23 @@
+import asyncio
+from pathlib import Path
+
+from leann.api import LeannChat
+
+INDEX_DIR = Path("./test_pdf_index_huawei")
+INDEX_PATH = str(INDEX_DIR / "pdf_documents.leann")
+
+
+async def main():
+    print("\n[PHASE 2] Starting Leann chat session...")
+    chat = LeannChat(index_path=INDEX_PATH)
+    query = "What is the main idea of RL and give me 5 exapmle of classic RL algorithms?"
+    query = "Based on the paper, what are the main techniques LEANN explores to reduce the storage overhead and DLPM explore to achieve Fairness and Efiiciency trade-off?"
+    # query = "什么是盘古大模型以及盘古开发过程中遇到了什么阴暗面,任务令一般在什么城市颁发"
+    response = chat.ask(
+        query, top_k=20, recompute_beighbor_embeddings=True, complexity=32, beam_width=1
+    )
+    print(f"\n[PHASE 2] Response: {response}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/run_evaluation.py

@@ -200,10 +200,10 @@ def main():
     args = parser.parse_args()
 
     # --- Path Configuration ---
-    # Assumes a project structure where the script is in 'benchmarks/'
-    # and evaluation data is in 'benchmarks/data/'.
-    script_dir = Path(__file__).resolve().parent
-    data_root = script_dir / "data"
+    # Assumes a project structure where the script is in 'examples/'
+    # and data is in 'data/' at the project root.
+    project_root = Path(__file__).resolve().parent.parent
+    data_root = project_root / "data"
 
     # Download data based on mode
     if args.mode == "build":
@@ -279,9 +279,7 @@ def main():
 
             if not args.index_path:
                 print("No indices found. The data download should have included pre-built indices.")
-                print(
-                    "Please check the benchmarks/data/indices/ directory or provide --index-path manually."
-                )
+                print("Please check the data/indices/ directory or provide --index-path manually.")
                 sys.exit(1)
 
     # Detect dataset type from index path to select the correct ground truth

examples/simple_demo.py

@@ -1,6 +1,6 @@
 """
 Simple demo showing basic leann usage
-Run: uv run python examples/basic_demo.py
+Run: uv run python examples/simple_demo.py
 """
 
 import argparse
@@ -81,7 +81,7 @@ def main():
         print()
 
     print("Demo completed! Try running:")
-    print("   uv run python apps/document_rag.py")
+    print("   uv run python examples/document_search.py")
 
 
 if __name__ == "__main__":

examples/wechat_history_reader_leann.py

@@ -0,0 +1,320 @@
+import argparse
+import asyncio
+import os
+from pathlib import Path
+
+import dotenv
+from leann.api import LeannBuilder, LeannChat
+from llama_index.core.node_parser import SentenceSplitter
+
+dotenv.load_dotenv()
+
+# Default WeChat export directory
+DEFAULT_WECHAT_EXPORT_DIR = "./wechat_export_direct"
+
+
+def create_leann_index_from_multiple_wechat_exports(
+    export_dirs: list[Path],
+    index_path: str = "wechat_history_index.leann",
+    max_count: int = -1,
+):
+    """
+    Create LEANN index from multiple WeChat export data sources.
+
+    Args:
+        export_dirs: List of Path objects pointing to WeChat export directories
+        index_path: Path to save the LEANN index
+        max_count: Maximum number of chat entries to process per export
+    """
+    print("Creating LEANN index from multiple WeChat export data sources...")
+
+    # Load documents using WeChatHistoryReader from history_data
+    from history_data.wechat_history import WeChatHistoryReader
+
+    reader = WeChatHistoryReader()
+
+    INDEX_DIR = Path(index_path).parent
+
+    if not INDEX_DIR.exists():
+        print("--- Index directory not found, building new index ---")
+        all_documents = []
+        total_processed = 0
+
+        # Process each WeChat export directory
+        for i, export_dir in enumerate(export_dirs):
+            print(f"\nProcessing WeChat export {i + 1}/{len(export_dirs)}: {export_dir}")
+
+            try:
+                documents = reader.load_data(
+                    wechat_export_dir=str(export_dir),
+                    max_count=max_count,
+                    concatenate_messages=True,  # Disable concatenation - one message per document
+                )
+                if documents:
+                    print(f"Loaded {len(documents)} chat documents from {export_dir}")
+                    all_documents.extend(documents)
+                    total_processed += len(documents)
+
+                    # Check if we've reached the max count
+                    if max_count > 0 and total_processed >= max_count:
+                        print(f"Reached max count of {max_count} documents")
+                        break
+                else:
+                    print(f"No documents loaded from {export_dir}")
+            except Exception as e:
+                print(f"Error processing {export_dir}: {e}")
+                continue
+
+        if not all_documents:
+            print("No documents loaded from any source. Exiting.")
+            return None
+
+        print(
+            f"\nTotal loaded {len(all_documents)} chat documents from {len(export_dirs)} exports and starting to split them into chunks"
+        )
+
+        # Create text splitter with 256 chunk size
+        text_splitter = SentenceSplitter(chunk_size=192, chunk_overlap=64)
+
+        # Convert Documents to text strings and chunk them
+        all_texts = []
+        for doc in all_documents:
+            # Split the document into chunks
+            nodes = text_splitter.get_nodes_from_documents([doc])
+            for node in nodes:
+                text = (
+                    "[Contact] means the message is from: "
+                    + doc.metadata["contact_name"]
+                    + "\n"
+                    + node.get_content()
+                )
+                all_texts.append(text)
+
+        print(
+            f"Finished splitting {len(all_documents)} documents into {len(all_texts)} text chunks"
+        )
+
+        # Create LEANN index directory
+        print("--- Index directory not found, building new index ---")
+        INDEX_DIR.mkdir(exist_ok=True)
+
+        print("--- Building new LEANN index ---")
+
+        print("\n[PHASE 1] Building Leann index...")
+
+        # Use HNSW backend for better macOS compatibility
+        builder = LeannBuilder(
+            backend_name="hnsw",
+            embedding_model="Qwen/Qwen3-Embedding-0.6B",
+            graph_degree=32,
+            complexity=64,
+            is_compact=True,
+            is_recompute=True,
+            num_threads=1,  # Force single-threaded mode
+        )
+
+        print(f"Adding {len(all_texts)} chat chunks to index...")
+        for chunk_text in all_texts:
+            builder.add_text(chunk_text)
+
+        builder.build_index(index_path)
+        print(f"\nLEANN index built at {index_path}!")
+    else:
+        print(f"--- Using existing index at {INDEX_DIR} ---")
+
+    return index_path
+
+
+def create_leann_index(
+    export_dir: str | None = None,
+    index_path: str = "wechat_history_index.leann",
+    max_count: int = 1000,
+):
+    """
+    Create LEANN index from WeChat chat history data.
+
+    Args:
+        export_dir: Path to the WeChat export directory (optional, uses default if None)
+        index_path: Path to save the LEANN index
+        max_count: Maximum number of chat entries to process
+    """
+    print("Creating LEANN index from WeChat chat history data...")
+    INDEX_DIR = Path(index_path).parent
+
+    if not INDEX_DIR.exists():
+        print("--- Index directory not found, building new index ---")
+        INDEX_DIR.mkdir(exist_ok=True)
+
+        print("--- Building new LEANN index ---")
+
+        print("\n[PHASE 1] Building Leann index...")
+
+        # Load documents using WeChatHistoryReader from history_data
+        from history_data.wechat_history import WeChatHistoryReader
+
+        reader = WeChatHistoryReader()
+
+        documents = reader.load_data(
+            wechat_export_dir=export_dir,
+            max_count=max_count,
+            concatenate_messages=False,  # Disable concatenation - one message per document
+        )
+
+        if not documents:
+            print("No documents loaded. Exiting.")
+            return None
+
+        print(f"Loaded {len(documents)} chat documents")
+
+        # Create text splitter with 256 chunk size
+        text_splitter = SentenceSplitter(chunk_size=256, chunk_overlap=25)
+
+        # Convert Documents to text strings and chunk them
+        all_texts = []
+        for doc in documents:
+            # Split the document into chunks
+            nodes = text_splitter.get_nodes_from_documents([doc])
+            for node in nodes:
+                all_texts.append(node.get_content())
+
+        print(f"Created {len(all_texts)} text chunks from {len(documents)} documents")
+
+        # Create LEANN index directory
+        print("--- Index directory not found, building new index ---")
+        INDEX_DIR.mkdir(exist_ok=True)
+
+        print("--- Building new LEANN index ---")
+
+        print("\n[PHASE 1] Building Leann index...")
+
+        # Use HNSW backend for better macOS compatibility
+        builder = LeannBuilder(
+            backend_name="hnsw",
+            embedding_model="mlx-community/Qwen3-Embedding-0.6B-4bit-DWQ",  # MLX-optimized model
+            graph_degree=32,
+            complexity=64,
+            is_compact=True,
+            is_recompute=True,
+            num_threads=1,  # Force single-threaded mode
+        )
+
+        print(f"Adding {len(all_texts)} chat chunks to index...")
+        for chunk_text in all_texts:
+            builder.add_text(chunk_text)
+
+        builder.build_index(index_path)
+        print(f"\nLEANN index built at {index_path}!")
+    else:
+        print(f"--- Using existing index at {INDEX_DIR} ---")
+
+    return index_path
+
+
+async def query_leann_index(index_path: str, query: str):
+    """
+    Query the LEANN index.
+
+    Args:
+        index_path: Path to the LEANN index
+        query: The query string
+    """
+    print("\n[PHASE 2] Starting Leann chat session...")
+    chat = LeannChat(index_path=index_path)
+
+    print(f"You: {query}")
+    chat_response = chat.ask(
+        query,
+        top_k=20,
+        recompute_beighbor_embeddings=True,
+        complexity=16,
+        beam_width=1,
+        llm_config={
+            "type": "openai",
+            "model": "gpt-4o",
+            "api_key": os.getenv("OPENAI_API_KEY"),
+        },
+        llm_kwargs={"temperature": 0.0, "max_tokens": 1000},
+    )
+    print(f"Leann chat response: \033[36m{chat_response}\033[0m")
+
+
+async def main():
+    """Main function with integrated WeChat export functionality."""
+
+    # Parse command line arguments
+    parser = argparse.ArgumentParser(
+        description="LEANN WeChat History Reader - Create and query WeChat chat history index"
+    )
+    parser.add_argument(
+        "--export-dir",
+        type=str,
+        default=DEFAULT_WECHAT_EXPORT_DIR,
+        help=f"Directory to store WeChat exports (default: {DEFAULT_WECHAT_EXPORT_DIR})",
+    )
+    parser.add_argument(
+        "--index-dir",
+        type=str,
+        default="./wechat_history_magic_test_11Debug_new",
+        help="Directory to store the LEANN index (default: ./wechat_history_index_leann_test)",
+    )
+    parser.add_argument(
+        "--max-entries",
+        type=int,
+        default=50,
+        help="Maximum number of chat entries to process (default: 5000)",
+    )
+    parser.add_argument(
+        "--query",
+        type=str,
+        default=None,
+        help="Single query to run (default: runs example queries)",
+    )
+    parser.add_argument(
+        "--force-export",
+        action="store_true",
+        default=False,
+        help="Force re-export of WeChat data even if exports exist",
+    )
+
+    args = parser.parse_args()
+
+    INDEX_DIR = Path(args.index_dir)
+    INDEX_PATH = str(INDEX_DIR / "wechat_history.leann")
+
+    print(f"Using WeChat export directory: {args.export_dir}")
+    print(f"Index directory: {INDEX_DIR}")
+    print(f"Max entries: {args.max_entries}")
+
+    # Initialize WeChat reader with export capabilities
+    from history_data.wechat_history import WeChatHistoryReader
+
+    reader = WeChatHistoryReader()
+
+    # Find existing exports or create new ones using the centralized method
+    export_dirs = reader.find_or_export_wechat_data(args.export_dir)
+    if not export_dirs:
+        print("Failed to find or export WeChat data. Exiting.")
+        return
+
+    # Create or load the LEANN index from all sources
+    index_path = create_leann_index_from_multiple_wechat_exports(
+        export_dirs, INDEX_PATH, max_count=args.max_entries
+    )
+
+    if index_path:
+        if args.query:
+            # Run single query
+            await query_leann_index(index_path, args.query)
+        else:
+            # Example queries
+            queries = [
+                "我想买魔术师约翰逊的球衣,给我一些对应聊天记录?",
+            ]
+
+            for query in queries:
+                print("\n" + "=" * 60)
+                await query_leann_index(index_path, query)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

packages/leann-backend-diskann/CMakeLists.txt

@@ -0,0 +1,8 @@
+# packages/leann-backend-diskann/CMakeLists.txt (simplified version)
+
+cmake_minimum_required(VERSION 3.20)
+project(leann_backend_diskann_wrapper)
+
+# Tell CMake to directly enter the DiskANN submodule and execute its own CMakeLists.txt
+# DiskANN will handle everything itself, including compiling Python bindings
+add_subdirectory(src/third_party/DiskANN)

packages/leann-backend-diskann/leann_backend_diskann/diskann_backend.py

@@ -4,10 +4,9 @@ import os
 import struct
 import sys
 from pathlib import Path
-from typing import Any, Literal, Optional
+from typing import Any, Literal
 
 import numpy as np
-import psutil
 from leann.interface import (
     LeannBackendBuilderInterface,
     LeannBackendFactoryInterface,
@@ -85,43 +84,6 @@ def _write_vectors_to_bin(data: np.ndarray, file_path: Path):
         f.write(data.tobytes())
 
 
-def _calculate_smart_memory_config(data: np.ndarray) -> tuple[float, float]:
-    """
-    Calculate smart memory configuration for DiskANN based on data size and system specs.
-
-    Args:
-        data: The embedding data array
-
-    Returns:
-        tuple: (search_memory_maximum, build_memory_maximum) in GB
-    """
-    num_vectors, dim = data.shape
-
-    # Calculate embedding storage size
-    embedding_size_bytes = num_vectors * dim * 4  # float32 = 4 bytes
-    embedding_size_gb = embedding_size_bytes / (1024**3)
-
-    # search_memory_maximum: 1/10 of embedding size for optimal PQ compression
-    # This controls Product Quantization size - smaller means more compression
-    search_memory_gb = max(0.1, embedding_size_gb / 10)  # At least 100MB
-
-    # build_memory_maximum: Based on available system RAM for sharding control
-    # This controls how much memory DiskANN uses during index construction
-    available_memory_gb = psutil.virtual_memory().available / (1024**3)
-    total_memory_gb = psutil.virtual_memory().total / (1024**3)
-
-    # Use 50% of available memory, but at least 2GB and at most 75% of total
-    build_memory_gb = max(2.0, min(available_memory_gb * 0.5, total_memory_gb * 0.75))
-
-    logger.info(
-        f"Smart memory config - Data: {embedding_size_gb:.2f}GB, "
-        f"Search mem: {search_memory_gb:.2f}GB (PQ control), "
-        f"Build mem: {build_memory_gb:.2f}GB (sharding control)"
-    )
-
-    return search_memory_gb, build_memory_gb
-
-
 @register_backend("diskann")
 class DiskannBackend(LeannBackendFactoryInterface):
     @staticmethod
@@ -159,16 +121,6 @@ class DiskannBuilder(LeannBackendBuilderInterface):
                 f"Unsupported distance_metric '{build_kwargs.get('distance_metric', 'unknown')}'."
             )
 
-        # Calculate smart memory configuration if not explicitly provided
-        if (
-            "search_memory_maximum" not in build_kwargs
-            or "build_memory_maximum" not in build_kwargs
-        ):
-            smart_search_mem, smart_build_mem = _calculate_smart_memory_config(data)
-        else:
-            smart_search_mem = build_kwargs.get("search_memory_maximum", 4.0)
-            smart_build_mem = build_kwargs.get("build_memory_maximum", 8.0)
-
         try:
             from . import _diskannpy as diskannpy  # type: ignore
 
@@ -179,8 +131,8 @@ class DiskannBuilder(LeannBackendBuilderInterface):
                     index_prefix,
                     build_kwargs.get("complexity", 64),
                     build_kwargs.get("graph_degree", 32),
-                    build_kwargs.get("search_memory_maximum", smart_search_mem),
-                    build_kwargs.get("build_memory_maximum", smart_build_mem),
+                    build_kwargs.get("search_memory_maximum", 4.0),
+                    build_kwargs.get("build_memory_maximum", 8.0),
                     build_kwargs.get("num_threads", 8),
                     build_kwargs.get("pq_disk_bytes", 0),
                     "",
@@ -259,7 +211,7 @@ class DiskannSearcher(BaseSearcher):
         prune_ratio: float = 0.0,
         recompute_embeddings: bool = False,
         pruning_strategy: Literal["global", "local", "proportional"] = "global",
-        zmq_port: Optional[int] = None,
+        zmq_port: int | None = None,
         batch_recompute: bool = False,
         dedup_node_dis: bool = False,
         **kwargs,
@@ -312,8 +264,6 @@ class DiskannSearcher(BaseSearcher):
             use_global_pruning = True
 
         # Perform search with suppressed C++ output based on log level
-        use_deferred_fetch = kwargs.get("USE_DEFERRED_FETCH", True)
-        recompute_neighors = False
         with suppress_cpp_output_if_needed():
             labels, distances = self._index.batch_search(
                 query,
@@ -322,9 +272,9 @@ class DiskannSearcher(BaseSearcher):
                 complexity,
                 beam_width,
                 self.num_threads,
-                use_deferred_fetch,
+                kwargs.get("USE_DEFERRED_FETCH", False),
                 kwargs.get("skip_search_reorder", False),
-                recompute_neighors,
+                recompute_embeddings,
                 dedup_node_dis,
                 prune_ratio,
                 batch_recompute,

packages/leann-backend-diskann/leann_backend_diskann/diskann_embedding_server.py

@@ -10,7 +10,6 @@ import sys
 import threading
 import time
 from pathlib import Path
-from typing import Optional
 
 import numpy as np
 import zmq
@@ -33,11 +32,10 @@ if not logger.handlers:
 
 
 def create_diskann_embedding_server(
-    passages_file: Optional[str] = None,
+    passages_file: str | None = None,
     zmq_port: int = 5555,
     model_name: str = "sentence-transformers/all-mpnet-base-v2",
     embedding_mode: str = "sentence-transformers",
-    distance_metric: str = "l2",
 ):
     """
     Create and start a ZMQ-based embedding server for DiskANN backend.
@@ -262,16 +260,9 @@ if __name__ == "__main__":
         "--embedding-mode",
         type=str,
         default="sentence-transformers",
-        choices=["sentence-transformers", "openai", "mlx", "ollama"],
+        choices=["sentence-transformers", "openai", "mlx"],
         help="Embedding backend mode",
     )
-    parser.add_argument(
-        "--distance-metric",
-        type=str,
-        default="l2",
-        choices=["l2", "mips", "cosine"],
-        help="Distance metric for similarity computation",
-    )
 
     args = parser.parse_args()
 
@@ -281,5 +272,4 @@ if __name__ == "__main__":
         zmq_port=args.zmq_port,
         model_name=args.model_name,
         embedding_mode=args.embedding_mode,
-        distance_metric=args.distance_metric,
     )

packages/leann-backend-diskann/pyproject.toml

@@ -4,8 +4,8 @@ build-backend = "scikit_build_core.build"
 
 [project]
 name = "leann-backend-diskann"
-version = "0.2.8"
-dependencies = ["leann-core==0.2.8", "numpy", "protobuf>=3.19.0"]
+version = "0.1.15"
+dependencies = ["leann-core==0.1.15", "numpy", "protobuf>=3.19.0"]
 
 [tool.scikit-build]
 # Key: simplified CMake path
@@ -17,5 +17,3 @@ editable.mode = "redirect"
 cmake.build-type = "Release"
 build.verbose = true
 build.tool-args = ["-j8"]
-# Let CMake find packages via Homebrew prefix
-cmake.define = {CMAKE_PREFIX_PATH = {env = "CMAKE_PREFIX_PATH"}, OpenMP_ROOT = {env = "OpenMP_ROOT"}}

packages/leann-backend-hnsw/CMakeLists.txt

@@ -5,28 +5,11 @@ set(CMAKE_CXX_COMPILER_WORKS 1)
 
 # Set OpenMP path for macOS
 if(APPLE)
-    # Detect Homebrew installation path (Apple Silicon vs Intel)
-    if(EXISTS "/opt/homebrew/opt/libomp")
-        set(HOMEBREW_PREFIX "/opt/homebrew")
-    elseif(EXISTS "/usr/local/opt/libomp")
-        set(HOMEBREW_PREFIX "/usr/local")
-    else()
-        message(FATAL_ERROR "Could not find libomp installation. Please install with: brew install libomp")
-    endif()
-
-    set(OpenMP_C_FLAGS "-Xpreprocessor -fopenmp -I${HOMEBREW_PREFIX}/opt/libomp/include")
-    set(OpenMP_CXX_FLAGS "-Xpreprocessor -fopenmp -I${HOMEBREW_PREFIX}/opt/libomp/include")
+    set(OpenMP_C_FLAGS "-Xpreprocessor -fopenmp -I/opt/homebrew/opt/libomp/include")
+    set(OpenMP_CXX_FLAGS "-Xpreprocessor -fopenmp -I/opt/homebrew/opt/libomp/include")
     set(OpenMP_C_LIB_NAMES "omp")
     set(OpenMP_CXX_LIB_NAMES "omp")
-    set(OpenMP_omp_LIBRARY "${HOMEBREW_PREFIX}/opt/libomp/lib/libomp.dylib")
-
-    # Force use of system libc++ to avoid version mismatch
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
-    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -stdlib=libc++")
-    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -stdlib=libc++")
-
-    # Set minimum macOS version for better compatibility
-    set(CMAKE_OSX_DEPLOYMENT_TARGET "11.0" CACHE STRING "Minimum macOS version")
+    set(OpenMP_omp_LIBRARY "/opt/homebrew/opt/libomp/lib/libomp.dylib")
 endif()
 
 # Use system ZeroMQ instead of building from source

packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_backend.py

@@ -2,7 +2,7 @@ import logging
 import os
 import shutil
 from pathlib import Path
-from typing import Any, Literal, Optional
+from typing import Any, Literal
 
 import numpy as np
 from leann.interface import (
@@ -152,7 +152,7 @@ class HNSWSearcher(BaseSearcher):
         self,
         query: np.ndarray,
         top_k: int,
-        zmq_port: Optional[int] = None,
+        zmq_port: int | None = None,
         complexity: int = 64,
         beam_width: int = 1,
         prune_ratio: float = 0.0,

packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_embedding_server.py

@@ -10,7 +10,6 @@ import sys
 import threading
 import time
 from pathlib import Path
-from typing import Union
 
 import msgpack
 import numpy as np
@@ -34,7 +33,7 @@ if not logger.handlers:
 
 
 def create_hnsw_embedding_server(
-    passages_file: Union[str, None] = None,
+    passages_file: str | None = None,
     zmq_port: int = 5555,
     model_name: str = "sentence-transformers/all-mpnet-base-v2",
     distance_metric: str = "mips",
@@ -296,7 +295,7 @@ if __name__ == "__main__":
         "--embedding-mode",
         type=str,
         default="sentence-transformers",
-        choices=["sentence-transformers", "openai", "mlx", "ollama"],
+        choices=["sentence-transformers", "openai", "mlx"],
         help="Embedding backend mode",
     )
 

packages/leann-backend-hnsw/pyproject.toml

@@ -6,10 +6,10 @@ build-backend = "scikit_build_core.build"
 
 [project]
 name = "leann-backend-hnsw"
-version = "0.2.8"
+version = "0.1.15"
 description = "Custom-built HNSW (Faiss) backend for the Leann toolkit."
 dependencies = [
-    "leann-core==0.2.8",
+    "leann-core==0.1.15",
     "numpy",
     "pyzmq>=23.0.0",
     "msgpack>=1.0.0",
@@ -22,8 +22,6 @@ cmake.build-type = "Release"
 build.verbose = true
 build.tool-args = ["-j8"]
 
-# CMake definitions to optimize compilation and find Homebrew packages
+# CMake definitions to optimize compilation
 [tool.scikit-build.cmake.define]
 CMAKE_BUILD_PARALLEL_LEVEL = "8"
-CMAKE_PREFIX_PATH = {env = "CMAKE_PREFIX_PATH"}
-OpenMP_ROOT = {env = "OpenMP_ROOT"}

packages/leann-core/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "leann-core"
-version = "0.2.8"
+version = "0.1.15"
 description = "Core API and plugin system for LEANN"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -31,10 +31,8 @@ dependencies = [
     "PyPDF2>=3.0.0",
     "pymupdf>=1.23.0",
     "pdfplumber>=0.10.0",
-    "nbconvert>=7.0.0",  # For .ipynb file support
-    "gitignore-parser>=0.1.12",  # For proper .gitignore handling
-    "mlx>=0.26.3; sys_platform == 'darwin' and platform_machine == 'arm64'",
-    "mlx-lm>=0.26.0; sys_platform == 'darwin' and platform_machine == 'arm64'",
+    "mlx>=0.26.3; sys_platform == 'darwin'",
+    "mlx-lm>=0.26.0; sys_platform == 'darwin'",
 ]
 
 [project.optional-dependencies]
@@ -46,7 +44,6 @@ colab = [
 
 [project.scripts]
 leann = "leann.cli:main"
-leann_mcp = "leann.mcp:main"
 
 [tool.setuptools.packages.find]
 where = ["src"]

packages/leann-core/src/leann/__init__.py

@@ -8,10 +8,6 @@ if platform.system() == "Darwin":
     os.environ["MKL_NUM_THREADS"] = "1"
     os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
     os.environ["KMP_BLOCKTIME"] = "0"
-    # Additional fixes for PyTorch/sentence-transformers on macOS ARM64 only in CI
-    if os.environ.get("CI") == "true":
-        os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "0"
-        os.environ["TOKENIZERS_PARALLELISM"] = "false"
 
 from .api import LeannBuilder, LeannChat, LeannSearcher
 from .registry import BACKEND_REGISTRY, autodiscover_backends

packages/leann-core/src/leann/api.py

@@ -10,7 +10,7 @@ import time
 import warnings
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Any, Literal, Optional
+from typing import Any, Literal
 
 import numpy as np
 
@@ -23,17 +23,12 @@ from .registry import BACKEND_REGISTRY
 logger = logging.getLogger(__name__)
 
 
-def get_registered_backends() -> list[str]:
-    """Get list of registered backend names."""
-    return list(BACKEND_REGISTRY.keys())
-
-
 def compute_embeddings(
     chunks: list[str],
     model_name: str,
     mode: str = "sentence-transformers",
     use_server: bool = True,
-    port: Optional[int] = None,
+    port: int | None = None,
     is_build=False,
 ) -> np.ndarray:
     """
@@ -157,12 +152,12 @@ class LeannBuilder:
         self,
         backend_name: str,
         embedding_model: str = "facebook/contriever",
-        dimensions: Optional[int] = None,
+        dimensions: int | None = None,
         embedding_mode: str = "sentence-transformers",
         **backend_kwargs,
     ):
         self.backend_name = backend_name
-        backend_factory: Optional[LeannBackendFactoryInterface] = BACKEND_REGISTRY.get(backend_name)
+        backend_factory: LeannBackendFactoryInterface | None = BACKEND_REGISTRY.get(backend_name)
         if backend_factory is None:
             raise ValueError(f"Backend '{backend_name}' not found or not registered.")
         self.backend_factory = backend_factory
@@ -242,7 +237,7 @@ class LeannBuilder:
         self.backend_kwargs = backend_kwargs
         self.chunks: list[dict[str, Any]] = []
 
-    def add_text(self, text: str, metadata: Optional[dict[str, Any]] = None):
+    def add_text(self, text: str, metadata: dict[str, Any] | None = None):
         if metadata is None:
             metadata = {}
         passage_id = metadata.get("id", str(len(self.chunks)))
@@ -459,14 +454,7 @@ class LeannSearcher:
 
         self.meta_path_str = f"{index_path}.meta.json"
         if not Path(self.meta_path_str).exists():
-            parent_dir = Path(index_path).parent
-            print(
-                f"Leann metadata file not found at {self.meta_path_str}, and you may need to rm -rf {parent_dir}"
-            )
-            # highlight in red the filenotfound error
-            raise FileNotFoundError(
-                f"Leann metadata file not found at {self.meta_path_str}, \033[91m you may need to rm -rf {parent_dir}\033[0m"
-            )
+            raise FileNotFoundError(f"Leann metadata file not found at {self.meta_path_str}")
         with open(self.meta_path_str, encoding="utf-8") as f:
             self.meta_data = json.load(f)
         backend_name = self.meta_data["backend_name"]
@@ -500,16 +488,6 @@ class LeannSearcher:
         logger.info(f"  Top_k: {top_k}")
         logger.info(f"  Additional kwargs: {kwargs}")
 
-        # Smart top_k detection and adjustment
-        total_docs = len(self.passage_manager.global_offset_map)
-        original_top_k = top_k
-        if top_k > total_docs:
-            top_k = total_docs
-            logger.warning(
-                f"  ⚠️  Requested top_k ({original_top_k}) exceeds total documents ({total_docs})"
-            )
-            logger.warning(f"  ✅ Auto-adjusted top_k to {top_k} to match available documents")
-
         zmq_port = None
 
         start_time = time.time()
@@ -554,7 +532,7 @@ class LeannSearcher:
         if "labels" in results and "distances" in results:
             logger.info(f"  Processing {len(results['labels'][0])} passage IDs:")
             for i, (string_id, dist) in enumerate(
-                zip(results["labels"][0], results["distances"][0])
+                zip(results["labels"][0], results["distances"][0], strict=False)
             ):
                 try:
                     passage_data = self.passage_manager.get_passage(string_id)
@@ -592,7 +570,7 @@ class LeannChat:
     def __init__(
         self,
         index_path: str,
-        llm_config: Optional[dict[str, Any]] = None,
+        llm_config: dict[str, Any] | None = None,
         enable_warmup: bool = False,
         **kwargs,
     ):
@@ -608,7 +586,7 @@ class LeannChat:
         prune_ratio: float = 0.0,
         recompute_embeddings: bool = True,
         pruning_strategy: Literal["global", "local", "proportional"] = "global",
-        llm_kwargs: Optional[dict[str, Any]] = None,
+        llm_kwargs: dict[str, Any] | None = None,
         expected_zmq_port: int = 5557,
         **search_kwargs,
     ):
@@ -636,10 +614,7 @@ class LeannChat:
             "Please provide the best answer you can based on this context and your knowledge."
         )
 
-        ask_time = time.time()
         ans = self.llm.ask(prompt, **llm_kwargs)
-        ask_time = time.time() - ask_time
-        logger.info(f"  Ask time: {ask_time} seconds")
         return ans
 
     def start_interactive(self):

packages/leann-core/src/leann/chat.py

@@ -8,7 +8,7 @@ import difflib
 import logging
 import os
 from abc import ABC, abstractmethod
-from typing import Any, Optional
+from typing import Any
 
 import torch
 
@@ -17,12 +17,12 @@ logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 
-def check_ollama_models(host: str) -> list[str]:
+def check_ollama_models() -> list[str]:
     """Check available Ollama models and return a list"""
     try:
         import requests
 
-        response = requests.get(f"{host}/api/tags", timeout=5)
+        response = requests.get("http://localhost:11434/api/tags", timeout=5)
         if response.status_code == 200:
             data = response.json()
             return [model["name"] for model in data.get("models", [])]
@@ -309,12 +309,10 @@ def search_hf_models(query: str, limit: int = 10) -> list[str]:
     return search_hf_models_fuzzy(query, limit)
 
 
-def validate_model_and_suggest(
-    model_name: str, llm_type: str, host: str = "http://localhost:11434"
-) -> Optional[str]:
+def validate_model_and_suggest(model_name: str, llm_type: str) -> str | None:
     """Validate model name and provide suggestions if invalid"""
     if llm_type == "ollama":
-        available_models = check_ollama_models(host)
+        available_models = check_ollama_models()
         if available_models and model_name not in available_models:
             error_msg = f"Model '{model_name}' not found in your local Ollama installation."
 
@@ -360,11 +358,7 @@ def validate_model_and_suggest(
                 error_msg += f"\n\nModel '{model_name}' was not found in Ollama's library."
 
                 if suggestions:
-                    error_msg += (
-                        "\n\nDid you mean one of these installed models?\n"
-                        + "\nTry to use ollama pull to install the model you need\n"
-                    )
-
+                    error_msg += "\n\nDid you mean one of these installed models?\n"
                     for i, suggestion in enumerate(suggestions, 1):
                         error_msg += f"  {i}. {suggestion}\n"
                 else:
@@ -471,7 +465,7 @@ class OllamaChat(LLMInterface):
                 requests.get(host)
 
             # Pre-check model availability with helpful suggestions
-            model_error = validate_model_and_suggest(model, "ollama", host)
+            model_error = validate_model_and_suggest(model, "ollama")
             if model_error:
                 raise ValueError(model_error)
 
@@ -491,35 +485,11 @@ class OllamaChat(LLMInterface):
         import requests
 
         full_url = f"{self.host}/api/generate"
-
-        # Handle thinking budget for reasoning models
-        options = kwargs.copy()
-        thinking_budget = kwargs.get("thinking_budget")
-        if thinking_budget:
-            # Remove thinking_budget from options as it's not a standard Ollama option
-            options.pop("thinking_budget", None)
-            # Only apply reasoning parameters to models that support it
-            reasoning_supported_models = [
-                "gpt-oss:20b",
-                "gpt-oss:120b",
-                "deepseek-r1",
-                "deepseek-coder",
-            ]
-
-            if thinking_budget in ["low", "medium", "high"]:
-                if any(model in self.model.lower() for model in reasoning_supported_models):
-                    options["reasoning"] = {"effort": thinking_budget, "exclude": False}
-                    logger.info(f"Applied reasoning effort={thinking_budget} to model {self.model}")
-                else:
-                    logger.warning(
-                        f"Thinking budget '{thinking_budget}' requested but model '{self.model}' may not support reasoning parameters. Proceeding without reasoning."
-                    )
-
         payload = {
             "model": self.model,
             "prompt": prompt,
             "stream": False,  # Keep it simple for now
-            "options": options,
+            "options": kwargs,
         }
         logger.debug(f"Sending request to Ollama: {payload}")
         try:
@@ -572,41 +542,14 @@ class HFChat(LLMInterface):
             self.device = "cpu"
             logger.info("No GPU detected. Using CPU.")
 
-        # Load tokenizer and model with timeout protection
-        try:
-            import signal
-
-            def timeout_handler(signum, frame):
-                raise TimeoutError("Model download/loading timed out")
-
-            # Set timeout for model loading (60 seconds)
-            old_handler = signal.signal(signal.SIGALRM, timeout_handler)
-            signal.alarm(60)
-
-            try:
-                logger.info(f"Loading tokenizer for {model_name}...")
-                self.tokenizer = AutoTokenizer.from_pretrained(model_name)
-
-                logger.info(f"Loading model {model_name}...")
-                self.model = AutoModelForCausalLM.from_pretrained(
-                    model_name,
-                    torch_dtype=torch.float16 if self.device != "cpu" else torch.float32,
-                    device_map="auto" if self.device != "cpu" else None,
-                    trust_remote_code=True,
-                )
-                logger.info(f"Successfully loaded {model_name}")
-            finally:
-                signal.alarm(0)  # Cancel the alarm
-                signal.signal(signal.SIGALRM, old_handler)  # Restore old handler
-
-        except TimeoutError:
-            logger.error(f"Model loading timed out for {model_name}")
-            raise RuntimeError(
-                f"Model loading timed out for {model_name}. Please check your internet connection or try a smaller model."
-            )
-        except Exception as e:
-            logger.error(f"Failed to load model {model_name}: {e}")
-            raise
+        # Load tokenizer and model
+        self.tokenizer = AutoTokenizer.from_pretrained(model_name)
+        self.model = AutoModelForCausalLM.from_pretrained(
+            model_name,
+            torch_dtype=torch.float16 if self.device != "cpu" else torch.float32,
+            device_map="auto" if self.device != "cpu" else None,
+            trust_remote_code=True,
+        )
 
         # Move model to device if not using device_map
         if self.device != "cpu" and "device_map" not in str(self.model):
@@ -685,7 +628,7 @@ class HFChat(LLMInterface):
 class OpenAIChat(LLMInterface):
     """LLM interface for OpenAI models."""
 
-    def __init__(self, model: str = "gpt-4o", api_key: Optional[str] = None):
+    def __init__(self, model: str = "gpt-4o", api_key: str | None = None):
         self.model = model
         self.api_key = api_key or os.getenv("OPENAI_API_KEY")
 
@@ -710,38 +653,11 @@ class OpenAIChat(LLMInterface):
         params = {
             "model": self.model,
             "messages": [{"role": "user", "content": prompt}],
+            "max_tokens": kwargs.get("max_tokens", 1000),
             "temperature": kwargs.get("temperature", 0.7),
+            **{k: v for k, v in kwargs.items() if k not in ["max_tokens", "temperature"]},
         }
 
-        # Handle max_tokens vs max_completion_tokens based on model
-        max_tokens = kwargs.get("max_tokens", 1000)
-        if "o3" in self.model or "o4" in self.model or "o1" in self.model:
-            # o-series models use max_completion_tokens
-            params["max_completion_tokens"] = max_tokens
-            params["temperature"] = 1.0
-        else:
-            # Other models use max_tokens
-            params["max_tokens"] = max_tokens
-
-        # Handle thinking budget for reasoning models
-        thinking_budget = kwargs.get("thinking_budget")
-        if thinking_budget and thinking_budget in ["low", "medium", "high"]:
-            # Check if this is an o-series model (partial match for model names)
-            o_series_models = ["o3", "o3-mini", "o4-mini", "o1", "o3-pro", "o3-deep-research"]
-            if any(model in self.model for model in o_series_models):
-                # Use the correct OpenAI reasoning parameter format
-                params["reasoning_effort"] = thinking_budget
-                logger.info(f"Applied reasoning_effort={thinking_budget} to model {self.model}")
-            else:
-                logger.warning(
-                    f"Thinking budget '{thinking_budget}' requested but model '{self.model}' may not support reasoning parameters. Proceeding without reasoning."
-                )
-
-        # Add other kwargs (excluding thinking_budget as it's handled above)
-        for k, v in kwargs.items():
-            if k not in ["max_tokens", "temperature", "thinking_budget"]:
-                params[k] = v
-
         logger.info(f"Sending request to OpenAI with model {self.model}")
 
         try:
@@ -761,7 +677,7 @@ class SimulatedChat(LLMInterface):
         return "This is a simulated answer from the LLM based on the retrieved context."
 
 
-def get_llm(llm_config: Optional[dict[str, Any]] = None) -> LLMInterface:
+def get_llm(llm_config: dict[str, Any] | None = None) -> LLMInterface:
     """
     Factory function to get an LLM interface based on configuration.
 

packages/leann-core/src/leann/cli.py

@@ -1,11 +1,9 @@
 import argparse
 import asyncio
 from pathlib import Path
-from typing import Union
 
 from llama_index.core import SimpleDirectoryReader
 from llama_index.core.node_parser import SentenceSplitter
-from tqdm import tqdm
 
 from .api import LeannBuilder, LeannChat, LeannSearcher
 
@@ -43,23 +41,13 @@ def extract_pdf_text_with_pdfplumber(file_path: str) -> str:
 
 class LeannCLI:
     def __init__(self):
-        # Always use project-local .leann directory (like .git)
-        self.indexes_dir = Path.cwd() / ".leann" / "indexes"
+        self.indexes_dir = Path.home() / ".leann" / "indexes"
         self.indexes_dir.mkdir(parents=True, exist_ok=True)
 
-        # Default parser for documents
         self.node_parser = SentenceSplitter(
             chunk_size=256, chunk_overlap=128, separator=" ", paragraph_separator="\n\n"
         )
 
-        # Code-optimized parser
-        self.code_parser = SentenceSplitter(
-            chunk_size=512,  # Larger chunks for code context
-            chunk_overlap=50,  # Less overlap to preserve function boundaries
-            separator="\n",  # Split by lines for code
-            paragraph_separator="\n\n",  # Preserve logical code blocks
-        )
-
     def get_index_path(self, index_name: str) -> str:
         index_dir = self.indexes_dir / index_name
         return str(index_dir / "documents.leann")
@@ -76,14 +64,10 @@ class LeannCLI:
             formatter_class=argparse.RawDescriptionHelpFormatter,
             epilog="""
 Examples:
-  leann build my-docs --docs ./documents                                  # Build index from directory
-  leann build my-code --docs ./src ./tests ./config                      # Build index from multiple directories
-  leann build my-files --docs ./file1.py ./file2.txt ./docs/             # Build index from files and directories
-  leann build my-mixed --docs ./readme.md ./src/ ./config.json           # Build index from mixed files/dirs
-  leann build my-ppts --docs ./ --file-types .pptx,.pdf                  # Index only PowerPoint and PDF files
-  leann search my-docs "query"                                           # Search in my-docs index
-  leann ask my-docs "question"                                           # Ask my-docs index
-  leann list                                                             # List all stored indexes
+  leann build my-docs --docs ./documents    # Build index named my-docs
+  leann search my-docs "query"             # Search in my-docs index
+  leann ask my-docs "question"             # Ask my-docs index
+  leann list                              # List all stored indexes
             """,
         )
 
@@ -91,38 +75,18 @@ Examples:
 
         # Build command
         build_parser = subparsers.add_parser("build", help="Build document index")
-        build_parser.add_argument(
-            "index_name", nargs="?", help="Index name (default: current directory name)"
-        )
-        build_parser.add_argument(
-            "--docs",
-            type=str,
-            nargs="+",
-            default=["."],
-            help="Documents directories and/or files (default: current directory)",
-        )
+        build_parser.add_argument("index_name", help="Index name")
+        build_parser.add_argument("--docs", type=str, required=True, help="Documents directory")
         build_parser.add_argument(
             "--backend", type=str, default="hnsw", choices=["hnsw", "diskann"]
         )
         build_parser.add_argument("--embedding-model", type=str, default="facebook/contriever")
-        build_parser.add_argument(
-            "--embedding-mode",
-            type=str,
-            default="sentence-transformers",
-            choices=["sentence-transformers", "openai", "mlx", "ollama"],
-            help="Embedding backend mode (default: sentence-transformers)",
-        )
         build_parser.add_argument("--force", "-f", action="store_true", help="Force rebuild")
         build_parser.add_argument("--graph-degree", type=int, default=32)
         build_parser.add_argument("--complexity", type=int, default=64)
         build_parser.add_argument("--num-threads", type=int, default=1)
         build_parser.add_argument("--compact", action="store_true", default=True)
         build_parser.add_argument("--recompute", action="store_true", default=True)
-        build_parser.add_argument(
-            "--file-types",
-            type=str,
-            help="Comma-separated list of file extensions to include (e.g., '.txt,.pdf,.pptx'). If not specified, uses default supported types.",
-        )
 
         # Search command
         search_parser = subparsers.add_parser("search", help="Search documents")
@@ -132,12 +96,7 @@ Examples:
         search_parser.add_argument("--complexity", type=int, default=64)
         search_parser.add_argument("--beam-width", type=int, default=1)
         search_parser.add_argument("--prune-ratio", type=float, default=0.0)
-        search_parser.add_argument(
-            "--recompute-embeddings",
-            action="store_true",
-            default=True,
-            help="Recompute embeddings (default: True)",
-        )
+        search_parser.add_argument("--recompute-embeddings", action="store_true")
         search_parser.add_argument(
             "--pruning-strategy",
             choices=["global", "local", "proportional"],
@@ -160,497 +119,94 @@ Examples:
         ask_parser.add_argument("--complexity", type=int, default=32)
         ask_parser.add_argument("--beam-width", type=int, default=1)
         ask_parser.add_argument("--prune-ratio", type=float, default=0.0)
-        ask_parser.add_argument(
-            "--recompute-embeddings",
-            action="store_true",
-            default=True,
-            help="Recompute embeddings (default: True)",
-        )
+        ask_parser.add_argument("--recompute-embeddings", action="store_true")
         ask_parser.add_argument(
             "--pruning-strategy",
             choices=["global", "local", "proportional"],
             default="global",
         )
-        ask_parser.add_argument(
-            "--thinking-budget",
-            type=str,
-            choices=["low", "medium", "high"],
-            default=None,
-            help="Thinking budget for reasoning models (low/medium/high). Supported by GPT-Oss:20b and other reasoning models.",
-        )
 
         # List command
         subparsers.add_parser("list", help="List all indexes")
 
         return parser
 
-    def register_project_dir(self):
-        """Register current project directory in global registry"""
-        global_registry = Path.home() / ".leann" / "projects.json"
-        global_registry.parent.mkdir(exist_ok=True)
-
-        current_dir = str(Path.cwd())
-
-        # Load existing registry
-        projects = []
-        if global_registry.exists():
-            try:
-                import json
-
-                with open(global_registry) as f:
-                    projects = json.load(f)
-            except Exception:
-                projects = []
-
-        # Add current directory if not already present
-        if current_dir not in projects:
-            projects.append(current_dir)
-
-        # Save registry
-        import json
-
-        with open(global_registry, "w") as f:
-            json.dump(projects, f, indent=2)
-
-    def _build_gitignore_parser(self, docs_dir: str):
-        """Build gitignore parser using gitignore-parser library."""
-        from gitignore_parser import parse_gitignore
-
-        # Try to parse the root .gitignore
-        gitignore_path = Path(docs_dir) / ".gitignore"
-
-        if gitignore_path.exists():
-            try:
-                # gitignore-parser automatically handles all subdirectory .gitignore files!
-                matches = parse_gitignore(str(gitignore_path))
-                print(f"📋 Loaded .gitignore from {docs_dir} (includes all subdirectories)")
-                return matches
-            except Exception as e:
-                print(f"Warning: Could not parse .gitignore: {e}")
-        else:
-            print("📋 No .gitignore found")
-
-        # Fallback: basic pattern matching for essential files
-        essential_patterns = {".git", ".DS_Store", "__pycache__", "node_modules", ".venv", "venv"}
-
-        def basic_matches(file_path):
-            path_parts = Path(file_path).parts
-            return any(part in essential_patterns for part in path_parts)
-
-        return basic_matches
-
-    def _should_exclude_file(self, relative_path: Path, gitignore_matches) -> bool:
-        """Check if a file should be excluded using gitignore parser."""
-        return gitignore_matches(str(relative_path))
-
-    def _is_git_submodule(self, path: Path) -> bool:
-        """Check if a path is a git submodule."""
-        try:
-            # Find the git repo root
-            current_dir = Path.cwd()
-            while current_dir != current_dir.parent:
-                if (current_dir / ".git").exists():
-                    gitmodules_path = current_dir / ".gitmodules"
-                    if gitmodules_path.exists():
-                        # Read .gitmodules to check if this path is a submodule
-                        gitmodules_content = gitmodules_path.read_text()
-                        # Convert path to relative to git root
-                        try:
-                            relative_path = path.resolve().relative_to(current_dir)
-                            # Check if this path appears in .gitmodules
-                            return f"path = {relative_path}" in gitmodules_content
-                        except ValueError:
-                            # Path is not under git root
-                            return False
-                    break
-                current_dir = current_dir.parent
-            return False
-        except Exception:
-            # If anything goes wrong, assume it's not a submodule
-            return False
-
     def list_indexes(self):
         print("Stored LEANN indexes:")
 
-        # Get all project directories with .leann
-        global_registry = Path.home() / ".leann" / "projects.json"
-        all_projects = []
-
-        if global_registry.exists():
-            try:
-                import json
-
-                with open(global_registry) as f:
-                    all_projects = json.load(f)
-            except Exception:
-                pass
-
-        # Filter to only existing directories with .leann
-        valid_projects = []
-        for project_dir in all_projects:
-            project_path = Path(project_dir)
-            if project_path.exists() and (project_path / ".leann" / "indexes").exists():
-                valid_projects.append(project_path)
-
-        # Add current project if it has .leann but not in registry
-        current_path = Path.cwd()
-        if (current_path / ".leann" / "indexes").exists() and current_path not in valid_projects:
-            valid_projects.append(current_path)
-
-        if not valid_projects:
-            print(
-                "No indexes found. Use 'leann build <name> --docs <dir> [<dir2> ...]' to create one."
-            )
+        if not self.indexes_dir.exists():
+            print("No indexes found. Use 'leann build <name> --docs <dir>' to create one.")
             return
 
-        total_indexes = 0
-        current_dir = Path.cwd()
+        index_dirs = [d for d in self.indexes_dir.iterdir() if d.is_dir()]
 
-        for project_path in valid_projects:
-            indexes_dir = project_path / ".leann" / "indexes"
-            if not indexes_dir.exists():
-                continue
+        if not index_dirs:
+            print("No indexes found. Use 'leann build <name> --docs <dir>' to create one.")
+            return
 
-            index_dirs = [d for d in indexes_dir.iterdir() if d.is_dir()]
-            if not index_dirs:
-                continue
+        print(f"Found {len(index_dirs)} indexes:")
+        for i, index_dir in enumerate(index_dirs, 1):
+            index_name = index_dir.name
+            status = "✓" if self.index_exists(index_name) else "✗"
 
-            # Show project header
-            if project_path == current_dir:
-                print(f"\n📁 Current project ({project_path}):")
+            print(f"  {i}. {index_name} [{status}]")
+            if self.index_exists(index_name):
+                index_dir / "documents.leann.meta.json"
+                size_mb = sum(f.stat().st_size for f in index_dir.iterdir() if f.is_file()) / (
+                    1024 * 1024
+                )
+                print(f"     Size: {size_mb:.1f} MB")
+
+        if index_dirs:
+            example_name = index_dirs[0].name
+            print("\nUsage:")
+            print(f'  leann search {example_name} "your query"')
+            print(f"  leann ask {example_name} --interactive")
+
+    def load_documents(self, docs_dir: str):
+        print(f"Loading documents from {docs_dir}...")
+
+        # Try to use better PDF parsers first
+        documents = []
+        docs_path = Path(docs_dir)
+
+        for file_path in docs_path.rglob("*.pdf"):
+            print(f"Processing PDF: {file_path}")
+
+            # Try PyMuPDF first (best quality)
+            text = extract_pdf_text_with_pymupdf(str(file_path))
+            if text is None:
+                # Try pdfplumber
+                text = extract_pdf_text_with_pdfplumber(str(file_path))
+
+            if text:
+                # Create a simple document structure
+                from llama_index.core import Document
+
+                doc = Document(text=text, metadata={"source": str(file_path)})
+                documents.append(doc)
             else:
-                print(f"\n📂 {project_path}:")
-
-            for index_dir in index_dirs:
-                total_indexes += 1
-                index_name = index_dir.name
-                meta_file = index_dir / "documents.leann.meta.json"
-                status = "✓" if meta_file.exists() else "✗"
-
-                print(f"  {total_indexes}. {index_name} [{status}]")
-                if status == "✓":
-                    size_mb = sum(f.stat().st_size for f in index_dir.iterdir() if f.is_file()) / (
-                        1024 * 1024
-                    )
-                    print(f"     Size: {size_mb:.1f} MB")
-
-        if total_indexes > 0:
-            print(f"\nTotal: {total_indexes} indexes across {len(valid_projects)} projects")
-            print("\nUsage (current project only):")
-
-            # Show example from current project
-            current_indexes_dir = current_dir / ".leann" / "indexes"
-            if current_indexes_dir.exists():
-                current_index_dirs = [d for d in current_indexes_dir.iterdir() if d.is_dir()]
-                if current_index_dirs:
-                    example_name = current_index_dirs[0].name
-                    print(f'  leann search {example_name} "your query"')
-                    print(f"  leann ask {example_name} --interactive")
-
-    def load_documents(
-        self, docs_paths: Union[str, list], custom_file_types: Union[str, None] = None
-    ):
-        # Handle both single path (string) and multiple paths (list) for backward compatibility
-        if isinstance(docs_paths, str):
-            docs_paths = [docs_paths]
-
-        # Separate files and directories
-        files = []
-        directories = []
-        for path in docs_paths:
-            path_obj = Path(path)
-            if path_obj.is_file():
-                files.append(str(path_obj))
-            elif path_obj.is_dir():
-                # Check if this is a git submodule - if so, skip it
-                if self._is_git_submodule(path_obj):
-                    print(f"⚠️  Skipping git submodule: {path}")
-                    continue
-                directories.append(str(path_obj))
-            else:
-                print(f"⚠️  Warning: Path '{path}' does not exist, skipping...")
-                continue
-
-        # Print summary of what we're processing
-        total_items = len(files) + len(directories)
-        items_desc = []
-        if files:
-            items_desc.append(f"{len(files)} file{'s' if len(files) > 1 else ''}")
-        if directories:
-            items_desc.append(
-                f"{len(directories)} director{'ies' if len(directories) > 1 else 'y'}"
-            )
-
-        print(f"Loading documents from {' and '.join(items_desc)} ({total_items} total):")
-        if files:
-            print(f"  📄 Files: {', '.join([Path(f).name for f in files])}")
-        if directories:
-            print(f"  📁 Directories: {', '.join(directories)}")
-
-        if custom_file_types:
-            print(f"Using custom file types: {custom_file_types}")
-
-        all_documents = []
-
-        # First, process individual files if any
-        if files:
-            print(f"\n🔄 Processing {len(files)} individual file{'s' if len(files) > 1 else ''}...")
-
-            # Load individual files using SimpleDirectoryReader with input_files
-            # Note: We skip gitignore filtering for explicitly specified files
-            try:
-                # Group files by their parent directory for efficient loading
-                from collections import defaultdict
-
-                files_by_dir = defaultdict(list)
-                for file_path in files:
-                    parent_dir = str(Path(file_path).parent)
-                    files_by_dir[parent_dir].append(file_path)
-
-                # Load files from each parent directory
-                for parent_dir, file_list in files_by_dir.items():
-                    print(
-                        f"  Loading {len(file_list)} file{'s' if len(file_list) > 1 else ''} from {parent_dir}"
-                    )
-                    try:
-                        file_docs = SimpleDirectoryReader(
-                            parent_dir,
-                            input_files=file_list,
-                            filename_as_id=True,
-                        ).load_data()
-                        all_documents.extend(file_docs)
-                        print(
-                            f"    ✅ Loaded {len(file_docs)} document{'s' if len(file_docs) > 1 else ''}"
-                        )
-                    except Exception as e:
-                        print(f"    ❌ Warning: Could not load files from {parent_dir}: {e}")
-
-            except Exception as e:
-                print(f"❌ Error processing individual files: {e}")
-
-        # Define file extensions to process
-        if custom_file_types:
-            # Parse custom file types from comma-separated string
-            code_extensions = [ext.strip() for ext in custom_file_types.split(",") if ext.strip()]
-            # Ensure extensions start with a dot
-            code_extensions = [ext if ext.startswith(".") else f".{ext}" for ext in code_extensions]
-        else:
-            # Use default supported file types
-            code_extensions = [
-                # Original document types
-                ".txt",
-                ".md",
-                ".docx",
-                ".pptx",
-                # Code files for Claude Code integration
-                ".py",
-                ".js",
-                ".ts",
-                ".jsx",
-                ".tsx",
-                ".java",
-                ".cpp",
-                ".c",
-                ".h",
-                ".hpp",
-                ".cs",
-                ".go",
-                ".rs",
-                ".rb",
-                ".php",
-                ".swift",
-                ".kt",
-                ".scala",
-                ".r",
-                ".sql",
-                ".sh",
-                ".bash",
-                ".zsh",
-                ".fish",
-                ".ps1",
-                ".bat",
-                # Config and markup files
-                ".json",
-                ".yaml",
-                ".yml",
-                ".xml",
-                ".toml",
-                ".ini",
-                ".cfg",
-                ".conf",
-                ".html",
-                ".css",
-                ".scss",
-                ".less",
-                ".vue",
-                ".svelte",
-                # Data science
-                ".ipynb",
-                ".R",
-                ".py",
-                ".jl",
-            ]
-
-        # Process each directory
-        if directories:
-            print(
-                f"\n🔄 Processing {len(directories)} director{'ies' if len(directories) > 1 else 'y'}..."
-            )
-
-        for docs_dir in directories:
-            print(f"Processing directory: {docs_dir}")
-            # Build gitignore parser for each directory
-            gitignore_matches = self._build_gitignore_parser(docs_dir)
-
-            # Try to use better PDF parsers first, but only if PDFs are requested
-            documents = []
-            docs_path = Path(docs_dir)
-
-            # Check if we should process PDFs
-            should_process_pdfs = custom_file_types is None or ".pdf" in custom_file_types
-
-            if should_process_pdfs:
-                for file_path in docs_path.rglob("*.pdf"):
-                    # Check if file matches any exclude pattern
-                    try:
-                        relative_path = file_path.relative_to(docs_path)
-                        if self._should_exclude_file(relative_path, gitignore_matches):
-                            continue
-                    except ValueError:
-                        # Skip files that can't be made relative to docs_path
-                        print(f"⚠️  Skipping file outside directory scope: {file_path}")
-                        continue
-
-                    print(f"Processing PDF: {file_path}")
-
-                    # Try PyMuPDF first (best quality)
-                    text = extract_pdf_text_with_pymupdf(str(file_path))
-                    if text is None:
-                        # Try pdfplumber
-                        text = extract_pdf_text_with_pdfplumber(str(file_path))
-
-                    if text:
-                        # Create a simple document structure
-                        from llama_index.core import Document
-
-                        doc = Document(text=text, metadata={"source": str(file_path)})
-                        documents.append(doc)
-                    else:
-                        # Fallback to default reader
-                        print(f"Using default reader for {file_path}")
-                        try:
-                            default_docs = SimpleDirectoryReader(
-                                str(file_path.parent),
-                                filename_as_id=True,
-                                required_exts=[file_path.suffix],
-                            ).load_data()
-                            documents.extend(default_docs)
-                        except Exception as e:
-                            print(f"Warning: Could not process {file_path}: {e}")
-
-            # Load other file types with default reader
-            try:
-                # Create a custom file filter function using our PathSpec
-                def file_filter(
-                    file_path: str, docs_dir=docs_dir, gitignore_matches=gitignore_matches
-                ) -> bool:
-                    """Return True if file should be included (not excluded)"""
-                    try:
-                        docs_path_obj = Path(docs_dir)
-                        file_path_obj = Path(file_path)
-                        relative_path = file_path_obj.relative_to(docs_path_obj)
-                        return not self._should_exclude_file(relative_path, gitignore_matches)
-                    except (ValueError, OSError):
-                        return True  # Include files that can't be processed
-
-                other_docs = SimpleDirectoryReader(
-                    docs_dir,
-                    recursive=True,
-                    encoding="utf-8",
-                    required_exts=code_extensions,
-                    file_extractor={},  # Use default extractors
+                # Fallback to default reader
+                print(f"Using default reader for {file_path}")
+                default_docs = SimpleDirectoryReader(
+                    str(file_path.parent),
                     filename_as_id=True,
-                ).load_data(show_progress=True)
+                    required_exts=[file_path.suffix],
+                ).load_data()
+                documents.extend(default_docs)
 
-                # Filter documents after loading based on gitignore rules
-                filtered_docs = []
-                for doc in other_docs:
-                    file_path = doc.metadata.get("file_path", "")
-                    if file_filter(file_path):
-                        filtered_docs.append(doc)
-
-                documents.extend(filtered_docs)
-            except ValueError as e:
-                if "No files found" in str(e):
-                    print(f"No additional files found for other supported types in {docs_dir}.")
-                else:
-                    raise e
-
-            all_documents.extend(documents)
-            print(f"Loaded {len(documents)} documents from {docs_dir}")
-
-        documents = all_documents
+        # Load other file types with default reader
+        other_docs = SimpleDirectoryReader(
+            docs_dir,
+            recursive=True,
+            encoding="utf-8",
+            required_exts=[".txt", ".md", ".docx"],
+        ).load_data(show_progress=True)
+        documents.extend(other_docs)
 
         all_texts = []
-
-        # Define code file extensions for intelligent chunking
-        code_file_exts = {
-            ".py",
-            ".js",
-            ".ts",
-            ".jsx",
-            ".tsx",
-            ".java",
-            ".cpp",
-            ".c",
-            ".h",
-            ".hpp",
-            ".cs",
-            ".go",
-            ".rs",
-            ".rb",
-            ".php",
-            ".swift",
-            ".kt",
-            ".scala",
-            ".r",
-            ".sql",
-            ".sh",
-            ".bash",
-            ".zsh",
-            ".fish",
-            ".ps1",
-            ".bat",
-            ".json",
-            ".yaml",
-            ".yml",
-            ".xml",
-            ".toml",
-            ".ini",
-            ".cfg",
-            ".conf",
-            ".html",
-            ".css",
-            ".scss",
-            ".less",
-            ".vue",
-            ".svelte",
-            ".ipynb",
-            ".R",
-            ".jl",
-        }
-
-        print("start chunking documents")
-        # Add progress bar for document chunking
-        for doc in tqdm(documents, desc="Chunking documents", unit="doc"):
-            # Check if this is a code file based on source path
-            source_path = doc.metadata.get("source", "")
-            is_code_file = any(source_path.endswith(ext) for ext in code_file_exts)
-
-            # Use appropriate parser based on file type
-            parser = self.code_parser if is_code_file else self.node_parser
-            nodes = parser.get_nodes_from_documents([doc])
-
+        for doc in documents:
+            nodes = self.node_parser.get_nodes_from_documents([doc])
             for node in nodes:
                 all_texts.append(node.get_content())
 
@@ -658,36 +214,16 @@ Examples:
         return all_texts
 
     async def build_index(self, args):
-        docs_paths = args.docs
-        # Use current directory name if index_name not provided
-        if args.index_name:
-            index_name = args.index_name
-        else:
-            index_name = Path.cwd().name
-            print(f"Using current directory name as index: '{index_name}'")
-
+        docs_dir = args.docs
+        index_name = args.index_name
         index_dir = self.indexes_dir / index_name
         index_path = self.get_index_path(index_name)
 
-        # Display all paths being indexed with file/directory distinction
-        files = [p for p in docs_paths if Path(p).is_file()]
-        directories = [p for p in docs_paths if Path(p).is_dir()]
-
-        print(f"📂 Indexing {len(docs_paths)} path{'s' if len(docs_paths) > 1 else ''}:")
-        if files:
-            print(f"  📄 Files ({len(files)}):")
-            for i, file_path in enumerate(files, 1):
-                print(f"    {i}. {Path(file_path).resolve()}")
-        if directories:
-            print(f"  📁 Directories ({len(directories)}):")
-            for i, dir_path in enumerate(directories, 1):
-                print(f"    {i}. {Path(dir_path).resolve()}")
-
         if index_dir.exists() and not args.force:
             print(f"Index '{index_name}' already exists. Use --force to rebuild.")
             return
 
-        all_texts = self.load_documents(docs_paths, args.file_types)
+        all_texts = self.load_documents(docs_dir)
         if not all_texts:
             print("No documents found")
             return
@@ -699,7 +235,6 @@ Examples:
         builder = LeannBuilder(
             backend_name=args.backend,
             embedding_model=args.embedding_model,
-            embedding_mode=args.embedding_mode,
             graph_degree=args.graph_degree,
             complexity=args.complexity,
             is_compact=args.compact,
@@ -713,9 +248,6 @@ Examples:
         builder.build_index(index_path)
         print(f"Index built at {index_path}")
 
-        # Register this project directory in global registry
-        self.register_project_dir()
-
     async def search_documents(self, args):
         index_name = args.index_name
         query = args.query
@@ -723,7 +255,7 @@ Examples:
 
         if not self.index_exists(index_name):
             print(
-                f"Index '{index_name}' not found. Use 'leann build {index_name} --docs <dir> [<dir2> ...]' to create it."
+                f"Index '{index_name}' not found. Use 'leann build {index_name} --docs <dir>' to create it."
             )
             return
 
@@ -750,7 +282,7 @@ Examples:
 
         if not self.index_exists(index_name):
             print(
-                f"Index '{index_name}' not found. Use 'leann build {index_name} --docs <dir> [<dir2> ...]' to create it."
+                f"Index '{index_name}' not found. Use 'leann build {index_name} --docs <dir>' to create it."
             )
             return
 
@@ -776,11 +308,6 @@ Examples:
                 if not user_input:
                     continue
 
-                # Prepare LLM kwargs with thinking budget if specified
-                llm_kwargs = {}
-                if args.thinking_budget:
-                    llm_kwargs["thinking_budget"] = args.thinking_budget
-
                 response = chat.ask(
                     user_input,
                     top_k=args.top_k,
@@ -789,17 +316,11 @@ Examples:
                     prune_ratio=args.prune_ratio,
                     recompute_embeddings=args.recompute_embeddings,
                     pruning_strategy=args.pruning_strategy,
-                    llm_kwargs=llm_kwargs,
                 )
                 print(f"LEANN: {response}")
         else:
             query = input("Enter your question: ").strip()
             if query:
-                # Prepare LLM kwargs with thinking budget if specified
-                llm_kwargs = {}
-                if args.thinking_budget:
-                    llm_kwargs["thinking_budget"] = args.thinking_budget
-
                 response = chat.ask(
                     query,
                     top_k=args.top_k,
@@ -808,7 +329,6 @@ Examples:
                     prune_ratio=args.prune_ratio,
                     recompute_embeddings=args.recompute_embeddings,
                     pruning_strategy=args.pruning_strategy,
-                    llm_kwargs=llm_kwargs,
                 )
                 print(f"LEANN: {response}")
 

packages/leann-core/src/leann/embedding_compute.py

@@ -35,7 +35,7 @@ def compute_embeddings(
     Args:
         texts: List of texts to compute embeddings for
         model_name: Model name
-        mode: Computation mode ('sentence-transformers', 'openai', 'mlx', 'ollama')
+        mode: Computation mode ('sentence-transformers', 'openai', 'mlx')
         is_build: Whether this is a build operation (shows progress bar)
         batch_size: Batch size for processing
         adaptive_optimization: Whether to use adaptive optimization based on batch size
@@ -55,8 +55,6 @@ def compute_embeddings(
         return compute_embeddings_openai(texts, model_name)
     elif mode == "mlx":
         return compute_embeddings_mlx(texts, model_name)
-    elif mode == "ollama":
-        return compute_embeddings_ollama(texts, model_name, is_build=is_build)
     else:
         raise ValueError(f"Unsupported embedding mode: {mode}")
 
@@ -367,286 +365,3 @@ def compute_embeddings_mlx(chunks: list[str], model_name: str, batch_size: int =
 
     # Stack numpy arrays
     return np.stack(all_embeddings)
-
-
-def compute_embeddings_ollama(
-    texts: list[str], model_name: str, is_build: bool = False, host: str = "http://localhost:11434"
-) -> np.ndarray:
-    """
-    Compute embeddings using Ollama API with simplified batch processing.
-
-    Uses batch size of 32 for MPS/CPU and 128 for CUDA to optimize performance.
-
-    Args:
-        texts: List of texts to compute embeddings for
-        model_name: Ollama model name (e.g., "nomic-embed-text", "mxbai-embed-large")
-        is_build: Whether this is a build operation (shows progress bar)
-        host: Ollama host URL (default: http://localhost:11434)
-
-    Returns:
-        Normalized embeddings array, shape: (len(texts), embedding_dim)
-    """
-    try:
-        import requests
-    except ImportError:
-        raise ImportError(
-            "The 'requests' library is required for Ollama embeddings. Install with: uv pip install requests"
-        )
-
-    if not texts:
-        raise ValueError("Cannot compute embeddings for empty text list")
-
-    logger.info(
-        f"Computing embeddings for {len(texts)} texts using Ollama API, model: '{model_name}'"
-    )
-
-    # Check if Ollama is running
-    try:
-        response = requests.get(f"{host}/api/version", timeout=5)
-        response.raise_for_status()
-    except requests.exceptions.ConnectionError:
-        error_msg = (
-            f"❌ Could not connect to Ollama at {host}.\n\n"
-            "Please ensure Ollama is running:\n"
-            "  • macOS/Linux: ollama serve\n"
-            "  • Windows: Make sure Ollama is running in the system tray\n\n"
-            "Installation: https://ollama.com/download"
-        )
-        raise RuntimeError(error_msg)
-    except Exception as e:
-        raise RuntimeError(f"Unexpected error connecting to Ollama: {e}")
-
-    # Check if model exists and provide helpful suggestions
-    try:
-        response = requests.get(f"{host}/api/tags", timeout=5)
-        response.raise_for_status()
-        models = response.json()
-        model_names = [model["name"] for model in models.get("models", [])]
-
-        # Filter for embedding models (models that support embeddings)
-        embedding_models = []
-        suggested_embedding_models = [
-            "nomic-embed-text",
-            "mxbai-embed-large",
-            "bge-m3",
-            "all-minilm",
-            "snowflake-arctic-embed",
-        ]
-
-        for model in model_names:
-            # Check if it's an embedding model (by name patterns or known models)
-            base_name = model.split(":")[0]
-            if any(emb in base_name for emb in ["embed", "bge", "minilm", "e5"]):
-                embedding_models.append(model)
-
-        # Check if model exists (handle versioned names) and resolve to full name
-        resolved_model_name = None
-        for name in model_names:
-            # Exact match
-            if model_name == name:
-                resolved_model_name = name
-                break
-            # Match without version tag (use the versioned name)
-            elif model_name == name.split(":")[0]:
-                resolved_model_name = name
-                break
-
-        if not resolved_model_name:
-            error_msg = f"❌ Model '{model_name}' not found in local Ollama.\n\n"
-
-            # Suggest pulling the model
-            error_msg += "📦 To install this embedding model:\n"
-            error_msg += f"   ollama pull {model_name}\n\n"
-
-            # Show available embedding models
-            if embedding_models:
-                error_msg += "✅ Available embedding models:\n"
-                for model in embedding_models[:5]:
-                    error_msg += f"   • {model}\n"
-                if len(embedding_models) > 5:
-                    error_msg += f"   ... and {len(embedding_models) - 5} more\n"
-            else:
-                error_msg += "💡 Popular embedding models to install:\n"
-                for model in suggested_embedding_models[:3]:
-                    error_msg += f"   • ollama pull {model}\n"
-
-            error_msg += "\n📚 Browse more: https://ollama.com/library"
-            raise ValueError(error_msg)
-
-        # Use the resolved model name for all subsequent operations
-        if resolved_model_name != model_name:
-            logger.info(f"Resolved model name '{model_name}' to '{resolved_model_name}'")
-        model_name = resolved_model_name
-
-        # Verify the model supports embeddings by testing it
-        try:
-            test_response = requests.post(
-                f"{host}/api/embeddings", json={"model": model_name, "prompt": "test"}, timeout=10
-            )
-            if test_response.status_code != 200:
-                error_msg = (
-                    f"⚠️ Model '{model_name}' exists but may not support embeddings.\n\n"
-                    f"Please use an embedding model like:\n"
-                )
-                for model in suggested_embedding_models[:3]:
-                    error_msg += f"   • {model}\n"
-                raise ValueError(error_msg)
-        except requests.exceptions.RequestException:
-            # If test fails, continue anyway - model might still work
-            pass
-
-    except requests.exceptions.RequestException as e:
-        logger.warning(f"Could not verify model existence: {e}")
-
-    # Determine batch size based on device availability
-    # Check for CUDA/MPS availability using torch if available
-    batch_size = 32  # Default for MPS/CPU
-    try:
-        import torch
-
-        if torch.cuda.is_available():
-            batch_size = 128  # CUDA gets larger batch size
-        elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
-            batch_size = 32  # MPS gets smaller batch size
-    except ImportError:
-        # If torch is not available, use conservative batch size
-        batch_size = 32
-
-    logger.info(f"Using batch size: {batch_size}")
-
-    def get_batch_embeddings(batch_texts):
-        """Get embeddings for a batch of texts."""
-        all_embeddings = []
-        failed_indices = []
-
-        for i, text in enumerate(batch_texts):
-            max_retries = 3
-            retry_count = 0
-
-            # Truncate very long texts to avoid API issues
-            truncated_text = text[:8000] if len(text) > 8000 else text
-            while retry_count < max_retries:
-                try:
-                    response = requests.post(
-                        f"{host}/api/embeddings",
-                        json={"model": model_name, "prompt": truncated_text},
-                        timeout=30,
-                    )
-                    response.raise_for_status()
-
-                    result = response.json()
-                    embedding = result.get("embedding")
-
-                    if embedding is None:
-                        raise ValueError(f"No embedding returned for text {i}")
-
-                    if not isinstance(embedding, list) or len(embedding) == 0:
-                        raise ValueError(f"Invalid embedding format for text {i}")
-
-                    all_embeddings.append(embedding)
-                    break
-
-                except requests.exceptions.Timeout:
-                    retry_count += 1
-                    if retry_count >= max_retries:
-                        logger.warning(f"Timeout for text {i} after {max_retries} retries")
-                        failed_indices.append(i)
-                        all_embeddings.append(None)
-                        break
-
-                except Exception as e:
-                    retry_count += 1
-                    if retry_count >= max_retries:
-                        logger.error(f"Failed to get embedding for text {i}: {e}")
-                        failed_indices.append(i)
-                        all_embeddings.append(None)
-                        break
-        return all_embeddings, failed_indices
-
-    # Process texts in batches
-    all_embeddings = []
-    all_failed_indices = []
-
-    # Setup progress bar if needed
-    show_progress = is_build or len(texts) > 10
-    try:
-        if show_progress:
-            from tqdm import tqdm
-    except ImportError:
-        show_progress = False
-
-    # Process batches
-    num_batches = (len(texts) + batch_size - 1) // batch_size
-
-    if show_progress:
-        batch_iterator = tqdm(range(num_batches), desc="Computing Ollama embeddings")
-    else:
-        batch_iterator = range(num_batches)
-
-    for batch_idx in batch_iterator:
-        start_idx = batch_idx * batch_size
-        end_idx = min(start_idx + batch_size, len(texts))
-        batch_texts = texts[start_idx:end_idx]
-
-        batch_embeddings, batch_failed = get_batch_embeddings(batch_texts)
-
-        # Adjust failed indices to global indices
-        global_failed = [start_idx + idx for idx in batch_failed]
-        all_failed_indices.extend(global_failed)
-        all_embeddings.extend(batch_embeddings)
-
-    # Handle failed embeddings
-    if all_failed_indices:
-        if len(all_failed_indices) == len(texts):
-            raise RuntimeError("Failed to compute any embeddings")
-
-        logger.warning(
-            f"Failed to compute embeddings for {len(all_failed_indices)}/{len(texts)} texts"
-        )
-
-        # Use zero embeddings as fallback for failed ones
-        valid_embedding = next((e for e in all_embeddings if e is not None), None)
-        if valid_embedding:
-            embedding_dim = len(valid_embedding)
-            for i, embedding in enumerate(all_embeddings):
-                if embedding is None:
-                    all_embeddings[i] = [0.0] * embedding_dim
-
-    # Remove None values
-    all_embeddings = [e for e in all_embeddings if e is not None]
-
-    if not all_embeddings:
-        raise RuntimeError("No valid embeddings were computed")
-
-    # Validate embedding dimensions
-    expected_dim = len(all_embeddings[0])
-    inconsistent_dims = []
-    for i, embedding in enumerate(all_embeddings):
-        if len(embedding) != expected_dim:
-            inconsistent_dims.append((i, len(embedding)))
-
-    if inconsistent_dims:
-        error_msg = f"Ollama returned inconsistent embedding dimensions. Expected {expected_dim}, but got:\n"
-        for idx, dim in inconsistent_dims[:10]:  # Show first 10 inconsistent ones
-            error_msg += f"  - Text {idx}: {dim} dimensions\n"
-        if len(inconsistent_dims) > 10:
-            error_msg += f"  ... and {len(inconsistent_dims) - 10} more\n"
-        error_msg += f"\nThis is likely an Ollama API bug with model '{model_name}'. Please try:\n"
-        error_msg += "1. Restart Ollama service: 'ollama serve'\n"
-        error_msg += f"2. Re-pull the model: 'ollama pull {model_name}'\n"
-        error_msg += (
-            "3. Use sentence-transformers instead: --embedding-mode sentence-transformers\n"
-        )
-        error_msg += "4. Report this issue to Ollama: https://github.com/ollama/ollama/issues"
-        raise ValueError(error_msg)
-
-    # Convert to numpy array and normalize
-    embeddings = np.array(all_embeddings, dtype=np.float32)
-
-    # Normalize embeddings (L2 normalization)
-    norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
-    embeddings = embeddings / (norms + 1e-8)  # Add small epsilon to avoid division by zero
-
-    logger.info(f"Generated {len(embeddings)} embeddings, dimension: {embeddings.shape[1]}")
-
-    return embeddings

packages/leann-core/src/leann/embedding_server_manager.py

@@ -6,7 +6,6 @@ import subprocess
 import sys
 import time
 from pathlib import Path
-from typing import Optional
 
 import psutil
 
@@ -183,8 +182,8 @@ class EmbeddingServerManager:
                                        e.g., "leann_backend_diskann.embedding_server"
         """
         self.backend_module_name = backend_module_name
-        self.server_process: Optional[subprocess.Popen] = None
-        self.server_port: Optional[int] = None
+        self.server_process: subprocess.Popen | None = None
+        self.server_port: int | None = None
         self._atexit_registered = False
 
     def start_server(
@@ -355,21 +354,13 @@ class EmbeddingServerManager:
         self.server_process.terminate()
 
         try:
-            self.server_process.wait(timeout=3)
+            self.server_process.wait(timeout=5)
             logger.info(f"Server process {self.server_process.pid} terminated.")
         except subprocess.TimeoutExpired:
             logger.warning(
-                f"Server process {self.server_process.pid} did not terminate gracefully within 3 seconds, killing it."
+                f"Server process {self.server_process.pid} did not terminate gracefully, killing it."
             )
             self.server_process.kill()
-            try:
-                self.server_process.wait(timeout=2)
-                logger.info(f"Server process {self.server_process.pid} killed successfully.")
-            except subprocess.TimeoutExpired:
-                logger.error(
-                    f"Failed to kill server process {self.server_process.pid} - it may be hung"
-                )
-                # Don't hang indefinitely
 
         # Clean up process resources to prevent resource tracker warnings
         try:

packages/leann-core/src/leann/interface.py

@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-from typing import Any, Literal, Union
+from typing import Any, Literal
 
 import numpy as np
 
@@ -34,9 +34,7 @@ class LeannBackendSearcherInterface(ABC):
         pass
 
     @abstractmethod
-    def _ensure_server_running(
-        self, passages_source_file: str, port: Union[int, None], **kwargs
-    ) -> int:
+    def _ensure_server_running(self, passages_source_file: str, port: int | None, **kwargs) -> int:
         """Ensure server is running"""
         pass
 
@@ -50,7 +48,7 @@ class LeannBackendSearcherInterface(ABC):
         prune_ratio: float = 0.0,
         recompute_embeddings: bool = False,
         pruning_strategy: Literal["global", "local", "proportional"] = "global",
-        zmq_port: Union[int, None] = None,
+        zmq_port: int | None = None,
         **kwargs,
     ) -> dict[str, Any]:
         """Search for nearest neighbors
@@ -76,7 +74,7 @@ class LeannBackendSearcherInterface(ABC):
         self,
         query: str,
         use_server_if_available: bool = True,
-        zmq_port: Union[int, None] = None,
+        zmq_port: int | None = None,
     ) -> np.ndarray:
         """Compute embedding for a query string
 

packages/leann-core/src/leann/mcp.py

@@ -1,176 +0,0 @@
-#!/usr/bin/env python3
-
-import json
-import subprocess
-import sys
-
-
-def handle_request(request):
-    if request.get("method") == "initialize":
-        return {
-            "jsonrpc": "2.0",
-            "id": request.get("id"),
-            "result": {
-                "capabilities": {"tools": {}},
-                "protocolVersion": "2024-11-05",
-                "serverInfo": {"name": "leann-mcp", "version": "1.0.0"},
-            },
-        }
-
-    elif request.get("method") == "tools/list":
-        return {
-            "jsonrpc": "2.0",
-            "id": request.get("id"),
-            "result": {
-                "tools": [
-                    {
-                        "name": "leann_search",
-                        "description": """🔍 Search code using natural language - like having a coding assistant who knows your entire codebase!
-
-🎯 **Perfect for**:
-- "How does authentication work?" → finds auth-related code
-- "Error handling patterns" → locates try-catch blocks and error logic
-- "Database connection setup" → finds DB initialization code
-- "API endpoint definitions" → locates route handlers
-- "Configuration management" → finds config files and usage
-
-💡 **Pro tip**: Use this before making any changes to understand existing patterns and conventions.""",
-                        "inputSchema": {
-                            "type": "object",
-                            "properties": {
-                                "index_name": {
-                                    "type": "string",
-                                    "description": "Name of the LEANN index to search. Use 'leann_list' first to see available indexes.",
-                                },
-                                "query": {
-                                    "type": "string",
-                                    "description": "Search query - can be natural language (e.g., 'how to handle errors') or technical terms (e.g., 'async function definition')",
-                                },
-                                "top_k": {
-                                    "type": "integer",
-                                    "default": 5,
-                                    "minimum": 1,
-                                    "maximum": 20,
-                                    "description": "Number of search results to return. Use 5-10 for focused results, 15-20 for comprehensive exploration.",
-                                },
-                                "complexity": {
-                                    "type": "integer",
-                                    "default": 32,
-                                    "minimum": 16,
-                                    "maximum": 128,
-                                    "description": "Search complexity level. Use 16-32 for fast searches (recommended), 64+ for higher precision when needed.",
-                                },
-                            },
-                            "required": ["index_name", "query"],
-                        },
-                    },
-                    {
-                        "name": "leann_status",
-                        "description": "📊 Check the health and stats of your code indexes - like a medical checkup for your codebase knowledge!",
-                        "inputSchema": {
-                            "type": "object",
-                            "properties": {
-                                "index_name": {
-                                    "type": "string",
-                                    "description": "Optional: Name of specific index to check. If not provided, shows status of all indexes.",
-                                }
-                            },
-                        },
-                    },
-                    {
-                        "name": "leann_list",
-                        "description": "📋 Show all your indexed codebases - your personal code library! Use this to see what's available for search.",
-                        "inputSchema": {"type": "object", "properties": {}},
-                    },
-                ]
-            },
-        }
-
-    elif request.get("method") == "tools/call":
-        tool_name = request["params"]["name"]
-        args = request["params"].get("arguments", {})
-
-        try:
-            if tool_name == "leann_search":
-                # Validate required parameters
-                if not args.get("index_name") or not args.get("query"):
-                    return {
-                        "jsonrpc": "2.0",
-                        "id": request.get("id"),
-                        "result": {
-                            "content": [
-                                {
-                                    "type": "text",
-                                    "text": "Error: Both index_name and query are required",
-                                }
-                            ]
-                        },
-                    }
-
-                # Build simplified command
-                cmd = [
-                    "leann",
-                    "search",
-                    args["index_name"],
-                    args["query"],
-                    f"--top-k={args.get('top_k', 5)}",
-                    f"--complexity={args.get('complexity', 32)}",
-                ]
-
-                result = subprocess.run(cmd, capture_output=True, text=True)
-
-            elif tool_name == "leann_status":
-                if args.get("index_name"):
-                    # Check specific index status - for now, we'll use leann list and filter
-                    result = subprocess.run(["leann", "list"], capture_output=True, text=True)
-                    # We could enhance this to show more detailed status per index
-                else:
-                    # Show all indexes status
-                    result = subprocess.run(["leann", "list"], capture_output=True, text=True)
-
-            elif tool_name == "leann_list":
-                result = subprocess.run(["leann", "list"], capture_output=True, text=True)
-
-            return {
-                "jsonrpc": "2.0",
-                "id": request.get("id"),
-                "result": {
-                    "content": [
-                        {
-                            "type": "text",
-                            "text": result.stdout
-                            if result.returncode == 0
-                            else f"Error: {result.stderr}",
-                        }
-                    ]
-                },
-            }
-
-        except Exception as e:
-            return {
-                "jsonrpc": "2.0",
-                "id": request.get("id"),
-                "error": {"code": -1, "message": str(e)},
-            }
-
-
-def main():
-    for line in sys.stdin:
-        try:
-            request = json.loads(line.strip())
-            response = handle_request(request)
-            if response:
-                print(json.dumps(response))
-                sys.stdout.flush()
-        except Exception as e:
-            error_response = {
-                "jsonrpc": "2.0",
-                "id": None,
-                "error": {"code": -1, "message": str(e)},
-            }
-            print(json.dumps(error_response))
-            sys.stdout.flush()
-
-
-if __name__ == "__main__":
-    main()

packages/leann-core/src/leann/searcher_base.py

@@ -1,7 +1,7 @@
 import json
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import Any, Literal, Optional
+from typing import Any, Literal
 
 import numpy as np
 
@@ -169,7 +169,7 @@ class BaseSearcher(LeannBackendSearcherInterface, ABC):
         prune_ratio: float = 0.0,
         recompute_embeddings: bool = False,
         pruning_strategy: Literal["global", "local", "proportional"] = "global",
-        zmq_port: Optional[int] = None,
+        zmq_port: int | None = None,
         **kwargs,
     ) -> dict[str, Any]:
         """

packages/leann-mcp/README.md

@@ -1,127 +0,0 @@
-# 🔥 LEANN Claude Code Integration
-
-Transform your development workflow with intelligent code assistance using LEANN's semantic search directly in Claude Code.
-
-## Prerequisites
-
-**Step 1:** First, complete the basic LEANN installation following the [📦 Installation guide](../../README.md#installation) in the root README:
-
-```bash
-uv venv
-source .venv/bin/activate
-uv pip install leann
-```
-
-**Step 2:** Install LEANN globally for MCP integration:
-```bash
-uv tool install leann-core
-```
-
-This makes the `leann` command available system-wide, which `leann_mcp` requires.
-
-## 🚀 Quick Setup
-
-Add the LEANN MCP server to Claude Code:
-
-```bash
-claude mcp add leann-server -- leann_mcp
-```
-
-## 🛠️ Available Tools
-
-Once connected, you'll have access to these powerful semantic search tools in Claude Code:
-
-- **`leann_list`** - List all available indexes across your projects
-- **`leann_search`** - Perform semantic searches across code and documents
-- **`leann_ask`** - Ask natural language questions and get AI-powered answers from your codebase
-
-## 🎯 Quick Start Example
-
-```bash
-# Build an index for your project (change to your actual path)
-leann build my-project --docs ./
-
-# Start Claude Code
-claude
-```
-
-## 🚀 Advanced Usage Examples
-
-### Index Entire Git Repository
-```bash
-# Index all tracked files in your git repository, note right now we will skip submodules, but we can add it back easily if you want
-leann build my-repo --docs $(git ls-files) --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw
-
-# Index only specific file types from git
-leann build my-python-code --docs $(git ls-files "*.py") --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw
-```
-
-### Multiple Directories and Files
-```bash
-# Index multiple directories
-leann build my-codebase --docs ./src ./tests ./docs ./config --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw
-
-# Mix files and directories
-leann build my-project --docs ./README.md ./src/ ./package.json ./docs/ --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw
-
-# Specific files only
-leann build my-configs --docs ./tsconfig.json ./package.json ./webpack.config.js --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw
-```
-
-### Advanced Git Integration
-```bash
-# Index recently modified files
-leann build recent-changes --docs $(git diff --name-only HEAD~10..HEAD) --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw
-
-# Index files matching pattern
-leann build frontend --docs $(git ls-files "*.tsx" "*.ts" "*.jsx" "*.js") --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw
-
-# Index documentation and config files
-leann build docs-and-configs --docs $(git ls-files "*.md" "*.yml" "*.yaml" "*.json" "*.toml") --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw
-```
-
-
-**Try this in Claude Code:**
-```
-Help me understand this codebase. List available indexes and search for authentication patterns.
-```
-
-<p align="center">
-  <img src="../../assets/claude_code_leann.png" alt="LEANN in Claude Code" width="80%">
-</p>
-
-
-## 🧠 How It Works
-
-The integration consists of three key components working seamlessly together:
-
-- **`leann`** - Core CLI tool for indexing and searching (installed globally via `uv tool install`)
-- **`leann_mcp`** - MCP server that wraps `leann` commands for Claude Code integration
-- **Claude Code** - Calls `leann_mcp`, which executes `leann` commands and returns intelligent results
-
-## 📁 File Support
-
-LEANN understands **30+ file types** including:
-- **Programming**: Python, JavaScript, TypeScript, Java, Go, Rust, C++, C#
-- **Data**: SQL, YAML, JSON, CSV, XML
-- **Documentation**: Markdown, TXT, PDF
-- **And many more!**
-
-## 💾 Storage & Organization
-
-- **Project indexes**: Stored in `.leann/` directory (just like `.git`)
-- **Global registry**: Project tracking at `~/.leann/projects.json`
-- **Multi-project support**: Switch between different codebases seamlessly
-- **Portable**: Transfer indexes between machines with minimal overhead
-
-## 🗑️ Uninstalling
-
-To remove the LEANN MCP server from Claude Code:
-
-```bash
-claude mcp remove leann-server
-```
-To remove LEANN
-```
-uv pip uninstall leann leann-backend-hnsw leann-core
-```

packages/leann/README.md

@@ -5,8 +5,11 @@ LEANN is a revolutionary vector database that democratizes personal AI. Transfor
 ## Installation
 
 ```bash
-# Default installation (includes both HNSW and DiskANN backends)
+# Default installation (HNSW backend, recommended)
 uv pip install leann
+
+# With DiskANN backend (for large-scale deployments)
+uv pip install leann[diskann]
 ```
 
 ## Quick Start
@@ -16,8 +19,8 @@ from leann import LeannBuilder, LeannSearcher, LeannChat
 from pathlib import Path
 INDEX_PATH = str(Path("./").resolve() / "demo.leann")
 
-# Build an index (choose backend: "hnsw" or "diskann")
-builder = LeannBuilder(backend_name="hnsw")  # or "diskann" for large-scale deployments
+# Build an index
+builder = LeannBuilder(backend_name="hnsw")
 builder.add_text("LEANN saves 97% storage compared to traditional vector databases.")
 builder.add_text("Tung Tung Tung Sahur called—they need their banana‑crocodile hybrid back")
 builder.build_index(INDEX_PATH)

packages/leann/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "leann"
-version = "0.2.8"
+version = "0.1.15"
 description = "LEANN - The smallest vector index in the world. RAG Everything with LEANN!"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -24,15 +24,16 @@ classifiers = [
     "Programming Language :: Python :: 3.12",
 ]
 
-# Default installation: core + hnsw + diskann
+# Default installation: core + hnsw
 dependencies = [
     "leann-core>=0.1.0",
     "leann-backend-hnsw>=0.1.0",
-    "leann-backend-diskann>=0.1.0",
 ]
 
 [project.optional-dependencies]
-# All backends now included by default
+diskann = [
+    "leann-backend-diskann>=0.1.0",
+]
 
 [project.urls]
 Repository = "https://github.com/yichuan-w/LEANN"

pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "leann-workspace"
 version = "0.1.0"
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 
 dependencies = [
     "leann-core",
@@ -32,27 +32,23 @@ dependencies = [
     "pypdfium2>=4.30.0",
     # LlamaIndex core and readers - updated versions
     "llama-index>=0.12.44",
-    "llama-index-readers-file>=0.4.0", # Essential for PDF parsing
-    # "llama-index-readers-docling",  # Requires Python >= 3.10
-    # "llama-index-node-parser-docling",  # Requires Python >= 3.10
+    "llama-index-readers-file>=0.4.0",  # Essential for PDF parsing
+    "llama-index-readers-docling",
+    "llama-index-node-parser-docling",
     "llama-index-vector-stores-faiss>=0.4.0",
     "llama-index-embeddings-huggingface>=0.5.5",
     # Other dependencies
     "ipykernel==6.29.5",
     "msgpack>=1.1.1",
-    "mlx>=0.26.3; sys_platform == 'darwin' and platform_machine == 'arm64'",
-    "mlx-lm>=0.26.0; sys_platform == 'darwin' and platform_machine == 'arm64'",
+    "mlx>=0.26.3; sys_platform == 'darwin'",
+    "mlx-lm>=0.26.0; sys_platform == 'darwin'",
     "psutil>=5.8.0",
-    "pathspec>=0.12.1",
-    "nbconvert>=7.16.6",
-    "gitignore-parser>=0.1.12",
 ]
 
 [project.optional-dependencies]
 dev = [
     "pytest>=7.0",
     "pytest-cov>=4.0",
-    "pytest-xdist>=3.0",  # For parallel test execution
     "black>=23.0",
     "ruff>=0.1.0",
     "matplotlib",
@@ -60,15 +56,6 @@ dev = [
     "pre-commit>=3.5.0",
 ]
 
-test = [
-    "pytest>=7.0",
-    "pytest-timeout>=2.0",
-    "llama-index-core>=0.12.0",
-    "llama-index-readers-file>=0.4.0",
-    "python-dotenv>=1.0.0",
-    "sentence-transformers>=2.2.0",
-]
-
 diskann = [
     "leann-backend-diskann",
 ]
@@ -91,7 +78,7 @@ leann-backend-diskann = { path = "packages/leann-backend-diskann", editable = tr
 leann-backend-hnsw = { path = "packages/leann-backend-hnsw", editable = true }
 
 [tool.ruff]
-target-version = "py39"
+target-version = "py310"
 line-length = 100
 extend-exclude = [
     "third_party",
@@ -136,32 +123,3 @@ line-ending = "auto"
 dev = [
     "ruff>=0.12.4",
 ]
-
-[tool.lychee]
-accept = ["200", "403", "429", "503"]
-timeout = 20
-max_retries = 2
-exclude = ["localhost", "127.0.0.1", "example.com"]
-exclude_path = [".git/", ".venv/", "__pycache__/", "third_party/"]
-scheme = ["https", "http"]
-
-[tool.pytest.ini_options]
-testpaths = ["tests"]
-python_files = ["test_*.py"]
-python_classes = ["Test*"]
-python_functions = ["test_*"]
-markers = [
-    "slow: marks tests as slow (deselect with '-m \"not slow\"')",
-    "openai: marks tests that require OpenAI API key",
-]
-timeout = 600
-addopts = [
-    "-v",
-    "--tb=short",
-    "--strict-markers",
-    "--disable-warnings",
-]
-env = [
-    "HF_HUB_DISABLE_SYMLINKS=1",
-    "TOKENIZERS_PARALLELISM=false",
-]

test/mail_reader_llamaindex.py

@@ -0,0 +1,161 @@
+import email
+import os
+from typing import Any
+
+from llama_index.core import Document, VectorStoreIndex
+from llama_index.core.readers.base import BaseReader
+
+
+class EmlxReader(BaseReader):
+    """
+    Apple Mail .emlx file reader.
+
+    Reads individual .emlx files from Apple Mail's storage format.
+    """
+
+    def __init__(self) -> None:
+        """Initialize."""
+        pass
+
+    def load_data(self, input_dir: str, **load_kwargs: Any) -> list[Document]:
+        """
+        Load data from the input directory containing .emlx files.
+
+        Args:
+            input_dir: Directory containing .emlx files
+            **load_kwargs:
+                max_count (int): Maximum amount of messages to read.
+        """
+        docs: list[Document] = []
+        max_count = load_kwargs.get("max_count", 1000)
+        count = 0
+
+        # Walk through the directory recursively
+        for dirpath, dirnames, filenames in os.walk(input_dir):
+            # Skip hidden directories
+            dirnames[:] = [d for d in dirnames if not d.startswith(".")]
+
+            for filename in filenames:
+                if count >= max_count:
+                    break
+
+                if filename.endswith(".emlx"):
+                    filepath = os.path.join(dirpath, filename)
+                    try:
+                        # Read the .emlx file
+                        with open(filepath, encoding="utf-8", errors="ignore") as f:
+                            content = f.read()
+
+                        # .emlx files have a length prefix followed by the email content
+                        # The first line contains the length, followed by the email
+                        lines = content.split("\n", 1)
+                        if len(lines) >= 2:
+                            email_content = lines[1]
+
+                            # Parse the email using Python's email module
+                            try:
+                                msg = email.message_from_string(email_content)
+
+                                # Extract email metadata
+                                subject = msg.get("Subject", "No Subject")
+                                from_addr = msg.get("From", "Unknown")
+                                to_addr = msg.get("To", "Unknown")
+                                date = msg.get("Date", "Unknown")
+
+                                # Extract email body
+                                body = ""
+                                if msg.is_multipart():
+                                    for part in msg.walk():
+                                        if (
+                                            part.get_content_type() == "text/plain"
+                                            or part.get_content_type() == "text/html"
+                                        ):
+                                            body += part.get_payload(decode=True).decode(
+                                                "utf-8", errors="ignore"
+                                            )
+                                            # break
+                                else:
+                                    body = msg.get_payload(decode=True).decode(
+                                        "utf-8", errors="ignore"
+                                    )
+
+                                # Create document content
+                                doc_content = f"""
+From: {from_addr}
+To: {to_addr}
+Subject: {subject}
+Date: {date}
+
+{body}
+"""
+
+                                # Create metadata
+                                metadata = {
+                                    "file_path": filepath,
+                                    "subject": subject,
+                                    "from": from_addr,
+                                    "to": to_addr,
+                                    "date": date,
+                                    "filename": filename,
+                                }
+                                if count == 0:
+                                    print("--------------------------------")
+                                    print("dir path", dirpath)
+                                    print(metadata)
+                                    print(doc_content)
+                                    print("--------------------------------")
+                                    body = []
+                                    if msg.is_multipart():
+                                        for part in msg.walk():
+                                            print(
+                                                "--------------------------------  get content type -------------------------------"
+                                            )
+                                            print(part.get_content_type())
+                                            print(part)
+                                            # body.append(part.get_payload(decode=True).decode('utf-8', errors='ignore'))
+                                            print(
+                                                "--------------------------------  get content type -------------------------------"
+                                            )
+                                    else:
+                                        body = msg.get_payload(decode=True).decode(
+                                            "utf-8", errors="ignore"
+                                        )
+                                        print(body)
+
+                                    print(body)
+                                    print("--------------------------------")
+                                doc = Document(text=doc_content, metadata=metadata)
+                                docs.append(doc)
+                                count += 1
+
+                            except Exception as e:
+                                print(f"!!!!!!! Error parsing email from {filepath}: {e} !!!!!!!!")
+                                continue
+
+                    except Exception as e:
+                        print(f"!!!!!!! Error reading file !!!!!!!! {filepath}: {e}")
+                        continue
+
+        print(f"Loaded {len(docs)} email documents")
+        return docs
+
+
+# Use the custom EmlxReader instead of MboxReader
+documents = EmlxReader().load_data(
+    "/Users/yichuan/Library/Mail/V10/0FCA0879-FD8C-4B7E-83BF-FDDA930791C5/[Gmail].mbox/All Mail.mbox/78BA5BE1-8819-4F9A-9613-EB63772F1DD0/Data/9/Messages",
+    max_count=1000,
+)  # Returns list of documents
+
+# Configure the index with larger chunk size to handle long metadata
+from llama_index.core.node_parser import SentenceSplitter
+
+# Create a custom text splitter with larger chunk size
+text_splitter = SentenceSplitter(chunk_size=2048, chunk_overlap=200)
+
+index = VectorStoreIndex.from_documents(
+    documents, transformations=[text_splitter]
+)  # Initialize index with documents
+
+query_engine = index.as_query_engine()
+res = query_engine.query("Hows Berkeley Graduate Student Instructor")
+print(res)

test/mail_reader_save_load.py

@@ -0,0 +1,219 @@
+import email
+import os
+from typing import Any
+
+from llama_index.core import Document, StorageContext, VectorStoreIndex
+from llama_index.core.node_parser import SentenceSplitter
+from llama_index.core.readers.base import BaseReader
+
+
+class EmlxReader(BaseReader):
+    """
+    Apple Mail .emlx file reader.
+
+    Reads individual .emlx files from Apple Mail's storage format.
+    """
+
+    def __init__(self) -> None:
+        """Initialize."""
+        pass
+
+    def load_data(self, input_dir: str, **load_kwargs: Any) -> list[Document]:
+        """
+        Load data from the input directory containing .emlx files.
+
+        Args:
+            input_dir: Directory containing .emlx files
+            **load_kwargs:
+                max_count (int): Maximum amount of messages to read.
+        """
+        docs: list[Document] = []
+        max_count = load_kwargs.get("max_count", 1000)
+        count = 0
+
+        # Walk through the directory recursively
+        for dirpath, dirnames, filenames in os.walk(input_dir):
+            # Skip hidden directories
+            dirnames[:] = [d for d in dirnames if not d.startswith(".")]
+
+            for filename in filenames:
+                if count >= max_count:
+                    break
+
+                if filename.endswith(".emlx"):
+                    filepath = os.path.join(dirpath, filename)
+                    try:
+                        # Read the .emlx file
+                        with open(filepath, encoding="utf-8", errors="ignore") as f:
+                            content = f.read()
+
+                        # .emlx files have a length prefix followed by the email content
+                        # The first line contains the length, followed by the email
+                        lines = content.split("\n", 1)
+                        if len(lines) >= 2:
+                            email_content = lines[1]
+
+                            # Parse the email using Python's email module
+                            try:
+                                msg = email.message_from_string(email_content)
+
+                                # Extract email metadata
+                                subject = msg.get("Subject", "No Subject")
+                                from_addr = msg.get("From", "Unknown")
+                                to_addr = msg.get("To", "Unknown")
+                                date = msg.get("Date", "Unknown")
+
+                                # Extract email body
+                                body = ""
+                                if msg.is_multipart():
+                                    for part in msg.walk():
+                                        if part.get_content_type() == "text/plain":
+                                            body = part.get_payload(decode=True).decode(
+                                                "utf-8", errors="ignore"
+                                            )
+                                            break
+                                else:
+                                    body = msg.get_payload(decode=True).decode(
+                                        "utf-8", errors="ignore"
+                                    )
+
+                                # Create document content
+                                doc_content = f"""
+From: {from_addr}
+To: {to_addr}
+Subject: {subject}
+Date: {date}
+
+{body}
+"""
+
+                                # Create metadata
+                                metadata = {
+                                    "file_path": filepath,
+                                    "subject": subject,
+                                    "from": from_addr,
+                                    "to": to_addr,
+                                    "date": date,
+                                    "filename": filename,
+                                }
+
+                                doc = Document(text=doc_content, metadata=metadata)
+                                docs.append(doc)
+                                count += 1
+
+                            except Exception as e:
+                                print(f"Error parsing email from {filepath}: {e}")
+                                continue
+
+                    except Exception as e:
+                        print(f"Error reading file {filepath}: {e}")
+                        continue
+
+        print(f"Loaded {len(docs)} email documents")
+        return docs
+
+
+def create_and_save_index(mail_path: str, save_dir: str = "mail_index", max_count: int = 1000):
+    """
+    Create the index from mail data and save it to disk.
+
+    Args:
+        mail_path: Path to the mail directory
+        save_dir: Directory to save the index
+        max_count: Maximum number of emails to process
+    """
+    print("Creating index from mail data...")
+
+    # Load documents
+    documents = EmlxReader().load_data(mail_path, max_count=max_count)
+
+    if not documents:
+        print("No documents loaded. Exiting.")
+        return None
+
+    # Create text splitter
+    text_splitter = SentenceSplitter(chunk_size=256, chunk_overlap=0)
+
+    # Create index
+    index = VectorStoreIndex.from_documents(documents, transformations=[text_splitter])
+
+    # Save the index
+    os.makedirs(save_dir, exist_ok=True)
+    index.storage_context.persist(persist_dir=save_dir)
+    print(f"Index saved to {save_dir}")
+
+    return index
+
+
+def load_index(save_dir: str = "mail_index"):
+    """
+    Load the saved index from disk.
+
+    Args:
+        save_dir: Directory where the index is saved
+
+    Returns:
+        Loaded index or None if loading fails
+    """
+    try:
+        # Load storage context
+        storage_context = StorageContext.from_defaults(persist_dir=save_dir)
+
+        # Load index
+        index = VectorStoreIndex.from_vector_store(
+            storage_context.vector_store, storage_context=storage_context
+        )
+
+        print(f"Index loaded from {save_dir}")
+        return index
+
+    except Exception as e:
+        print(f"Error loading index: {e}")
+        return None
+
+
+def query_index(index, query: str):
+    """
+    Query the loaded index.
+
+    Args:
+        index: The loaded index
+        query: The query string
+    """
+    if index is None:
+        print("No index available for querying.")
+        return
+
+    query_engine = index.as_query_engine()
+    response = query_engine.query(query)
+    print(f"Query: {query}")
+    print(f"Response: {response}")
+
+
+def main():
+    mail_path = "/Users/yichuan/Library/Mail/V10/0FCA0879-FD8C-4B7E-83BF-FDDA930791C5/[Gmail].mbox/All Mail.mbox/78BA5BE1-8819-4F9A-9613-EB63772F1DD0/Data/9/Messages"
+    save_dir = "mail_index"
+
+    # Check if index already exists
+    if os.path.exists(save_dir) and os.path.exists(os.path.join(save_dir, "vector_store.json")):
+        print("Loading existing index...")
+        index = load_index(save_dir)
+    else:
+        print("Creating new index...")
+        index = create_and_save_index(mail_path, save_dir, max_count=1000)
+
+    if index:
+        # Example queries
+        queries = [
+            "Hows Berkeley Graduate Student Instructor",
+            "What emails mention GSR appointments?",
+            "Find emails about deadlines",
+        ]
+
+        for query in queries:
+            print("\n" + "=" * 50)
+            query_index(index, query)
+
+
+if __name__ == "__main__":
+    main()

test/mail_reader_small_chunks.py

@@ -0,0 +1,219 @@
+import email
+import os
+from typing import Any
+
+from llama_index.core import Document, StorageContext, VectorStoreIndex
+from llama_index.core.node_parser import SentenceSplitter
+from llama_index.core.readers.base import BaseReader
+
+
+class EmlxReader(BaseReader):
+    """
+    Apple Mail .emlx file reader with reduced metadata.
+
+    Reads individual .emlx files from Apple Mail's storage format.
+    """
+
+    def __init__(self) -> None:
+        """Initialize."""
+        pass
+
+    def load_data(self, input_dir: str, **load_kwargs: Any) -> list[Document]:
+        """
+        Load data from the input directory containing .emlx files.
+
+        Args:
+            input_dir: Directory containing .emlx files
+            **load_kwargs:
+                max_count (int): Maximum amount of messages to read.
+        """
+        docs: list[Document] = []
+        max_count = load_kwargs.get("max_count", 1000)
+        count = 0
+
+        # Walk through the directory recursively
+        for dirpath, dirnames, filenames in os.walk(input_dir):
+            # Skip hidden directories
+            dirnames[:] = [d for d in dirnames if not d.startswith(".")]
+
+            for filename in filenames:
+                if count >= max_count:
+                    break
+
+                if filename.endswith(".emlx"):
+                    filepath = os.path.join(dirpath, filename)
+                    try:
+                        # Read the .emlx file
+                        with open(filepath, encoding="utf-8", errors="ignore") as f:
+                            content = f.read()
+
+                        # .emlx files have a length prefix followed by the email content
+                        # The first line contains the length, followed by the email
+                        lines = content.split("\n", 1)
+                        if len(lines) >= 2:
+                            email_content = lines[1]
+
+                            # Parse the email using Python's email module
+                            try:
+                                msg = email.message_from_string(email_content)
+
+                                # Extract email metadata
+                                subject = msg.get("Subject", "No Subject")
+                                from_addr = msg.get("From", "Unknown")
+                                to_addr = msg.get("To", "Unknown")
+                                date = msg.get("Date", "Unknown")
+
+                                # Extract email body
+                                body = ""
+                                if msg.is_multipart():
+                                    for part in msg.walk():
+                                        if part.get_content_type() == "text/plain":
+                                            body = part.get_payload(decode=True).decode(
+                                                "utf-8", errors="ignore"
+                                            )
+                                            break
+                                else:
+                                    body = msg.get_payload(decode=True).decode(
+                                        "utf-8", errors="ignore"
+                                    )
+
+                                # Create document content with metadata embedded in text
+                                doc_content = f"""
+From: {from_addr}
+To: {to_addr}
+Subject: {subject}
+Date: {date}
+
+{body}
+"""
+
+                                # Create minimal metadata (only essential info)
+                                metadata = {
+                                    "subject": subject[:50],  # Truncate subject
+                                    "from": from_addr[:30],  # Truncate from
+                                    "date": date[:20],  # Truncate date
+                                    "filename": filename,  # Keep filename
+                                }
+
+                                doc = Document(text=doc_content, metadata=metadata)
+                                docs.append(doc)
+                                count += 1
+
+                            except Exception as e:
+                                print(f"Error parsing email from {filepath}: {e}")
+                                continue
+
+                    except Exception as e:
+                        print(f"Error reading file {filepath}: {e}")
+                        continue
+
+        print(f"Loaded {len(docs)} email documents")
+        return docs
+
+
+def create_and_save_index(
+    mail_path: str, save_dir: str = "mail_index_small", max_count: int = 1000
+):
+    """
+    Create the index from mail data and save it to disk.
+
+    Args:
+        mail_path: Path to the mail directory
+        save_dir: Directory to save the index
+        max_count: Maximum number of emails to process
+    """
+    print("Creating index from mail data with small chunks...")
+
+    # Load documents
+    documents = EmlxReader().load_data(mail_path, max_count=max_count)
+
+    if not documents:
+        print("No documents loaded. Exiting.")
+        return None
+
+    # Create text splitter with small chunk size
+    text_splitter = SentenceSplitter(chunk_size=512, chunk_overlap=50)
+
+    # Create index
+    index = VectorStoreIndex.from_documents(documents, transformations=[text_splitter])
+
+    # Save the index
+    os.makedirs(save_dir, exist_ok=True)
+    index.storage_context.persist(persist_dir=save_dir)
+    print(f"Index saved to {save_dir}")
+
+    return index
+
+
+def load_index(save_dir: str = "mail_index_small"):
+    """
+    Load the saved index from disk.
+
+    Args:
+        save_dir: Directory where the index is saved
+
+    Returns:
+        Loaded index or None if loading fails
+    """
+    try:
+        # Load storage context
+        storage_context = StorageContext.from_defaults(persist_dir=save_dir)
+
+        # Load index
+        index = VectorStoreIndex.from_vector_store(
+            storage_context.vector_store, storage_context=storage_context
+        )
+
+        print(f"Index loaded from {save_dir}")
+        return index
+
+    except Exception as e:
+        print(f"Error loading index: {e}")
+        return None
+
+
+def query_index(index, query: str):
+    """
+    Query the loaded index.
+
+    Args:
+        index: The loaded index
+        query: The query string
+    """
+    if index is None:
+        print("No index available for querying.")
+        return
+
+    query_engine = index.as_query_engine()
+    response = query_engine.query(query)
+    print(f"Query: {query}")
+    print(f"Response: {response}")
+
+
+def main():
+    mail_path = "/Users/yichuan/Library/Mail/V10/0FCA0879-FD8C-4B7E-83BF-FDDA930791C5/[Gmail].mbox/All Mail.mbox/78BA5BE1-8819-4F9A-9613-EB63772F1DD0/Data/9/Messages"
+    save_dir = "mail_index_small"
+
+    # Check if index already exists
+    if os.path.exists(save_dir) and os.path.exists(os.path.join(save_dir, "vector_store.json")):
+        print("Loading existing index...")
+        index = load_index(save_dir)
+    else:
+        print("Creating new index...")
+        index = create_and_save_index(mail_path, save_dir, max_count=1000)
+
+    if index:
+        # Example queries
+        queries = [
+            "Hows Berkeley Graduate Student Instructor",
+            "What emails mention GSR appointments?",
+            "Find emails about deadlines",
+        ]
+
+        for query in queries:
+            print("\n" + "=" * 50)
+            query_index(index, query)
+
+
+if __name__ == "__main__":
+    main()

test/mail_reader_test.py

@@ -0,0 +1,154 @@
+import email
+import os
+from typing import Any
+
+from llama_index.core import Document, VectorStoreIndex
+from llama_index.core.readers.base import BaseReader
+
+
+class EmlxReader(BaseReader):
+    """
+    Apple Mail .emlx file reader.
+
+    Reads individual .emlx files from Apple Mail's storage format.
+    """
+
+    def __init__(self) -> None:
+        """Initialize."""
+        pass
+
+    def load_data(self, input_dir: str, **load_kwargs: Any) -> list[Document]:
+        """
+        Load data from the input directory containing .emlx files.
+
+        Args:
+            input_dir: Directory containing .emlx files
+            **load_kwargs:
+                max_count (int): Maximum amount of messages to read.
+        """
+        docs: list[Document] = []
+        max_count = load_kwargs.get("max_count", 1000)
+        count = 0
+
+        # Check if directory exists and is accessible
+        if not os.path.exists(input_dir):
+            print(f"Error: Directory '{input_dir}' does not exist")
+            return docs
+
+        if not os.access(input_dir, os.R_OK):
+            print(f"Error: Directory '{input_dir}' is not accessible (permission denied)")
+            print("This is likely due to macOS security restrictions on Mail app data")
+            return docs
+
+        print(f"Scanning directory: {input_dir}")
+
+        # Walk through the directory recursively
+        for dirpath, dirnames, filenames in os.walk(input_dir):
+            # Skip hidden directories
+            dirnames[:] = [d for d in dirnames if not d.startswith(".")]
+
+            for filename in filenames:
+                if count >= max_count:
+                    break
+
+                if filename.endswith(".emlx"):
+                    filepath = os.path.join(dirpath, filename)
+                    print(f"Found .emlx file: {filepath}")
+                    try:
+                        # Read the .emlx file
+                        with open(filepath, encoding="utf-8", errors="ignore") as f:
+                            content = f.read()
+
+                        # .emlx files have a length prefix followed by the email content
+                        # The first line contains the length, followed by the email
+                        lines = content.split("\n", 1)
+                        if len(lines) >= 2:
+                            email_content = lines[1]
+
+                            # Parse the email using Python's email module
+                            try:
+                                msg = email.message_from_string(email_content)
+
+                                # Extract email metadata
+                                subject = msg.get("Subject", "No Subject")
+                                from_addr = msg.get("From", "Unknown")
+                                to_addr = msg.get("To", "Unknown")
+                                date = msg.get("Date", "Unknown")
+
+                                # Extract email body
+                                body = ""
+                                if msg.is_multipart():
+                                    for part in msg.walk():
+                                        if part.get_content_type() == "text/plain":
+                                            body = part.get_payload(decode=True).decode(
+                                                "utf-8", errors="ignore"
+                                            )
+                                            break
+                                else:
+                                    body = msg.get_payload(decode=True).decode(
+                                        "utf-8", errors="ignore"
+                                    )
+
+                                # Create document content
+                                doc_content = f"""
+From: {from_addr}
+To: {to_addr}
+Subject: {subject}
+Date: {date}
+
+{body}
+"""
+
+                                # Create metadata
+                                metadata = {
+                                    "file_path": filepath,
+                                    "subject": subject,
+                                    "from": from_addr,
+                                    "to": to_addr,
+                                    "date": date,
+                                    "filename": filename,
+                                }
+
+                                doc = Document(text=doc_content, metadata=metadata)
+                                docs.append(doc)
+                                count += 1
+
+                            except Exception as e:
+                                print(f"Error parsing email from {filepath}: {e}")
+                                continue
+
+                    except Exception as e:
+                        print(f"Error reading file {filepath}: {e}")
+                        continue
+
+        print(f"Loaded {len(docs)} email documents")
+        return docs
+
+
+def main():
+    # Use the current directory where the sample.emlx file is located
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+
+    print("Testing EmlxReader with sample .emlx file...")
+    print(f"Scanning directory: {current_dir}")
+
+    # Use the custom EmlxReader
+    documents = EmlxReader().load_data(current_dir, max_count=1000)
+
+    if not documents:
+        print("No documents loaded. Make sure sample.emlx exists in the examples directory.")
+        return
+
+    print(f"\nSuccessfully loaded {len(documents)} document(s)")
+
+    # Initialize index with documents
+    index = VectorStoreIndex.from_documents(documents)
+    query_engine = index.as_query_engine()
+
+    print("\nTesting query: 'Hows Berkeley Graduate Student Instructor'")
+    res = query_engine.query("Hows Berkeley Graduate Student Instructor")
+    print(f"Response: {res}")
+
+
+if __name__ == "__main__":
+    main()

test/query_saved_index.py

@@ -0,0 +1,105 @@
+import os
+
+from llama_index.core import StorageContext, VectorStoreIndex
+
+
+def load_index(save_dir: str = "mail_index"):
+    """
+    Load the saved index from disk.
+
+    Args:
+        save_dir: Directory where the index is saved
+
+    Returns:
+        Loaded index or None if loading fails
+    """
+    try:
+        # Load storage context
+        storage_context = StorageContext.from_defaults(persist_dir=save_dir)
+
+        # Load index
+        index = VectorStoreIndex.from_vector_store(
+            storage_context.vector_store, storage_context=storage_context
+        )
+
+        print(f"Index loaded from {save_dir}")
+        return index
+
+    except Exception as e:
+        print(f"Error loading index: {e}")
+        return None
+
+
+def query_index(index, query: str):
+    """
+    Query the loaded index.
+
+    Args:
+        index: The loaded index
+        query: The query string
+    """
+    if index is None:
+        print("No index available for querying.")
+        return
+
+    query_engine = index.as_query_engine()
+    response = query_engine.query(query)
+    print(f"\nQuery: {query}")
+    print(f"Response: {response}")
+
+
+def main():
+    save_dir = "mail_index"
+
+    # Check if index exists
+    if not os.path.exists(save_dir) or not os.path.exists(
+        os.path.join(save_dir, "vector_store.json")
+    ):
+        print(f"Index not found in {save_dir}")
+        print("Please run mail_reader_save_load.py first to create the index.")
+        return
+
+    # Load the index
+    index = load_index(save_dir)
+
+    if not index:
+        print("Failed to load index.")
+        return
+
+    print("\n" + "=" * 60)
+    print("Email Query Interface")
+    print("=" * 60)
+    print("Type 'quit' to exit")
+    print("Type 'help' for example queries")
+    print("=" * 60)
+
+    # Interactive query loop
+    while True:
+        try:
+            query = input("\nEnter your query: ").strip()
+
+            if query.lower() == "quit":
+                print("Goodbye!")
+                break
+            elif query.lower() == "help":
+                print("\nExample queries:")
+                print("- Hows Berkeley Graduate Student Instructor")
+                print("- What emails mention GSR appointments?")
+                print("- Find emails about deadlines")
+                print("- Search for emails from specific sender")
+                print("- Find emails about meetings")
+                continue
+            elif not query:
+                continue
+
+            query_index(index, query)
+
+        except KeyboardInterrupt:
+            print("\nGoodbye!")
+            break
+        except Exception as e:
+            print(f"Error processing query: {e}")
+
+
+if __name__ == "__main__":
+    main()

test/sanity_checks/debug_zmq_issue.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+"""
+Debug script to test ZMQ communication with the exact same setup as main_cli_example.py
+"""
+
+import sys
+import time
+
+import zmq
+
+sys.path.append("packages/leann-backend-diskann")
+from leann_backend_diskann import embedding_pb2
+
+
+def test_zmq_with_same_model():
+    print("=== Testing ZMQ with same model as main_cli_example.py ===")
+
+    # Test the exact same model that main_cli_example.py uses
+    model_name = "sentence-transformers/all-mpnet-base-v2"
+
+    # Start server with the same model
+    import subprocess
+
+    server_cmd = [
+        sys.executable,
+        "-m",
+        "packages.leann-backend-diskann.leann_backend_diskann.embedding_server",
+        "--zmq-port",
+        "5556",  # Use different port to avoid conflicts
+        "--model-name",
+        model_name,
+    ]
+
+    print(f"Starting server with command: {' '.join(server_cmd)}")
+    server_process = subprocess.Popen(
+        server_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
+    )
+
+    # Wait for server to start
+    print("Waiting for server to start...")
+    time.sleep(10)
+
+    # Check if server is running
+    if server_process.poll() is not None:
+        stdout, stderr = server_process.communicate()
+        print(f"Server failed to start. stdout: {stdout}")
+        print(f"Server failed to start. stderr: {stderr}")
+        return False
+
+    print(f"Server started with PID: {server_process.pid}")
+
+    try:
+        # Test client
+        context = zmq.Context()
+        socket = context.socket(zmq.REQ)
+        socket.connect("tcp://127.0.0.1:5556")
+        socket.setsockopt(zmq.RCVTIMEO, 30000)  # 30 second timeout like C++
+        socket.setsockopt(zmq.SNDTIMEO, 30000)
+
+        # Create request with same format as C++
+        request = embedding_pb2.NodeEmbeddingRequest()
+        request.node_ids.extend([0, 1, 2, 3, 4])  # Test with some node IDs
+
+        print(f"Sending request with {len(request.node_ids)} node IDs...")
+        start_time = time.time()
+
+        # Send request
+        socket.send(request.SerializeToString())
+
+        # Receive response
+        response_data = socket.recv()
+        end_time = time.time()
+
+        print(f"Received response in {end_time - start_time:.3f} seconds")
+        print(f"Response size: {len(response_data)} bytes")
+
+        # Parse response
+        response = embedding_pb2.NodeEmbeddingResponse()
+        response.ParseFromString(response_data)
+
+        print(f"Response dimensions: {list(response.dimensions)}")
+        print(f"Embeddings data size: {len(response.embeddings_data)} bytes")
+        print(f"Missing IDs: {list(response.missing_ids)}")
+
+        # Calculate expected size
+        if len(response.dimensions) == 2:
+            batch_size = response.dimensions[0]
+            embedding_dim = response.dimensions[1]
+            expected_bytes = batch_size * embedding_dim * 4  # 4 bytes per float
+            print(f"Expected bytes: {expected_bytes}, Actual: {len(response.embeddings_data)}")
+
+            if len(response.embeddings_data) == expected_bytes:
+                print("✅ Response format is correct!")
+                return True
+            else:
+                print("❌ Response format mismatch!")
+                return False
+        else:
+            print("❌ Invalid response dimensions!")
+            return False
+
+    except Exception as e:
+        print(f"❌ Error during ZMQ test: {e}")
+        return False
+    finally:
+        # Clean up
+        server_process.terminate()
+        server_process.wait()
+        print("Server terminated")
+
+
+if __name__ == "__main__":
+    success = test_zmq_with_same_model()
+    if success:
+        print("\n✅ ZMQ communication test passed!")
+    else:
+        print("\n❌ ZMQ communication test failed!")

tests/README.md

@@ -1,87 +0,0 @@
-# LEANN Tests
-
-This directory contains automated tests for the LEANN project using pytest.
-
-## Test Files
-
-### `test_readme_examples.py`
-Tests the examples shown in README.md:
-- The basic example code that users see first
-- Import statements work correctly
-- Different backend options (HNSW, DiskANN)
-- Different LLM configuration options
-
-### `test_basic.py`
-Basic functionality tests that verify:
-- All packages can be imported correctly
-- C++ extensions (FAISS, DiskANN) load properly
-- Basic index building and searching works for both HNSW and DiskANN backends
-- Uses parametrized tests to test both backends
-
-### `test_document_rag.py`
-Tests the document RAG example functionality:
-- Tests with facebook/contriever embeddings
-- Tests with OpenAI embeddings (if API key is available)
-- Tests error handling with invalid parameters
-- Verifies that normalized embeddings are detected and cosine distance is used
-
-## Running Tests
-
-### Install test dependencies:
-```bash
-# Using extras
-uv pip install -e ".[test]"
-```
-
-### Run all tests:
-```bash
-pytest tests/
-
-# Or with coverage
-pytest tests/ --cov=leann --cov-report=html
-
-# Run in parallel (faster)
-pytest tests/ -n auto
-```
-
-### Run specific tests:
-```bash
-# Only basic tests
-pytest tests/test_basic.py
-
-# Only tests that don't require OpenAI
-pytest tests/ -m "not openai"
-
-# Skip slow tests
-pytest tests/ -m "not slow"
-```
-
-### Run with specific backend:
-```bash
-# Test only HNSW backend
-pytest tests/test_basic.py::test_backend_basic[hnsw]
-
-# Test only DiskANN backend
-pytest tests/test_basic.py::test_backend_basic[diskann]
-```
-
-## CI/CD Integration
-
-Tests are automatically run in GitHub Actions:
-1. After building wheel packages
-2. On multiple Python versions (3.9 - 3.13)
-3. On both Ubuntu and macOS
-4. Using pytest with appropriate markers and flags
-
-### pytest.ini Configuration
-
-The `pytest.ini` file configures:
-- Test discovery paths
-- Default timeout (600 seconds)
-- Environment variables (HF_HUB_DISABLE_SYMLINKS, TOKENIZERS_PARALLELISM)
-- Custom markers for slow and OpenAI tests
-- Verbose output with short tracebacks
-
-### Known Issues
-
-- OpenAI tests are automatically skipped if no API key is provided

tests/test_basic.py

@@ -1,92 +0,0 @@
-"""
-Basic functionality tests for CI pipeline using pytest.
-"""
-
-import os
-import tempfile
-from pathlib import Path
-
-import pytest
-
-
-def test_imports():
-    """Test that all packages can be imported."""
-
-    # Test C++ extensions
-
-
-@pytest.mark.skipif(
-    os.environ.get("CI") == "true", reason="Skip model tests in CI to avoid MPS memory issues"
-)
-@pytest.mark.parametrize("backend_name", ["hnsw", "diskann"])
-def test_backend_basic(backend_name):
-    """Test basic functionality for each backend."""
-    from leann.api import LeannBuilder, LeannSearcher, SearchResult
-
-    # Create temporary directory for index
-    with tempfile.TemporaryDirectory() as temp_dir:
-        index_path = str(Path(temp_dir) / f"test.{backend_name}")
-
-        # Test with small data
-        texts = [f"This is document {i} about topic {i % 5}" for i in range(100)]
-
-        # Configure builder based on backend
-        if backend_name == "hnsw":
-            builder = LeannBuilder(
-                backend_name="hnsw",
-                embedding_model="facebook/contriever",
-                embedding_mode="sentence-transformers",
-                M=16,
-                efConstruction=200,
-            )
-        else:  # diskann
-            builder = LeannBuilder(
-                backend_name="diskann",
-                embedding_model="facebook/contriever",
-                embedding_mode="sentence-transformers",
-                num_neighbors=32,
-                search_list_size=50,
-            )
-
-        # Add texts
-        for text in texts:
-            builder.add_text(text)
-
-        # Build index
-        builder.build_index(index_path)
-
-        # Test search
-        searcher = LeannSearcher(index_path)
-        results = searcher.search("document about topic 2", top_k=5)
-
-        # Verify results
-        assert len(results) > 0
-        assert isinstance(results[0], SearchResult)
-        assert "topic 2" in results[0].text or "document" in results[0].text
-
-
-@pytest.mark.skipif(
-    os.environ.get("CI") == "true", reason="Skip model tests in CI to avoid MPS memory issues"
-)
-def test_large_index():
-    """Test with larger dataset."""
-    from leann.api import LeannBuilder, LeannSearcher
-
-    with tempfile.TemporaryDirectory() as temp_dir:
-        index_path = str(Path(temp_dir) / "test_large.hnsw")
-        texts = [f"Document {i}: {' '.join([f'word{j}' for j in range(50)])}" for i in range(1000)]
-
-        builder = LeannBuilder(
-            backend_name="hnsw",
-            embedding_model="facebook/contriever",
-            embedding_mode="sentence-transformers",
-        )
-
-        for text in texts:
-            builder.add_text(text)
-
-        builder.build_index(index_path)
-
-        searcher = LeannSearcher(index_path)
-        results = searcher.search(["word10 word20"], top_k=10)
-        assert len(results[0]) == 10

tests/test_ci_minimal.py

@@ -1,49 +0,0 @@
-"""
-Minimal tests for CI that don't require model loading or significant memory.
-"""
-
-import subprocess
-import sys
-
-
-def test_package_imports():
-    """Test that all core packages can be imported."""
-    # Core package
-
-    # Backend packages
-
-    # Core modules
-
-    assert True  # If we get here, imports worked
-
-
-def test_cli_help():
-    """Test that CLI example shows help."""
-    result = subprocess.run(
-        [sys.executable, "apps/document_rag.py", "--help"], capture_output=True, text=True
-    )
-
-    assert result.returncode == 0
-    assert "usage:" in result.stdout.lower() or "usage:" in result.stderr.lower()
-    assert "--llm" in result.stdout or "--llm" in result.stderr
-
-
-def test_backend_registration():
-    """Test that backends are properly registered."""
-    from leann.api import get_registered_backends
-
-    backends = get_registered_backends()
-    assert "hnsw" in backends
-    assert "diskann" in backends
-
-
-def test_version_info():
-    """Test that packages have version information."""
-    import leann
-    import leann_backend_diskann
-    import leann_backend_hnsw
-
-    # Check that packages have __version__ or can be imported
-    assert hasattr(leann, "__version__") or True
-    assert hasattr(leann_backend_hnsw, "__version__") or True
-    assert hasattr(leann_backend_diskann, "__version__") or True

tests/test_document_rag.py

@@ -1,120 +0,0 @@
-"""
-Test document_rag functionality using pytest.
-"""
-
-import os
-import subprocess
-import sys
-import tempfile
-from pathlib import Path
-
-import pytest
-
-
-@pytest.fixture
-def test_data_dir():
-    """Return the path to test data directory."""
-    return Path("data")
-
-
-@pytest.mark.skipif(
-    os.environ.get("CI") == "true", reason="Skip model tests in CI to avoid MPS memory issues"
-)
-def test_document_rag_simulated(test_data_dir):
-    """Test document_rag with simulated LLM."""
-    with tempfile.TemporaryDirectory() as temp_dir:
-        # Use a subdirectory that doesn't exist yet to force index creation
-        index_dir = Path(temp_dir) / "test_index"
-        cmd = [
-            sys.executable,
-            "apps/document_rag.py",
-            "--llm",
-            "simulated",
-            "--embedding-model",
-            "facebook/contriever",
-            "--embedding-mode",
-            "sentence-transformers",
-            "--index-dir",
-            str(index_dir),
-            "--data-dir",
-            str(test_data_dir),
-            "--query",
-            "What is Pride and Prejudice about?",
-        ]
-
-        env = os.environ.copy()
-        env["HF_HUB_DISABLE_SYMLINKS"] = "1"
-        env["TOKENIZERS_PARALLELISM"] = "false"
-
-        result = subprocess.run(cmd, capture_output=True, text=True, timeout=600, env=env)
-
-        # Check return code
-        assert result.returncode == 0, f"Command failed: {result.stderr}"
-
-        # Verify output
-        output = result.stdout + result.stderr
-        assert "Index saved to" in output or "Using existing index" in output
-        assert "This is a simulated answer" in output
-
-
-@pytest.mark.skipif(not os.environ.get("OPENAI_API_KEY"), reason="OpenAI API key not available")
-def test_document_rag_openai(test_data_dir):
-    """Test document_rag with OpenAI embeddings."""
-    with tempfile.TemporaryDirectory() as temp_dir:
-        # Use a subdirectory that doesn't exist yet to force index creation
-        index_dir = Path(temp_dir) / "test_index_openai"
-        cmd = [
-            sys.executable,
-            "apps/document_rag.py",
-            "--llm",
-            "simulated",  # Use simulated LLM to avoid GPT-4 costs
-            "--embedding-model",
-            "text-embedding-3-small",
-            "--embedding-mode",
-            "openai",
-            "--index-dir",
-            str(index_dir),
-            "--data-dir",
-            str(test_data_dir),
-            "--query",
-            "What is Pride and Prejudice about?",
-        ]
-
-        env = os.environ.copy()
-        env["TOKENIZERS_PARALLELISM"] = "false"
-
-        result = subprocess.run(cmd, capture_output=True, text=True, timeout=600, env=env)
-
-        assert result.returncode == 0, f"Command failed: {result.stderr}"
-
-        # Verify cosine distance was used
-        output = result.stdout + result.stderr
-        assert any(
-            msg in output
-            for msg in [
-                "distance_metric='cosine'",
-                "Automatically setting distance_metric='cosine'",
-                "Using cosine distance",
-            ]
-        )
-
-
-def test_document_rag_error_handling(test_data_dir):
-    """Test document_rag with invalid parameters."""
-    with tempfile.TemporaryDirectory() as temp_dir:
-        cmd = [
-            sys.executable,
-            "apps/document_rag.py",
-            "--llm",
-            "invalid_llm_type",
-            "--index-dir",
-            temp_dir,
-            "--data-dir",
-            str(test_data_dir),
-        ]
-
-        result = subprocess.run(cmd, capture_output=True, text=True, timeout=60)
-
-        # Should fail with invalid LLM type
-        assert result.returncode != 0
-        assert "invalid choice" in result.stderr or "invalid_llm_type" in result.stderr

tests/test_readme_examples.py

@@ -1,165 +0,0 @@
-"""
-Test examples from README.md to ensure documentation is accurate.
-"""
-
-import os
-import platform
-import tempfile
-from pathlib import Path
-
-import pytest
-
-
-def test_readme_basic_example():
-    """Test the basic example from README.md."""
-    # Skip on macOS CI due to MPS environment issues with all-MiniLM-L6-v2
-    if os.environ.get("CI") == "true" and platform.system() == "Darwin":
-        pytest.skip("Skipping on macOS CI due to MPS environment issues with all-MiniLM-L6-v2")
-
-    # This is the exact code from README (with smaller model for CI)
-    from leann import LeannBuilder, LeannChat, LeannSearcher
-    from leann.api import SearchResult
-
-    with tempfile.TemporaryDirectory() as temp_dir:
-        INDEX_PATH = str(Path(temp_dir) / "demo.leann")
-
-        # Build an index
-        # In CI, use a smaller model to avoid memory issues
-        if os.environ.get("CI") == "true":
-            builder = LeannBuilder(
-                backend_name="hnsw",
-                embedding_model="sentence-transformers/all-MiniLM-L6-v2",  # Smaller model
-                dimensions=384,  # Smaller dimensions
-            )
-        else:
-            builder = LeannBuilder(backend_name="hnsw")
-        builder.add_text("LEANN saves 97% storage compared to traditional vector databases.")
-        builder.add_text("Tung Tung Tung Sahur called—they need their banana-crocodile hybrid back")
-        builder.build_index(INDEX_PATH)
-
-        # Verify index was created
-        # The index path should be a directory containing index files
-        index_dir = Path(INDEX_PATH).parent
-        assert index_dir.exists()
-        # Check that index files were created
-        index_files = list(index_dir.glob(f"{Path(INDEX_PATH).stem}.*"))
-        assert len(index_files) > 0
-
-        # Search
-        searcher = LeannSearcher(INDEX_PATH)
-        results = searcher.search("fantastical AI-generated creatures", top_k=1)
-
-        # Verify search results
-        assert len(results) > 0
-        assert isinstance(results[0], SearchResult)
-        # The second text about banana-crocodile should be more relevant
-        assert "banana" in results[0].text or "crocodile" in results[0].text
-
-        # Chat with your data (using simulated LLM to avoid external dependencies)
-        chat = LeannChat(INDEX_PATH, llm_config={"type": "simulated"})
-        response = chat.ask("How much storage does LEANN save?", top_k=1)
-
-        # Verify chat works
-        assert isinstance(response, str)
-        assert len(response) > 0
-
-
-def test_readme_imports():
-    """Test that the imports shown in README work correctly."""
-    # These are the imports shown in README
-    from leann import LeannBuilder, LeannChat, LeannSearcher
-
-    # Verify they are the correct types
-    assert callable(LeannBuilder)
-    assert callable(LeannSearcher)
-    assert callable(LeannChat)
-
-
-def test_backend_options():
-    """Test different backend options mentioned in documentation."""
-    # Skip on macOS CI due to MPS environment issues with all-MiniLM-L6-v2
-    if os.environ.get("CI") == "true" and platform.system() == "Darwin":
-        pytest.skip("Skipping on macOS CI due to MPS environment issues with all-MiniLM-L6-v2")
-
-    from leann import LeannBuilder
-
-    with tempfile.TemporaryDirectory() as temp_dir:
-        # Use smaller model in CI to avoid memory issues
-        if os.environ.get("CI") == "true":
-            model_args = {
-                "embedding_model": "sentence-transformers/all-MiniLM-L6-v2",
-                "dimensions": 384,
-            }
-        else:
-            model_args = {}
-
-        # Test HNSW backend (as shown in README)
-        hnsw_path = str(Path(temp_dir) / "test_hnsw.leann")
-        builder_hnsw = LeannBuilder(backend_name="hnsw", **model_args)
-        builder_hnsw.add_text("Test document for HNSW backend")
-        builder_hnsw.build_index(hnsw_path)
-        assert Path(hnsw_path).parent.exists()
-        assert len(list(Path(hnsw_path).parent.glob(f"{Path(hnsw_path).stem}.*"))) > 0
-
-        # Test DiskANN backend (mentioned as available option)
-        diskann_path = str(Path(temp_dir) / "test_diskann.leann")
-        builder_diskann = LeannBuilder(backend_name="diskann", **model_args)
-        builder_diskann.add_text("Test document for DiskANN backend")
-        builder_diskann.build_index(diskann_path)
-        assert Path(diskann_path).parent.exists()
-        assert len(list(Path(diskann_path).parent.glob(f"{Path(diskann_path).stem}.*"))) > 0
-
-
-def test_llm_config_simulated():
-    """Test simulated LLM configuration option."""
-    # Skip on macOS CI due to MPS environment issues with all-MiniLM-L6-v2
-    if os.environ.get("CI") == "true" and platform.system() == "Darwin":
-        pytest.skip("Skipping on macOS CI due to MPS environment issues with all-MiniLM-L6-v2")
-
-    from leann import LeannBuilder, LeannChat
-
-    with tempfile.TemporaryDirectory() as temp_dir:
-        # Build a simple index
-        index_path = str(Path(temp_dir) / "test.leann")
-        # Use smaller model in CI to avoid memory issues
-        if os.environ.get("CI") == "true":
-            builder = LeannBuilder(
-                backend_name="hnsw",
-                embedding_model="sentence-transformers/all-MiniLM-L6-v2",
-                dimensions=384,
-            )
-        else:
-            builder = LeannBuilder(backend_name="hnsw")
-        builder.add_text("Test document for LLM testing")
-        builder.build_index(index_path)
-
-        # Test simulated LLM config
-        llm_config = {"type": "simulated"}
-        chat = LeannChat(index_path, llm_config=llm_config)
-        response = chat.ask("What is this document about?", top_k=1)
-
-        assert isinstance(response, str)
-        assert len(response) > 0
-
-
-@pytest.mark.skip(reason="Requires HF model download and may timeout")
-def test_llm_config_hf():
-    """Test HuggingFace LLM configuration option."""
-    from leann import LeannBuilder, LeannChat
-
-    pytest.importorskip("transformers")  # Skip if transformers not installed
-
-    with tempfile.TemporaryDirectory() as temp_dir:
-        # Build a simple index
-        index_path = str(Path(temp_dir) / "test.leann")
-        builder = LeannBuilder(backend_name="hnsw")
-        builder.add_text("Test document for LLM testing")
-        builder.build_index(index_path)
-
-        # Test HF LLM config
-        llm_config = {"type": "hf", "model": "Qwen/Qwen3-0.6B"}
-        chat = LeannChat(index_path, llm_config=llm_config)
-        response = chat.ask("What is this document about?", top_k=1)
-
-        assert isinstance(response, str)
-        assert len(response) > 0