fix readme

examples/dynamic_update_no_recompute.py

@@ -0,0 +1,429 @@
+"""Dynamic HNSW update demo without compact storage.
+
+This script reproduces the minimal scenario we used while debugging on-the-fly
+recompute:
+
+1. Build a non-compact HNSW index from the first few paragraphs of a text file.
+2. Print the top results with `recompute_embeddings=True`.
+3. Append additional paragraphs with :meth:`LeannBuilder.update_index`.
+4. Run the same query again to show the newly inserted passages.
+
+Run it with ``uv`` (optionally pointing LEANN_HNSW_LOG_PATH at a file to inspect
+ZMQ activity)::
+
+    LEANN_HNSW_LOG_PATH=embedding_fetch.log \
+    uv run -m examples.dynamic_update_no_recompute \
+      --index-path .leann/examples/leann-demo.leann
+
+By default the script builds an index from ``data/2501.14312v1 (1).pdf`` and
+then updates it with LEANN-related material from ``data/2506.08276v1.pdf``.
+It issues the query "What's LEANN?" before and after the update to show how the
+new passages become immediately searchable. The script uses the
+``sentence-transformers/all-MiniLM-L6-v2`` model with ``is_recompute=True`` so
+Faiss pulls existing vectors on demand via the ZMQ embedding server, while
+freshly added passages are embedded locally just like the initial build.
+
+To make storage comparisons easy, the script can also build a matching
+``is_recompute=False`` baseline (enabled by default) and report the index size
+delta after the update. Disable the baseline run with
+``--skip-compare-no-recompute`` if you only need the recompute flow.
+"""
+
+import argparse
+import json
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Any
+
+from leann.api import LeannBuilder, LeannSearcher
+from leann.registry import register_project_directory
+
+from apps.chunking import create_text_chunks
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+
+DEFAULT_QUERY = "What's LEANN?"
+DEFAULT_INITIAL_FILES = [
+    REPO_ROOT / "data" / "2501.14312v1 (1).pdf",
+    REPO_ROOT / "data" / "huawei_pangu.md",
+    REPO_ROOT / "data" / "PrideandPrejudice.txt",
+]
+DEFAULT_UPDATE_FILES = [REPO_ROOT / "data" / "2506.08276v1.pdf"]
+
+
+def load_chunks_from_files(paths: list[Path]) -> list[str]:
+    from llama_index.core import SimpleDirectoryReader
+
+    documents = []
+    for path in paths:
+        p = path.expanduser().resolve()
+        if not p.exists():
+            raise FileNotFoundError(f"Input path not found: {p}")
+        if p.is_dir():
+            reader = SimpleDirectoryReader(str(p), recursive=False)
+            documents.extend(reader.load_data(show_progress=True))
+        else:
+            reader = SimpleDirectoryReader(input_files=[str(p)])
+            documents.extend(reader.load_data(show_progress=True))
+
+    if not documents:
+        return []
+
+    chunks = create_text_chunks(
+        documents,
+        chunk_size=512,
+        chunk_overlap=128,
+        use_ast_chunking=False,
+    )
+    return [c for c in chunks if isinstance(c, str) and c.strip()]
+
+
+def run_search(index_path: Path, query: str, top_k: int, *, recompute_embeddings: bool) -> list:
+    searcher = LeannSearcher(str(index_path))
+    try:
+        return searcher.search(
+            query=query,
+            top_k=top_k,
+            recompute_embeddings=recompute_embeddings,
+            batch_size=16,
+        )
+    finally:
+        searcher.cleanup()
+
+
+def print_results(title: str, results: Iterable) -> None:
+    print(f"\n=== {title} ===")
+    res_list = list(results)
+    print(f"results count: {len(res_list)}")
+    print("passages:")
+    if not res_list:
+        print("  (no passages returned)")
+    for res in res_list:
+        snippet = res.text.replace("\n", " ")[:120]
+        print(f"  - {res.id}: {snippet}... (score={res.score:.4f})")
+
+
+def build_initial_index(
+    index_path: Path,
+    paragraphs: list[str],
+    model_name: str,
+    embedding_mode: str,
+    is_recompute: bool,
+) -> None:
+    builder = LeannBuilder(
+        backend_name="hnsw",
+        embedding_model=model_name,
+        embedding_mode=embedding_mode,
+        is_compact=False,
+        is_recompute=is_recompute,
+    )
+    for idx, passage in enumerate(paragraphs):
+        builder.add_text(passage, metadata={"id": str(idx)})
+    builder.build_index(str(index_path))
+
+
+def update_index(
+    index_path: Path,
+    start_id: int,
+    paragraphs: list[str],
+    model_name: str,
+    embedding_mode: str,
+    is_recompute: bool,
+) -> None:
+    updater = LeannBuilder(
+        backend_name="hnsw",
+        embedding_model=model_name,
+        embedding_mode=embedding_mode,
+        is_compact=False,
+        is_recompute=is_recompute,
+    )
+    for offset, passage in enumerate(paragraphs, start=start_id):
+        updater.add_text(passage, metadata={"id": str(offset)})
+    updater.update_index(str(index_path))
+
+
+def ensure_index_dir(index_path: Path) -> None:
+    index_path.parent.mkdir(parents=True, exist_ok=True)
+
+
+def cleanup_index_files(index_path: Path) -> None:
+    """Remove leftover index artifacts for a clean rebuild."""
+
+    parent = index_path.parent
+    if not parent.exists():
+        return
+    stem = index_path.stem
+    for file in parent.glob(f"{stem}*"):
+        if file.is_file():
+            file.unlink()
+
+
+def index_file_size(index_path: Path) -> int:
+    """Return the size of the primary .index file for the given index path."""
+
+    index_file = index_path.parent / f"{index_path.stem}.index"
+    return index_file.stat().st_size if index_file.exists() else 0
+
+
+def load_metadata_snapshot(index_path: Path) -> dict[str, Any] | None:
+    meta_path = index_path.parent / f"{index_path.name}.meta.json"
+    if not meta_path.exists():
+        return None
+    try:
+        return json.loads(meta_path.read_text())
+    except json.JSONDecodeError:
+        return None
+
+
+def run_workflow(
+    *,
+    label: str,
+    index_path: Path,
+    initial_paragraphs: list[str],
+    update_paragraphs: list[str],
+    model_name: str,
+    embedding_mode: str,
+    is_recompute: bool,
+    query: str,
+    top_k: int,
+    skip_search: bool,
+) -> dict[str, Any]:
+    prefix = f"[{label}] " if label else ""
+
+    ensure_index_dir(index_path)
+    cleanup_index_files(index_path)
+
+    print(f"{prefix}Building initial index...")
+    build_initial_index(
+        index_path,
+        initial_paragraphs,
+        model_name,
+        embedding_mode,
+        is_recompute=is_recompute,
+    )
+
+    initial_size = index_file_size(index_path)
+    if not skip_search:
+        before_results = run_search(
+            index_path,
+            query,
+            top_k,
+            recompute_embeddings=is_recompute,
+        )
+    else:
+        before_results = None
+
+    print(f"\n{prefix}Updating index with additional passages...")
+    update_index(
+        index_path,
+        start_id=len(initial_paragraphs),
+        paragraphs=update_paragraphs,
+        model_name=model_name,
+        embedding_mode=embedding_mode,
+        is_recompute=is_recompute,
+    )
+
+    if not skip_search:
+        after_results = run_search(
+            index_path,
+            query,
+            top_k,
+            recompute_embeddings=is_recompute,
+        )
+    else:
+        after_results = None
+    updated_size = index_file_size(index_path)
+
+    return {
+        "initial_size": initial_size,
+        "updated_size": updated_size,
+        "delta": updated_size - initial_size,
+        "before_results": before_results if not skip_search else None,
+        "after_results": after_results if not skip_search else None,
+        "metadata": load_metadata_snapshot(index_path),
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--initial-files",
+        type=Path,
+        nargs="+",
+        default=DEFAULT_INITIAL_FILES,
+        help="Initial document files (PDF/TXT) used to build the base index",
+    )
+    parser.add_argument(
+        "--index-path",
+        type=Path,
+        default=Path(".leann/examples/leann-demo.leann"),
+        help="Destination index path (default: .leann/examples/leann-demo.leann)",
+    )
+    parser.add_argument(
+        "--initial-count",
+        type=int,
+        default=8,
+        help="Number of chunks to use from the initial documents (default: 8)",
+    )
+    parser.add_argument(
+        "--update-files",
+        type=Path,
+        nargs="*",
+        default=DEFAULT_UPDATE_FILES,
+        help="Additional documents to add during update (PDF/TXT)",
+    )
+    parser.add_argument(
+        "--update-count",
+        type=int,
+        default=4,
+        help="Number of chunks to append from update documents (default: 4)",
+    )
+    parser.add_argument(
+        "--update-text",
+        type=str,
+        default=(
+            "LEANN (Lightweight Embedding ANN) is an indexing toolkit focused on "
+            "recompute-aware HNSW graphs, allowing embeddings to be regenerated "
+            "on demand to keep disk usage minimal."
+        ),
+        help="Fallback text to append if --update-files is omitted",
+    )
+    parser.add_argument(
+        "--top-k",
+        type=int,
+        default=4,
+        help="Number of results to show for each search (default: 4)",
+    )
+    parser.add_argument(
+        "--query",
+        type=str,
+        default=DEFAULT_QUERY,
+        help="Query to run before/after the update",
+    )
+    parser.add_argument(
+        "--embedding-model",
+        type=str,
+        default="sentence-transformers/all-MiniLM-L6-v2",
+        help="Embedding model name",
+    )
+    parser.add_argument(
+        "--embedding-mode",
+        type=str,
+        default="sentence-transformers",
+        choices=["sentence-transformers", "openai", "mlx", "ollama"],
+        help="Embedding backend mode",
+    )
+    parser.add_argument(
+        "--compare-no-recompute",
+        dest="compare_no_recompute",
+        action="store_true",
+        help="Also run a baseline with is_recompute=False and report its index growth.",
+    )
+    parser.add_argument(
+        "--skip-compare-no-recompute",
+        dest="compare_no_recompute",
+        action="store_false",
+        help="Skip building the no-recompute baseline.",
+    )
+    parser.add_argument(
+        "--skip-search",
+        dest="skip_search",
+        action="store_true",
+        help="Skip the search step.",
+    )
+    parser.set_defaults(compare_no_recompute=True)
+    args = parser.parse_args()
+
+    ensure_index_dir(args.index_path)
+    register_project_directory(REPO_ROOT)
+
+    initial_chunks = load_chunks_from_files(list(args.initial_files))
+    if not initial_chunks:
+        raise ValueError("No text chunks extracted from the initial files.")
+
+    initial = initial_chunks[: args.initial_count]
+    if not initial:
+        raise ValueError("Initial chunk set is empty after applying --initial-count.")
+
+    if args.update_files:
+        update_chunks = load_chunks_from_files(list(args.update_files))
+        if not update_chunks:
+            raise ValueError("No text chunks extracted from the update files.")
+        to_add = update_chunks[: args.update_count]
+    else:
+        if not args.update_text:
+            raise ValueError("Provide --update-files or --update-text for the update step.")
+        to_add = [args.update_text]
+    if not to_add:
+        raise ValueError("Update chunk set is empty after applying --update-count.")
+
+    recompute_stats = run_workflow(
+        label="recompute",
+        index_path=args.index_path,
+        initial_paragraphs=initial,
+        update_paragraphs=to_add,
+        model_name=args.embedding_model,
+        embedding_mode=args.embedding_mode,
+        is_recompute=True,
+        query=args.query,
+        top_k=args.top_k,
+        skip_search=args.skip_search,
+    )
+
+    if not args.skip_search:
+        print_results("initial search", recompute_stats["before_results"])
+    if not args.skip_search:
+        print_results("after update", recompute_stats["after_results"])
+    print(
+        f"\n[recompute] Index file size change: {recompute_stats['initial_size']} -> {recompute_stats['updated_size']} bytes"
+        f" (Δ {recompute_stats['delta']})"
+    )
+
+    if recompute_stats["metadata"]:
+        meta_view = {k: recompute_stats["metadata"].get(k) for k in ("is_compact", "is_pruned")}
+        print("[recompute] metadata snapshot:")
+        print(json.dumps(meta_view, indent=2))
+
+    if args.compare_no_recompute:
+        baseline_path = (
+            args.index_path.parent / f"{args.index_path.stem}-norecompute{args.index_path.suffix}"
+        )
+        baseline_stats = run_workflow(
+            label="no-recompute",
+            index_path=baseline_path,
+            initial_paragraphs=initial,
+            update_paragraphs=to_add,
+            model_name=args.embedding_model,
+            embedding_mode=args.embedding_mode,
+            is_recompute=False,
+            query=args.query,
+            top_k=args.top_k,
+            skip_search=args.skip_search,
+        )
+
+        print(
+            f"\n[no-recompute] Index file size change: {baseline_stats['initial_size']} -> {baseline_stats['updated_size']} bytes"
+            f" (Δ {baseline_stats['delta']})"
+        )
+
+        after_texts = (
+            [res.text for res in recompute_stats["after_results"]] if not args.skip_search else None
+        )
+        baseline_after_texts = (
+            [res.text for res in baseline_stats["after_results"]] if not args.skip_search else None
+        )
+        if after_texts == baseline_after_texts:
+            print(
+                "[no-recompute] Search results match recompute baseline; see above for the shared output."
+            )
+        else:
+            print("[no-recompute] WARNING: search results differ from recompute baseline.")
+
+        if baseline_stats["metadata"]:
+            meta_view = {k: baseline_stats["metadata"].get(k) for k in ("is_compact", "is_pruned")}
+            print("[no-recompute] metadata snapshot:")
+            print(json.dumps(meta_view, indent=2))
+
+
+if __name__ == "__main__":
+    main()

examples/mcp_integration_demo.py

@@ -0,0 +1,178 @@
+#!/usr/bin/env python3
+"""
+MCP Integration Examples for LEANN
+
+This script demonstrates how to use LEANN with different MCP servers for
+RAG on various platforms like Slack and Twitter.
+
+Examples:
+1. Slack message RAG via MCP
+2. Twitter bookmark RAG via MCP
+3. Testing MCP server connections
+"""
+
+import asyncio
+import sys
+from pathlib import Path
+
+# Add the parent directory to the path so we can import from apps
+sys.path.append(str(Path(__file__).parent.parent))
+
+
+async def demo_slack_mcp():
+    """Demonstrate Slack MCP integration."""
+    print("=" * 60)
+    print("🔥 Slack MCP RAG Demo")
+    print("=" * 60)
+
+    print("\n1. Testing Slack MCP server connection...")
+
+    # This would typically use a real MCP server command
+    # For demo purposes, we show what the command would look like
+    # slack_app = SlackMCPRAG()  # Would be used for actual testing
+
+    # Simulate command line arguments for testing
+    class MockArgs:
+        mcp_server = "slack-mcp-server"  # This would be the actual MCP server command
+        workspace_name = "my-workspace"
+        channels = ["general", "random", "dev-team"]
+        no_concatenate_conversations = False
+        max_messages_per_channel = 50
+        test_connection = True
+
+    print(f"MCP Server Command: {MockArgs.mcp_server}")
+    print(f"Workspace: {MockArgs.workspace_name}")
+    print(f"Channels: {', '.join(MockArgs.channels)}")
+
+    # In a real scenario, you would run:
+    # success = await slack_app.test_mcp_connection(MockArgs)
+
+    print("\n📝 Example usage:")
+    print("python -m apps.slack_rag \\")
+    print("  --mcp-server 'slack-mcp-server' \\")
+    print("  --workspace-name 'my-team' \\")
+    print("  --channels general dev-team \\")
+    print("  --test-connection")
+
+    print("\n🔍 After indexing, you could query:")
+    print("- 'What did the team discuss about the project deadline?'")
+    print("- 'Find messages about the new feature launch'")
+    print("- 'Show me conversations about budget planning'")
+
+
+async def demo_twitter_mcp():
+    """Demonstrate Twitter MCP integration."""
+    print("\n" + "=" * 60)
+    print("🐦 Twitter MCP RAG Demo")
+    print("=" * 60)
+
+    print("\n1. Testing Twitter MCP server connection...")
+
+    # twitter_app = TwitterMCPRAG()  # Would be used for actual testing
+
+    class MockArgs:
+        mcp_server = "twitter-mcp-server"
+        username = None  # Fetch all bookmarks
+        max_bookmarks = 500
+        no_tweet_content = False
+        no_metadata = False
+        test_connection = True
+
+    print(f"MCP Server Command: {MockArgs.mcp_server}")
+    print(f"Max Bookmarks: {MockArgs.max_bookmarks}")
+    print(f"Include Content: {not MockArgs.no_tweet_content}")
+    print(f"Include Metadata: {not MockArgs.no_metadata}")
+
+    print("\n📝 Example usage:")
+    print("python -m apps.twitter_rag \\")
+    print("  --mcp-server 'twitter-mcp-server' \\")
+    print("  --max-bookmarks 1000 \\")
+    print("  --test-connection")
+
+    print("\n🔍 After indexing, you could query:")
+    print("- 'What AI articles did I bookmark last month?'")
+    print("- 'Find tweets about machine learning techniques'")
+    print("- 'Show me bookmarked threads about startup advice'")
+
+
+async def show_mcp_server_setup():
+    """Show how to set up MCP servers."""
+    print("\n" + "=" * 60)
+    print("⚙️  MCP Server Setup Guide")
+    print("=" * 60)
+
+    print("\n🔧 Setting up Slack MCP Server:")
+    print("1. Install a Slack MCP server (example commands):")
+    print("   npm install -g slack-mcp-server")
+    print("   # OR")
+    print("   pip install slack-mcp-server")
+
+    print("\n2. Configure Slack credentials:")
+    print("   export SLACK_BOT_TOKEN='xoxb-your-bot-token'")
+    print("   export SLACK_APP_TOKEN='xapp-your-app-token'")
+
+    print("\n3. Test the server:")
+    print("   slack-mcp-server --help")
+
+    print("\n🔧 Setting up Twitter MCP Server:")
+    print("1. Install a Twitter MCP server:")
+    print("   npm install -g twitter-mcp-server")
+    print("   # OR")
+    print("   pip install twitter-mcp-server")
+
+    print("\n2. Configure Twitter API credentials:")
+    print("   export TWITTER_API_KEY='your-api-key'")
+    print("   export TWITTER_API_SECRET='your-api-secret'")
+    print("   export TWITTER_ACCESS_TOKEN='your-access-token'")
+    print("   export TWITTER_ACCESS_TOKEN_SECRET='your-access-token-secret'")
+
+    print("\n3. Test the server:")
+    print("   twitter-mcp-server --help")
+
+
+async def show_integration_benefits():
+    """Show the benefits of MCP integration."""
+    print("\n" + "=" * 60)
+    print("🌟 Benefits of MCP Integration")
+    print("=" * 60)
+
+    benefits = [
+        ("🔄 Live Data Access", "Fetch real-time data from platforms without manual exports"),
+        ("🔌 Standardized Protocol", "Use any MCP-compatible server with minimal code changes"),
+        ("🚀 Easy Extension", "Add new platforms by implementing MCP readers"),
+        ("🔒 Secure Access", "MCP servers handle authentication and API management"),
+        ("📊 Rich Metadata", "Access full platform metadata (timestamps, engagement, etc.)"),
+        ("⚡ Efficient Processing", "Stream data directly into LEANN without intermediate files"),
+    ]
+
+    for title, description in benefits:
+        print(f"\n{title}")
+        print(f"   {description}")
+
+
+async def main():
+    """Main demo function."""
+    print("🎯 LEANN MCP Integration Examples")
+    print("This demo shows how to integrate LEANN with MCP servers for various platforms.")
+
+    await demo_slack_mcp()
+    await demo_twitter_mcp()
+    await show_mcp_server_setup()
+    await show_integration_benefits()
+
+    print("\n" + "=" * 60)
+    print("✨ Next Steps")
+    print("=" * 60)
+    print("1. Install and configure MCP servers for your platforms")
+    print("2. Test connections using --test-connection flag")
+    print("3. Run indexing to build your RAG knowledge base")
+    print("4. Start querying your personal data!")
+
+    print("\n📚 For more information:")
+    print("- Check the README for detailed setup instructions")
+    print("- Look at the apps/slack_rag.py and apps/twitter_rag.py for implementation details")
+    print("- Explore other MCP servers for additional platforms")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())