site-packages back

- Remove unused variables in benchmark scripts
- Rename unused loop variables to follow convention

.gitignore

@@ -99,9 +99,3 @@ benchmarks/data/
 
 ## multi vector
 apps/multimodal/vision-based-pdf-multi-vector/multi-vector-colpali-native-weaviate.py
-
-# Ignore all PDFs (keep data exceptions above) and do not track demo PDFs
-# If you need to commit a specific demo PDF, remove this negation locally.
-# The following line used to force-add a large demo PDF; remove it to satisfy pre-commit:
-# !apps/multimodal/vision-based-pdf-multi-vector/pdfs/2004.12832v2.pdf
-!apps/multimodal/vision-based-pdf-multi-vector/fig/*

README.md

@@ -8,12 +8,8 @@
   <img src="https://img.shields.io/badge/Platform-Ubuntu%20%26%20Arch%20%26%20WSL%20%7C%20macOS%20(ARM64%2FIntel)-lightgrey" alt="Platform">
   <img src="https://img.shields.io/badge/License-MIT-green.svg" alt="MIT License">
   <img src="https://img.shields.io/badge/MCP-Native%20Integration-blue" alt="MCP Integration">
-  <a href="https://join.slack.com/t/leann-e2u9779/shared_invite/zt-3ckd2f6w1-OX08~NN4gkWhh10PRVBj1Q">
-    <img src="https://img.shields.io/badge/Slack-Join-4A154B?logo=slack&logoColor=white" alt="Join Slack">
-  </a>
-  <a href="assets/wechat_user_group.JPG" title="Join WeChat group">
-    <img src="https://img.shields.io/badge/WeChat-Join-2DC100?logo=wechat&logoColor=white" alt="Join WeChat group">
-  </a>
+  <a href="https://join.slack.com/t/leann-e2u9779/shared_invite/zt-3ckd2f6w1-OX08~NN4gkWhh10PRVBj1Q"><img src="https://img.shields.io/badge/Slack-Join-4A154B?logo=slack&logoColor=white" alt="Join Slack">
+  <a href="assets/wechat_user_group.JPG" title="Join WeChat group"><img src="https://img.shields.io/badge/WeChat-Join-2DC100?logo=wechat&logoColor=white" alt="Join WeChat group"></a>
 </p>
 
 <h2 align="center" tabindex="-1" class="heading-element" dir="auto">
@@ -24,7 +20,7 @@ LEANN is an innovative vector database that democratizes personal AI. Transform
 
 LEANN achieves this through *graph-based selective recomputation* with *high-degree preserving pruning*, computing embeddings on-demand instead of storing them all. [Illustration Fig →](#️-architecture--how-it-works) | [Paper →](https://arxiv.org/abs/2506.08276)
 
-**Ready to RAG Everything?** Transform your laptop into a personal AI assistant that can semantic search your **[file system](#-personal-data-manager-process-any-documents-pdf-txt-md)**, **[emails](#-your-personal-email-secretary-rag-on-apple-mail)**, **[browser history](#-time-machine-for-the-web-rag-your-entire-browser-history)**, **[chat history](#-wechat-detective-unlock-your-golden-memories)** ([WeChat](#-wechat-detective-unlock-your-golden-memories), [iMessage](#-imessage-history-your-personal-conversation-archive)), **[agent memory](#-chatgpt-chat-history-your-personal-ai-conversation-archive)** ([ChatGPT](#-chatgpt-chat-history-your-personal-ai-conversation-archive), [Claude](#-claude-chat-history-your-personal-ai-conversation-archive)), **[live data](#mcp-integration-rag-on-live-data-from-any-platform)** ([Slack](#mcp-integration-rag-on-live-data-from-any-platform), [Twitter](#mcp-integration-rag-on-live-data-from-any-platform)), **[codebase](#-claude-code-integration-transform-your-development-workflow)**\* , or external knowledge bases (i.e., 60M documents) - all on your laptop, with zero cloud costs and complete privacy.
+**Ready to RAG Everything?** Transform your laptop into a personal AI assistant that can semantic search your **[file system](#-personal-data-manager-process-any-documents-pdf-txt-md)**, **[emails](#-your-personal-email-secretary-rag-on-apple-mail)**, **[browser history](#-time-machine-for-the-web-rag-your-entire-browser-history)**, **[chat history](#-wechat-detective-unlock-your-golden-memories)**, **[codebase](#-claude-code-integration-transform-your-development-workflow)**\* , or external knowledge bases (i.e., 60M documents) - all on your laptop, with zero cloud costs and complete privacy.
 
 
 \* Claude Code only supports basic `grep`-style keyword search. **LEANN** is a drop-in **semantic search MCP service fully compatible with Claude Code**, unlocking intelligent retrieval without changing your workflow. 🔥 Check out [the easy setup →](packages/leann-mcp/README.md)
@@ -76,9 +72,8 @@ uv venv
 source .venv/bin/activate
 uv pip install leann
 ```
-
 <!--
-> Low-resource? See "Low-resource setups" in the [Configuration Guide](docs/configuration-guide.md#low-resource-setups). -->
+> Low-resource? See “Low-resource setups” in the [Configuration Guide](docs/configuration-guide.md#low-resource-setups). -->
 
 <details>
 <summary>
@@ -181,7 +176,7 @@ response = chat.ask("How much storage does LEANN save?", top_k=1)
 
 ## RAG on Everything!
 
-LEANN supports RAG on various data sources including documents (`.pdf`, `.txt`, `.md`), Apple Mail, Google Search History, WeChat, ChatGPT conversations, Claude conversations, iMessage conversations, and **live data from any platform through MCP (Model Context Protocol) servers** - including Slack, Twitter, and more.
+LEANN supports RAG on various data sources including documents (`.pdf`, `.txt`, `.md`), Apple Mail, Google Search History, WeChat, and more.
 
 
 
@@ -547,386 +542,10 @@ Once the index is built, you can ask questions like:
 
 </details>
 
-### 🤖 ChatGPT Chat History: Your Personal AI Conversation Archive!
-
-Transform your ChatGPT conversations into a searchable knowledge base! Search through all your ChatGPT discussions about coding, research, brainstorming, and more.
-
-```bash
-python -m apps.chatgpt_rag --export-path chatgpt_export.html --query "How do I create a list in Python?"
-```
-
-**Unlock your AI conversation history.** Never lose track of valuable insights from your ChatGPT discussions again.
-
-<details>
-<summary><strong>📋 Click to expand: How to Export ChatGPT Data</strong></summary>
-
-**Step-by-step export process:**
-
-1. **Sign in to ChatGPT**
-2. **Click your profile icon** in the top right corner
-3. **Navigate to Settings** → **Data Controls**
-4. **Click "Export"** under Export Data
-5. **Confirm the export** request
-6. **Download the ZIP file** from the email link (expires in 24 hours)
-7. **Extract or use directly** with LEANN
-
-**Supported formats:**
-- `.html` files from ChatGPT exports
-- `.zip` archives from ChatGPT
-- Directories with multiple export files
-
-</details>
-
-<details>
-<summary><strong>📋 Click to expand: ChatGPT-Specific Arguments</strong></summary>
-
-#### Parameters
-```bash
---export-path PATH           # Path to ChatGPT export file (.html/.zip) or directory (default: ./chatgpt_export)
---separate-messages         # Process each message separately instead of concatenated conversations
---chunk-size N              # Text chunk size (default: 512)
---chunk-overlap N           # Overlap between chunks (default: 128)
-```
-
-#### Example Commands
-```bash
-# Basic usage with HTML export
-python -m apps.chatgpt_rag --export-path conversations.html
-
-# Process ZIP archive from ChatGPT
-python -m apps.chatgpt_rag --export-path chatgpt_export.zip
-
-# Search with specific query
-python -m apps.chatgpt_rag --export-path chatgpt_data.html --query "Python programming help"
-
-# Process individual messages for fine-grained search
-python -m apps.chatgpt_rag --separate-messages --export-path chatgpt_export.html
-
-# Process directory containing multiple exports
-python -m apps.chatgpt_rag --export-path ./chatgpt_exports/ --max-items 1000
-```
-
-</details>
-
-<details>
-<summary><strong>💡 Click to expand: Example queries you can try</strong></summary>
-
-Once your ChatGPT conversations are indexed, you can search with queries like:
-- "What did I ask ChatGPT about Python programming?"
-- "Show me conversations about machine learning algorithms"
-- "Find discussions about web development frameworks"
-- "What coding advice did ChatGPT give me?"
-- "Search for conversations about debugging techniques"
-- "Find ChatGPT's recommendations for learning resources"
-
-</details>
-
-### 🤖 Claude Chat History: Your Personal AI Conversation Archive!
-
-Transform your Claude conversations into a searchable knowledge base! Search through all your Claude discussions about coding, research, brainstorming, and more.
-
-```bash
-python -m apps.claude_rag --export-path claude_export.json --query "What did I ask about Python dictionaries?"
-```
-
-**Unlock your AI conversation history.** Never lose track of valuable insights from your Claude discussions again.
-
-<details>
-<summary><strong>📋 Click to expand: How to Export Claude Data</strong></summary>
-
-**Step-by-step export process:**
-
-1. **Open Claude** in your browser
-2. **Navigate to Settings** (look for gear icon or settings menu)
-3. **Find Export/Download** options in your account settings
-4. **Download conversation data** (usually in JSON format)
-5. **Place the file** in your project directory
-
-*Note: Claude export methods may vary depending on the interface you're using. Check Claude's help documentation for the most current export instructions.*
-
-**Supported formats:**
-- `.json` files (recommended)
-- `.zip` archives containing JSON data
-- Directories with multiple export files
-
-</details>
-
-<details>
-<summary><strong>📋 Click to expand: Claude-Specific Arguments</strong></summary>
-
-#### Parameters
-```bash
---export-path PATH           # Path to Claude export file (.json/.zip) or directory (default: ./claude_export)
---separate-messages         # Process each message separately instead of concatenated conversations
---chunk-size N              # Text chunk size (default: 512)
---chunk-overlap N           # Overlap between chunks (default: 128)
-```
-
-#### Example Commands
-```bash
-# Basic usage with JSON export
-python -m apps.claude_rag --export-path my_claude_conversations.json
-
-# Process ZIP archive from Claude
-python -m apps.claude_rag --export-path claude_export.zip
-
-# Search with specific query
-python -m apps.claude_rag --export-path claude_data.json --query "machine learning advice"
-
-# Process individual messages for fine-grained search
-python -m apps.claude_rag --separate-messages --export-path claude_export.json
-
-# Process directory containing multiple exports
-python -m apps.claude_rag --export-path ./claude_exports/ --max-items 1000
-```
-
-</details>
-
-<details>
-<summary><strong>💡 Click to expand: Example queries you can try</strong></summary>
-
-Once your Claude conversations are indexed, you can search with queries like:
-- "What did I ask Claude about Python programming?"
-- "Show me conversations about machine learning algorithms"
-- "Find discussions about software architecture patterns"
-- "What debugging advice did Claude give me?"
-- "Search for conversations about data structures"
-- "Find Claude's recommendations for learning resources"
-
-</details>
-
-### 💬 iMessage History: Your Personal Conversation Archive!
-
-Transform your iMessage conversations into a searchable knowledge base! Search through all your text messages, group chats, and conversations with friends, family, and colleagues.
-
-```bash
-python -m apps.imessage_rag --query "What did we discuss about the weekend plans?"
-```
-
-**Unlock your message history.** Never lose track of important conversations, shared links, or memorable moments from your iMessage history.
-
-<details>
-<summary><strong>📋 Click to expand: How to Access iMessage Data</strong></summary>
-
-**iMessage data location:**
-
-iMessage conversations are stored in a SQLite database on your Mac at:
-```
-~/Library/Messages/chat.db
-```
-
-**Important setup requirements:**
-
-1. **Grant Full Disk Access** to your terminal or IDE:
-   - Open **System Preferences** → **Security & Privacy** → **Privacy**
-   - Select **Full Disk Access** from the left sidebar
-   - Click the **+** button and add your terminal app (Terminal, iTerm2) or IDE (VS Code, etc.)
-   - Restart your terminal/IDE after granting access
-
-2. **Alternative: Use a backup database**
-   - If you have Time Machine backups or manual copies of the database
-   - Use `--db-path` to specify a custom location
-
-**Supported formats:**
-- Direct access to `~/Library/Messages/chat.db` (default)
-- Custom database path with `--db-path`
-- Works with backup copies of the database
-
-</details>
-
-<details>
-<summary><strong>📋 Click to expand: iMessage-Specific Arguments</strong></summary>
-
-#### Parameters
-```bash
---db-path PATH                    # Path to chat.db file (default: ~/Library/Messages/chat.db)
---concatenate-conversations       # Group messages by conversation (default: True)
---no-concatenate-conversations    # Process each message individually
---chunk-size N                    # Text chunk size (default: 1000)
---chunk-overlap N                 # Overlap between chunks (default: 200)
-```
-
-#### Example Commands
-```bash
-# Basic usage (requires Full Disk Access)
-python -m apps.imessage_rag
-
-# Search with specific query
-python -m apps.imessage_rag --query "family dinner plans"
-
-# Use custom database path
-python -m apps.imessage_rag --db-path /path/to/backup/chat.db
-
-# Process individual messages instead of conversations
-python -m apps.imessage_rag --no-concatenate-conversations
-
-# Limit processing for testing
-python -m apps.imessage_rag --max-items 100 --query "weekend"
-```
-
-</details>
-
-<details>
-<summary><strong>💡 Click to expand: Example queries you can try</strong></summary>
-
-Once your iMessage conversations are indexed, you can search with queries like:
-- "What did we discuss about vacation plans?"
-- "Find messages about restaurant recommendations"
-- "Show me conversations with John about the project"
-- "Search for shared links about technology"
-- "Find group chat discussions about weekend events"
-- "What did mom say about the family gathering?"
-
-</details>
-
-### MCP Integration: RAG on Live Data from Any Platform
-
-Connect to live data sources through the Model Context Protocol (MCP). LEANN now supports real-time RAG on platforms like Slack, Twitter, and more through standardized MCP servers.
-
-**Key Benefits:**
-- **Live Data Access**: Fetch real-time data without manual exports
-- **Standardized Protocol**: Use any MCP-compatible server
-- **Easy Extension**: Add new platforms with minimal code
-- **Secure Access**: MCP servers handle authentication
-
-#### 💬 Slack Messages: Search Your Team Conversations
-
-Transform your Slack workspace into a searchable knowledge base! Find discussions, decisions, and shared knowledge across all your channels.
-
-```bash
-# Test MCP server connection
-python -m apps.slack_rag --mcp-server "slack-mcp-server" --test-connection
-
-# Index and search Slack messages
-python -m apps.slack_rag \
-  --mcp-server "slack-mcp-server" \
-  --workspace-name "my-team" \
-  --channels general dev-team random \
-  --query "What did we decide about the product launch?"
-```
-
-**📖 Comprehensive Setup Guide**: For detailed setup instructions, troubleshooting common issues (like "users cache is not ready yet"), and advanced configuration options, see our [**Slack Setup Guide**](docs/slack-setup-guide.md).
-
-**Quick Setup:**
-1. Install a Slack MCP server (e.g., `npm install -g slack-mcp-server`)
-2. Create a Slack App and get API credentials (see detailed guide above)
-3. Set environment variables:
-   ```bash
-   export SLACK_BOT_TOKEN="xoxb-your-bot-token"
-   export SLACK_APP_TOKEN="xapp-your-app-token"  # Optional
-   ```
-4. Test connection with `--test-connection` flag
-
-**Arguments:**
-- `--mcp-server`: Command to start the Slack MCP server
-- `--workspace-name`: Slack workspace name for organization
-- `--channels`: Specific channels to index (optional)
-- `--concatenate-conversations`: Group messages by channel (default: true)
-- `--max-messages-per-channel`: Limit messages per channel (default: 100)
-- `--max-retries`: Maximum retries for cache sync issues (default: 5)
-- `--retry-delay`: Initial delay between retries in seconds (default: 2.0)
-
-#### 🐦 Twitter Bookmarks: Your Personal Tweet Library
-
-Search through your Twitter bookmarks! Find that perfect article, thread, or insight you saved for later.
-
-```bash
-# Test MCP server connection
-python -m apps.twitter_rag --mcp-server "twitter-mcp-server" --test-connection
-
-# Index and search Twitter bookmarks
-python -m apps.twitter_rag \
-  --mcp-server "twitter-mcp-server" \
-  --max-bookmarks 1000 \
-  --query "What AI articles did I bookmark about machine learning?"
-```
-
-**Setup Requirements:**
-1. Install a Twitter MCP server (e.g., `npm install -g twitter-mcp-server`)
-2. Get Twitter API credentials:
-   - Apply for a Twitter Developer Account at [developer.twitter.com](https://developer.twitter.com)
-   - Create a new app in the Twitter Developer Portal
-   - Generate API keys and access tokens with "Read" permissions
-   - For bookmarks access, you may need Twitter API v2 with appropriate scopes
-   ```bash
-   export TWITTER_API_KEY="your-api-key"
-   export TWITTER_API_SECRET="your-api-secret"
-   export TWITTER_ACCESS_TOKEN="your-access-token"
-   export TWITTER_ACCESS_TOKEN_SECRET="your-access-token-secret"
-   ```
-3. Test connection with `--test-connection` flag
-
-**Arguments:**
-- `--mcp-server`: Command to start the Twitter MCP server
-- `--username`: Filter bookmarks by username (optional)
-- `--max-bookmarks`: Maximum bookmarks to fetch (default: 1000)
-- `--no-tweet-content`: Exclude tweet content, only metadata
-- `--no-metadata`: Exclude engagement metadata
-
-</details>
-
-<details>
-<summary><strong>💡 Click to expand: Example queries you can try</strong></summary>
-
-**Slack Queries:**
-- "What did the team discuss about the project deadline?"
-- "Find messages about the new feature launch"
-- "Show me conversations about budget planning"
-- "What decisions were made in the dev-team channel?"
-
-**Twitter Queries:**
-- "What AI articles did I bookmark last month?"
-- "Find tweets about machine learning techniques"
-- "Show me bookmarked threads about startup advice"
-- "What Python tutorials did I save?"
-
-</details>
-<summary><strong>🔧 Using MCP with CLI Commands</strong></summary>
-
-**Want to use MCP data with regular LEANN CLI?** You can combine MCP apps with CLI commands:
-
-```bash
-# Step 1: Use MCP app to fetch and index data
-python -m apps.slack_rag --mcp-server "slack-mcp-server" --workspace-name "my-team"
-
-# Step 2: The data is now indexed and available via CLI
-leann search slack_messages "project deadline"
-leann ask slack_messages "What decisions were made about the product launch?"
-
-# Same for Twitter bookmarks
-python -m apps.twitter_rag --mcp-server "twitter-mcp-server"
-leann search twitter_bookmarks "machine learning articles"
-```
-
-**MCP vs Manual Export:**
-- **MCP**: Live data, automatic updates, requires server setup
-- **Manual Export**: One-time setup, works offline, requires manual data export
-
-</details>
-
-<details>
-<summary><strong>🔧 Adding New MCP Platforms</strong></summary>
-
-Want to add support for other platforms? LEANN's MCP integration is designed for easy extension:
-
-1. **Find or create an MCP server** for your platform
-2. **Create a reader class** following the pattern in `apps/slack_data/slack_mcp_reader.py`
-3. **Create a RAG application** following the pattern in `apps/slack_rag.py`
-4. **Test and contribute** back to the community!
-
-**Popular MCP servers to explore:**
-- GitHub repositories and issues
-- Discord messages
-- Notion pages
-- Google Drive documents
-- And many more in the MCP ecosystem!
-
-</details>
-
 ### 🚀 Claude Code Integration: Transform Your Development Workflow!
 
 <details>
-<summary><strong>AST‑Aware Code Chunking</strong></summary>
+<summary><strong>NEW!! AST‑Aware Code Chunking</strong></summary>
 
 LEANN features intelligent code chunking that preserves semantic boundaries (functions, classes, methods) for Python, Java, C#, and TypeScript, improving code understanding compared to text-based chunking.
 
@@ -954,7 +573,7 @@ Try our fully agentic pipeline with auto query rewriting, semantic search planni
 
 **🔥 Ready to supercharge your coding?** [Complete Setup Guide →](packages/leann-mcp/README.md)
 
-## Command Line Interface
+## 🖥️ Command Line Interface
 
 LEANN includes a powerful CLI for document processing and search. Perfect for quick document indexing and interactive chat.
 
@@ -1196,7 +815,7 @@ MIT License - see [LICENSE](LICENSE) for details.
 
 Core Contributors: [Yichuan Wang](https://yichuan-w.github.io/) & [Zhifei Li](https://github.com/andylizf).
 
-Active Contributors: [Gabriel Dehan](https://github.com/gabriel-dehan), [Aakash Suresh](https://github.com/ASuresh0524)
+Active Contributors: [Gabriel Dehan](https://github.com/gabriel-dehan)
 
 
 We welcome more contributors! Feel free to open issues or submit PRs.

apps/base_rag_example.py

@@ -10,39 +10,8 @@ from typing import Any
 
 import dotenv
 from leann.api import LeannBuilder, LeannChat
-
-# Optional import: older PyPI builds may not include interactive_utils
-try:
-    from leann.interactive_utils import create_rag_session
-except ImportError:
-
-    def create_rag_session(app_name: str, data_description: str):
-        class _SimpleSession:
-            def run_interactive_loop(self, handler):
-                print(f"Interactive session for {app_name}: {data_description}")
-                print("Interactive mode not available in this build")
-
-        return _SimpleSession()
-
-
 from leann.registry import register_project_directory
-
-# Optional import: older PyPI builds may not include settings
-try:
-    from leann.settings import resolve_ollama_host, resolve_openai_api_key, resolve_openai_base_url
-except ImportError:
-    # Minimal fallbacks if settings helpers are unavailable
-    import os
-
-    def resolve_ollama_host(value: str | None) -> str | None:
-        return value or os.getenv("LEANN_OLLAMA_HOST") or os.getenv("OLLAMA_HOST")
-
-    def resolve_openai_api_key(value: str | None) -> str | None:
-        return value or os.getenv("OPENAI_API_KEY")
-
-    def resolve_openai_base_url(value: str | None) -> str | None:
-        return value or os.getenv("OPENAI_BASE_URL")
-
+from leann.settings import resolve_ollama_host, resolve_openai_api_key, resolve_openai_base_url
 
 dotenv.load_dotenv()
 
@@ -338,26 +307,37 @@ class BaseRAGExample(ABC):
             complexity=args.search_complexity,
         )
 
-        # Create interactive session
-        session = create_rag_session(
-            app_name=self.name.lower().replace(" ", "_"), data_description=self.name
-        )
+        print(f"\n[Interactive Mode] Chat with your {self.name} data!")
+        print("Type 'quit' or 'exit' to stop.\n")
 
-        def handle_query(query: str):
-            # Prepare LLM kwargs with thinking budget if specified
-            llm_kwargs = {}
-            if hasattr(args, "thinking_budget") and args.thinking_budget:
-                llm_kwargs["thinking_budget"] = args.thinking_budget
+        while True:
+            try:
+                query = input("You: ").strip()
+                if query.lower() in ["quit", "exit", "q"]:
+                    print("Goodbye!")
+                    break
 
-            response = chat.ask(
-                query,
-                top_k=args.top_k,
-                complexity=args.search_complexity,
-                llm_kwargs=llm_kwargs,
-            )
-            print(f"\nAssistant: {response}\n")
+                if not query:
+                    continue
 
-        session.run_interactive_loop(handle_query)
+                # Prepare LLM kwargs with thinking budget if specified
+                llm_kwargs = {}
+                if hasattr(args, "thinking_budget") and args.thinking_budget:
+                    llm_kwargs["thinking_budget"] = args.thinking_budget
+
+                response = chat.ask(
+                    query,
+                    top_k=args.top_k,
+                    complexity=args.search_complexity,
+                    llm_kwargs=llm_kwargs,
+                )
+                print(f"\nAssistant: {response}\n")
+
+            except KeyboardInterrupt:
+                print("\nGoodbye!")
+                break
+            except Exception as e:
+                print(f"Error: {e}")
 
     async def run_single_query(self, args, index_path: str, query: str):
         """Run a single query against the index."""

apps/chatgpt_data/chatgpt_reader.py

@@ -1,413 +0,0 @@
-"""
-ChatGPT export data reader.
-
-Reads and processes ChatGPT export data from chat.html files.
-"""
-
-import re
-from pathlib import Path
-from typing import Any
-from zipfile import ZipFile
-
-from bs4 import BeautifulSoup
-from llama_index.core import Document
-from llama_index.core.readers.base import BaseReader
-
-
-class ChatGPTReader(BaseReader):
-    """
-    ChatGPT export data reader.
-
-    Reads ChatGPT conversation data from exported chat.html files or zip archives.
-    Processes conversations into structured documents with metadata.
-    """
-
-    def __init__(self, concatenate_conversations: bool = True) -> None:
-        """
-        Initialize.
-
-        Args:
-            concatenate_conversations: Whether to concatenate messages within conversations for better context
-        """
-        try:
-            from bs4 import BeautifulSoup  # noqa
-        except ImportError:
-            raise ImportError("`beautifulsoup4` package not found: `pip install beautifulsoup4`")
-
-        self.concatenate_conversations = concatenate_conversations
-
-    def _extract_html_from_zip(self, zip_path: Path) -> str | None:
-        """
-        Extract chat.html from ChatGPT export zip file.
-
-        Args:
-            zip_path: Path to the ChatGPT export zip file
-
-        Returns:
-            HTML content as string, or None if not found
-        """
-        try:
-            with ZipFile(zip_path, "r") as zip_file:
-                # Look for chat.html or conversations.html
-                html_files = [
-                    f
-                    for f in zip_file.namelist()
-                    if f.endswith(".html") and ("chat" in f.lower() or "conversation" in f.lower())
-                ]
-
-                if not html_files:
-                    print(f"No HTML chat file found in {zip_path}")
-                    return None
-
-                # Use the first HTML file found
-                html_file = html_files[0]
-                print(f"Found HTML file: {html_file}")
-
-                with zip_file.open(html_file) as f:
-                    return f.read().decode("utf-8", errors="ignore")
-
-        except Exception as e:
-            print(f"Error extracting HTML from zip {zip_path}: {e}")
-            return None
-
-    def _parse_chatgpt_html(self, html_content: str) -> list[dict]:
-        """
-        Parse ChatGPT HTML export to extract conversations.
-
-        Args:
-            html_content: HTML content from ChatGPT export
-
-        Returns:
-            List of conversation dictionaries
-        """
-        soup = BeautifulSoup(html_content, "html.parser")
-        conversations = []
-
-        # Try different possible structures for ChatGPT exports
-        # Structure 1: Look for conversation containers
-        conversation_containers = soup.find_all(
-            ["div", "section"], class_=re.compile(r"conversation|chat", re.I)
-        )
-
-        if not conversation_containers:
-            # Structure 2: Look for message containers directly
-            conversation_containers = [soup]  # Use the entire document as one conversation
-
-        for container in conversation_containers:
-            conversation = self._extract_conversation_from_container(container)
-            if conversation and conversation.get("messages"):
-                conversations.append(conversation)
-
-        # If no structured conversations found, try to extract all text as one conversation
-        if not conversations:
-            all_text = soup.get_text(separator="\n", strip=True)
-            if all_text:
-                conversations.append(
-                    {
-                        "title": "ChatGPT Conversation",
-                        "messages": [{"role": "mixed", "content": all_text, "timestamp": None}],
-                        "timestamp": None,
-                    }
-                )
-
-        return conversations
-
-    def _extract_conversation_from_container(self, container) -> dict | None:
-        """
-        Extract conversation data from a container element.
-
-        Args:
-            container: BeautifulSoup element containing conversation
-
-        Returns:
-            Dictionary with conversation data or None
-        """
-        messages = []
-
-        # Look for message elements with various possible structures
-        message_selectors = ['[class*="message"]', '[class*="chat"]', "[data-message]", "p", "div"]
-
-        for selector in message_selectors:
-            message_elements = container.select(selector)
-            if message_elements:
-                break
-        else:
-            message_elements = []
-
-        # If no structured messages found, treat the entire container as one message
-        if not message_elements:
-            text_content = container.get_text(separator="\n", strip=True)
-            if text_content:
-                messages.append({"role": "mixed", "content": text_content, "timestamp": None})
-        else:
-            for element in message_elements:
-                message = self._extract_message_from_element(element)
-                if message:
-                    messages.append(message)
-
-        if not messages:
-            return None
-
-        # Try to extract conversation title
-        title_element = container.find(["h1", "h2", "h3", "title"])
-        title = title_element.get_text(strip=True) if title_element else "ChatGPT Conversation"
-
-        # Try to extract timestamp from various possible locations
-        timestamp = self._extract_timestamp_from_container(container)
-
-        return {"title": title, "messages": messages, "timestamp": timestamp}
-
-    def _extract_message_from_element(self, element) -> dict | None:
-        """
-        Extract message data from an element.
-
-        Args:
-            element: BeautifulSoup element containing message
-
-        Returns:
-            Dictionary with message data or None
-        """
-        text_content = element.get_text(separator=" ", strip=True)
-
-        # Skip empty or very short messages
-        if not text_content or len(text_content.strip()) < 3:
-            return None
-
-        # Try to determine role (user/assistant) from class names or content
-        role = "mixed"  # Default role
-
-        class_names = " ".join(element.get("class", [])).lower()
-        if "user" in class_names or "human" in class_names:
-            role = "user"
-        elif "assistant" in class_names or "ai" in class_names or "gpt" in class_names:
-            role = "assistant"
-        elif text_content.lower().startswith(("you:", "user:", "me:")):
-            role = "user"
-            text_content = re.sub(r"^(you|user|me):\s*", "", text_content, flags=re.IGNORECASE)
-        elif text_content.lower().startswith(("chatgpt:", "assistant:", "ai:")):
-            role = "assistant"
-            text_content = re.sub(
-                r"^(chatgpt|assistant|ai):\s*", "", text_content, flags=re.IGNORECASE
-            )
-
-        # Try to extract timestamp
-        timestamp = self._extract_timestamp_from_element(element)
-
-        return {"role": role, "content": text_content, "timestamp": timestamp}
-
-    def _extract_timestamp_from_element(self, element) -> str | None:
-        """Extract timestamp from element."""
-        # Look for timestamp in various attributes and child elements
-        timestamp_attrs = ["data-timestamp", "timestamp", "datetime"]
-        for attr in timestamp_attrs:
-            if element.get(attr):
-                return element.get(attr)
-
-        # Look for time elements
-        time_element = element.find("time")
-        if time_element:
-            return time_element.get("datetime") or time_element.get_text(strip=True)
-
-        # Look for date-like text patterns
-        text = element.get_text()
-        date_patterns = [r"\d{4}-\d{2}-\d{2}", r"\d{1,2}/\d{1,2}/\d{4}", r"\w+ \d{1,2}, \d{4}"]
-
-        for pattern in date_patterns:
-            match = re.search(pattern, text)
-            if match:
-                return match.group()
-
-        return None
-
-    def _extract_timestamp_from_container(self, container) -> str | None:
-        """Extract timestamp from conversation container."""
-        return self._extract_timestamp_from_element(container)
-
-    def _create_concatenated_content(self, conversation: dict) -> str:
-        """
-        Create concatenated content from conversation messages.
-
-        Args:
-            conversation: Dictionary containing conversation data
-
-        Returns:
-            Formatted concatenated content
-        """
-        title = conversation.get("title", "ChatGPT Conversation")
-        messages = conversation.get("messages", [])
-        timestamp = conversation.get("timestamp", "Unknown")
-
-        # Build message content
-        message_parts = []
-        for message in messages:
-            role = message.get("role", "mixed")
-            content = message.get("content", "")
-            msg_timestamp = message.get("timestamp", "")
-
-            if role == "user":
-                prefix = "[You]"
-            elif role == "assistant":
-                prefix = "[ChatGPT]"
-            else:
-                prefix = "[Message]"
-
-            # Add timestamp if available
-            if msg_timestamp:
-                prefix += f" ({msg_timestamp})"
-
-            message_parts.append(f"{prefix}: {content}")
-
-        concatenated_text = "\n\n".join(message_parts)
-
-        # Create final document content
-        doc_content = f"""Conversation: {title}
-Date: {timestamp}
-Messages ({len(messages)} messages):
-
-{concatenated_text}
-"""
-        return doc_content
-
-    def load_data(self, input_dir: str | None = None, **load_kwargs: Any) -> list[Document]:
-        """
-        Load ChatGPT export data.
-
-        Args:
-            input_dir: Directory containing ChatGPT export files or path to specific file
-            **load_kwargs:
-                max_count (int): Maximum number of conversations to process
-                chatgpt_export_path (str): Specific path to ChatGPT export file/directory
-                include_metadata (bool): Whether to include metadata in documents
-        """
-        docs: list[Document] = []
-        max_count = load_kwargs.get("max_count", -1)
-        chatgpt_export_path = load_kwargs.get("chatgpt_export_path", input_dir)
-        include_metadata = load_kwargs.get("include_metadata", True)
-
-        if not chatgpt_export_path:
-            print("No ChatGPT export path provided")
-            return docs
-
-        export_path = Path(chatgpt_export_path)
-
-        if not export_path.exists():
-            print(f"ChatGPT export path not found: {export_path}")
-            return docs
-
-        html_content = None
-
-        # Handle different input types
-        if export_path.is_file():
-            if export_path.suffix.lower() == ".zip":
-                # Extract HTML from zip file
-                html_content = self._extract_html_from_zip(export_path)
-            elif export_path.suffix.lower() == ".html":
-                # Read HTML file directly
-                try:
-                    with open(export_path, encoding="utf-8", errors="ignore") as f:
-                        html_content = f.read()
-                except Exception as e:
-                    print(f"Error reading HTML file {export_path}: {e}")
-                    return docs
-            else:
-                print(f"Unsupported file type: {export_path.suffix}")
-                return docs
-
-        elif export_path.is_dir():
-            # Look for HTML files in directory
-            html_files = list(export_path.glob("*.html"))
-            zip_files = list(export_path.glob("*.zip"))
-
-            if html_files:
-                # Use first HTML file found
-                html_file = html_files[0]
-                print(f"Found HTML file: {html_file}")
-                try:
-                    with open(html_file, encoding="utf-8", errors="ignore") as f:
-                        html_content = f.read()
-                except Exception as e:
-                    print(f"Error reading HTML file {html_file}: {e}")
-                    return docs
-
-            elif zip_files:
-                # Use first zip file found
-                zip_file = zip_files[0]
-                print(f"Found zip file: {zip_file}")
-                html_content = self._extract_html_from_zip(zip_file)
-
-            else:
-                print(f"No HTML or zip files found in {export_path}")
-                return docs
-
-        if not html_content:
-            print("No HTML content found to process")
-            return docs
-
-        # Parse conversations from HTML
-        print("Parsing ChatGPT conversations from HTML...")
-        conversations = self._parse_chatgpt_html(html_content)
-
-        if not conversations:
-            print("No conversations found in HTML content")
-            return docs
-
-        print(f"Found {len(conversations)} conversations")
-
-        # Process conversations into documents
-        count = 0
-        for conversation in conversations:
-            if max_count > 0 and count >= max_count:
-                break
-
-            if self.concatenate_conversations:
-                # Create one document per conversation with concatenated messages
-                doc_content = self._create_concatenated_content(conversation)
-
-                metadata = {}
-                if include_metadata:
-                    metadata = {
-                        "title": conversation.get("title", "ChatGPT Conversation"),
-                        "timestamp": conversation.get("timestamp", "Unknown"),
-                        "message_count": len(conversation.get("messages", [])),
-                        "source": "ChatGPT Export",
-                    }
-
-                doc = Document(text=doc_content, metadata=metadata)
-                docs.append(doc)
-                count += 1
-
-            else:
-                # Create separate documents for each message
-                for message in conversation.get("messages", []):
-                    if max_count > 0 and count >= max_count:
-                        break
-
-                    role = message.get("role", "mixed")
-                    content = message.get("content", "")
-                    msg_timestamp = message.get("timestamp", "")
-
-                    if not content.strip():
-                        continue
-
-                    # Create document content with context
-                    doc_content = f"""Conversation: {conversation.get("title", "ChatGPT Conversation")}
-Role: {role}
-Timestamp: {msg_timestamp or conversation.get("timestamp", "Unknown")}
-Message: {content}
-"""
-
-                    metadata = {}
-                    if include_metadata:
-                        metadata = {
-                            "conversation_title": conversation.get("title", "ChatGPT Conversation"),
-                            "role": role,
-                            "timestamp": msg_timestamp or conversation.get("timestamp", "Unknown"),
-                            "source": "ChatGPT Export",
-                        }
-
-                    doc = Document(text=doc_content, metadata=metadata)
-                    docs.append(doc)
-                    count += 1
-
-        print(f"Created {len(docs)} documents from ChatGPT export")
-        return docs

apps/chatgpt_rag.py

@@ -1,186 +0,0 @@
-"""
-ChatGPT RAG example using the unified interface.
-Supports ChatGPT export data from chat.html files.
-"""
-
-import sys
-from pathlib import Path
-
-# Add parent directory to path for imports
-sys.path.insert(0, str(Path(__file__).parent))
-
-from base_rag_example import BaseRAGExample
-from chunking import create_text_chunks
-
-from .chatgpt_data.chatgpt_reader import ChatGPTReader
-
-
-class ChatGPTRAG(BaseRAGExample):
-    """RAG example for ChatGPT conversation data."""
-
-    def __init__(self):
-        # Set default values BEFORE calling super().__init__
-        self.max_items_default = -1  # Process all conversations by default
-        self.embedding_model_default = (
-            "sentence-transformers/all-MiniLM-L6-v2"  # Fast 384-dim model
-        )
-
-        super().__init__(
-            name="ChatGPT",
-            description="Process and query ChatGPT conversation exports with LEANN",
-            default_index_name="chatgpt_conversations_index",
-        )
-
-    def _add_specific_arguments(self, parser):
-        """Add ChatGPT-specific arguments."""
-        chatgpt_group = parser.add_argument_group("ChatGPT Parameters")
-        chatgpt_group.add_argument(
-            "--export-path",
-            type=str,
-            default="./chatgpt_export",
-            help="Path to ChatGPT export file (.zip or .html) or directory containing exports (default: ./chatgpt_export)",
-        )
-        chatgpt_group.add_argument(
-            "--concatenate-conversations",
-            action="store_true",
-            default=True,
-            help="Concatenate messages within conversations for better context (default: True)",
-        )
-        chatgpt_group.add_argument(
-            "--separate-messages",
-            action="store_true",
-            help="Process each message as a separate document (overrides --concatenate-conversations)",
-        )
-        chatgpt_group.add_argument(
-            "--chunk-size", type=int, default=512, help="Text chunk size (default: 512)"
-        )
-        chatgpt_group.add_argument(
-            "--chunk-overlap", type=int, default=128, help="Text chunk overlap (default: 128)"
-        )
-
-    def _find_chatgpt_exports(self, export_path: Path) -> list[Path]:
-        """
-        Find ChatGPT export files in the given path.
-
-        Args:
-            export_path: Path to search for exports
-
-        Returns:
-            List of paths to ChatGPT export files
-        """
-        export_files = []
-
-        if export_path.is_file():
-            if export_path.suffix.lower() in [".zip", ".html"]:
-                export_files.append(export_path)
-        elif export_path.is_dir():
-            # Look for zip and html files
-            export_files.extend(export_path.glob("*.zip"))
-            export_files.extend(export_path.glob("*.html"))
-
-        return export_files
-
-    async def load_data(self, args) -> list[str]:
-        """Load ChatGPT export data and convert to text chunks."""
-        export_path = Path(args.export_path)
-
-        if not export_path.exists():
-            print(f"ChatGPT export path not found: {export_path}")
-            print(
-                "Please ensure you have exported your ChatGPT data and placed it in the correct location."
-            )
-            print("\nTo export your ChatGPT data:")
-            print("1. Sign in to ChatGPT")
-            print("2. Click on your profile icon → Settings → Data Controls")
-            print("3. Click 'Export' under Export Data")
-            print("4. Download the zip file from the email link")
-            print("5. Extract or place the file/directory at the specified path")
-            return []
-
-        # Find export files
-        export_files = self._find_chatgpt_exports(export_path)
-
-        if not export_files:
-            print(f"No ChatGPT export files (.zip or .html) found in: {export_path}")
-            return []
-
-        print(f"Found {len(export_files)} ChatGPT export files")
-
-        # Create reader with appropriate settings
-        concatenate = args.concatenate_conversations and not args.separate_messages
-        reader = ChatGPTReader(concatenate_conversations=concatenate)
-
-        # Process each export file
-        all_documents = []
-        total_processed = 0
-
-        for i, export_file in enumerate(export_files):
-            print(f"\nProcessing export file {i + 1}/{len(export_files)}: {export_file.name}")
-
-            try:
-                # Apply max_items limit per file
-                max_per_file = -1
-                if args.max_items > 0:
-                    remaining = args.max_items - total_processed
-                    if remaining <= 0:
-                        break
-                    max_per_file = remaining
-
-                # Load conversations
-                documents = reader.load_data(
-                    chatgpt_export_path=str(export_file),
-                    max_count=max_per_file,
-                    include_metadata=True,
-                )
-
-                if documents:
-                    all_documents.extend(documents)
-                    total_processed += len(documents)
-                    print(f"Processed {len(documents)} conversations from this file")
-                else:
-                    print(f"No conversations loaded from {export_file}")
-
-            except Exception as e:
-                print(f"Error processing {export_file}: {e}")
-                continue
-
-        if not all_documents:
-            print("No conversations found to process!")
-            print("\nTroubleshooting:")
-            print("- Ensure the export file is a valid ChatGPT export")
-            print("- Check that the HTML file contains conversation data")
-            print("- Try extracting the zip file and pointing to the HTML file directly")
-            return []
-
-        print(f"\nTotal conversations processed: {len(all_documents)}")
-        print("Now starting to split into text chunks... this may take some time")
-
-        # Convert to text chunks
-        all_texts = create_text_chunks(
-            all_documents, chunk_size=args.chunk_size, chunk_overlap=args.chunk_overlap
-        )
-
-        print(f"Created {len(all_texts)} text chunks from {len(all_documents)} conversations")
-        return all_texts
-
-
-if __name__ == "__main__":
-    import asyncio
-
-    # Example queries for ChatGPT RAG
-    print("\n🤖 ChatGPT RAG Example")
-    print("=" * 50)
-    print("\nExample queries you can try:")
-    print("- 'What did I ask about Python programming?'")
-    print("- 'Show me conversations about machine learning'")
-    print("- 'Find discussions about travel planning'")
-    print("- 'What advice did ChatGPT give me about career development?'")
-    print("- 'Search for conversations about cooking recipes'")
-    print("\nTo get started:")
-    print("1. Export your ChatGPT data from Settings → Data Controls → Export")
-    print("2. Place the downloaded zip file or extracted HTML in ./chatgpt_export/")
-    print("3. Run this script to build your personal ChatGPT knowledge base!")
-    print("\nOr run without --query for interactive mode\n")
-
-    rag = ChatGPTRAG()
-    asyncio.run(rag.run())

apps/claude_data/claude_reader.py

@@ -1,420 +0,0 @@
-"""
-Claude export data reader.
-
-Reads and processes Claude conversation data from exported JSON files.
-"""
-
-import json
-from pathlib import Path
-from typing import Any
-from zipfile import ZipFile
-
-from llama_index.core import Document
-from llama_index.core.readers.base import BaseReader
-
-
-class ClaudeReader(BaseReader):
-    """
-    Claude export data reader.
-
-    Reads Claude conversation data from exported JSON files or zip archives.
-    Processes conversations into structured documents with metadata.
-    """
-
-    def __init__(self, concatenate_conversations: bool = True) -> None:
-        """
-        Initialize.
-
-        Args:
-            concatenate_conversations: Whether to concatenate messages within conversations for better context
-        """
-        self.concatenate_conversations = concatenate_conversations
-
-    def _extract_json_from_zip(self, zip_path: Path) -> list[str]:
-        """
-        Extract JSON files from Claude export zip file.
-
-        Args:
-            zip_path: Path to the Claude export zip file
-
-        Returns:
-            List of JSON content strings, or empty list if not found
-        """
-        json_contents = []
-        try:
-            with ZipFile(zip_path, "r") as zip_file:
-                # Look for JSON files
-                json_files = [f for f in zip_file.namelist() if f.endswith(".json")]
-
-                if not json_files:
-                    print(f"No JSON files found in {zip_path}")
-                    return []
-
-                print(f"Found {len(json_files)} JSON files in archive")
-
-                for json_file in json_files:
-                    with zip_file.open(json_file) as f:
-                        content = f.read().decode("utf-8", errors="ignore")
-                        json_contents.append(content)
-
-        except Exception as e:
-            print(f"Error extracting JSON from zip {zip_path}: {e}")
-
-        return json_contents
-
-    def _parse_claude_json(self, json_content: str) -> list[dict]:
-        """
-        Parse Claude JSON export to extract conversations.
-
-        Args:
-            json_content: JSON content from Claude export
-
-        Returns:
-            List of conversation dictionaries
-        """
-        try:
-            data = json.loads(json_content)
-        except json.JSONDecodeError as e:
-            print(f"Error parsing JSON: {e}")
-            return []
-
-        conversations = []
-
-        # Handle different possible JSON structures
-        if isinstance(data, list):
-            # If data is a list of conversations
-            for item in data:
-                conversation = self._extract_conversation_from_json(item)
-                if conversation:
-                    conversations.append(conversation)
-        elif isinstance(data, dict):
-            # Check for common structures
-            if "conversations" in data:
-                # Structure: {"conversations": [...]}
-                for item in data["conversations"]:
-                    conversation = self._extract_conversation_from_json(item)
-                    if conversation:
-                        conversations.append(conversation)
-            elif "messages" in data:
-                # Single conversation with messages
-                conversation = self._extract_conversation_from_json(data)
-                if conversation:
-                    conversations.append(conversation)
-            else:
-                # Try to treat the whole object as a conversation
-                conversation = self._extract_conversation_from_json(data)
-                if conversation:
-                    conversations.append(conversation)
-
-        return conversations
-
-    def _extract_conversation_from_json(self, conv_data: dict) -> dict | None:
-        """
-        Extract conversation data from a JSON object.
-
-        Args:
-            conv_data: Dictionary containing conversation data
-
-        Returns:
-            Dictionary with conversation data or None
-        """
-        if not isinstance(conv_data, dict):
-            return None
-
-        messages = []
-
-        # Look for messages in various possible structures
-        message_sources = []
-        if "messages" in conv_data:
-            message_sources = conv_data["messages"]
-        elif "chat" in conv_data:
-            message_sources = conv_data["chat"]
-        elif "conversation" in conv_data:
-            message_sources = conv_data["conversation"]
-        else:
-            # If no clear message structure, try to extract from the object itself
-            if "content" in conv_data and "role" in conv_data:
-                message_sources = [conv_data]
-
-        for msg_data in message_sources:
-            message = self._extract_message_from_json(msg_data)
-            if message:
-                messages.append(message)
-
-        if not messages:
-            return None
-
-        # Extract conversation metadata
-        title = self._extract_title_from_conversation(conv_data, messages)
-        timestamp = self._extract_timestamp_from_conversation(conv_data)
-
-        return {"title": title, "messages": messages, "timestamp": timestamp}
-
-    def _extract_message_from_json(self, msg_data: dict) -> dict | None:
-        """
-        Extract message data from a JSON message object.
-
-        Args:
-            msg_data: Dictionary containing message data
-
-        Returns:
-            Dictionary with message data or None
-        """
-        if not isinstance(msg_data, dict):
-            return None
-
-        # Extract content from various possible fields
-        content = ""
-        content_fields = ["content", "text", "message", "body"]
-        for field in content_fields:
-            if msg_data.get(field):
-                content = str(msg_data[field])
-                break
-
-        if not content or len(content.strip()) < 3:
-            return None
-
-        # Extract role (user/assistant/human/ai/claude)
-        role = "mixed"  # Default role
-        role_fields = ["role", "sender", "from", "author", "type"]
-        for field in role_fields:
-            if msg_data.get(field):
-                role_value = str(msg_data[field]).lower()
-                if role_value in ["user", "human", "person"]:
-                    role = "user"
-                elif role_value in ["assistant", "ai", "claude", "bot"]:
-                    role = "assistant"
-                break
-
-        # Extract timestamp
-        timestamp = self._extract_timestamp_from_message(msg_data)
-
-        return {"role": role, "content": content, "timestamp": timestamp}
-
-    def _extract_timestamp_from_message(self, msg_data: dict) -> str | None:
-        """Extract timestamp from message data."""
-        timestamp_fields = ["timestamp", "created_at", "date", "time"]
-        for field in timestamp_fields:
-            if msg_data.get(field):
-                return str(msg_data[field])
-        return None
-
-    def _extract_timestamp_from_conversation(self, conv_data: dict) -> str | None:
-        """Extract timestamp from conversation data."""
-        timestamp_fields = ["timestamp", "created_at", "date", "updated_at", "last_updated"]
-        for field in timestamp_fields:
-            if conv_data.get(field):
-                return str(conv_data[field])
-        return None
-
-    def _extract_title_from_conversation(self, conv_data: dict, messages: list) -> str:
-        """Extract or generate title for conversation."""
-        # Try to find explicit title
-        title_fields = ["title", "name", "subject", "topic"]
-        for field in title_fields:
-            if conv_data.get(field):
-                return str(conv_data[field])
-
-        # Generate title from first user message
-        for message in messages:
-            if message.get("role") == "user":
-                content = message.get("content", "")
-                if content:
-                    # Use first 50 characters as title
-                    title = content[:50].strip()
-                    if len(content) > 50:
-                        title += "..."
-                    return title
-
-        return "Claude Conversation"
-
-    def _create_concatenated_content(self, conversation: dict) -> str:
-        """
-        Create concatenated content from conversation messages.
-
-        Args:
-            conversation: Dictionary containing conversation data
-
-        Returns:
-            Formatted concatenated content
-        """
-        title = conversation.get("title", "Claude Conversation")
-        messages = conversation.get("messages", [])
-        timestamp = conversation.get("timestamp", "Unknown")
-
-        # Build message content
-        message_parts = []
-        for message in messages:
-            role = message.get("role", "mixed")
-            content = message.get("content", "")
-            msg_timestamp = message.get("timestamp", "")
-
-            if role == "user":
-                prefix = "[You]"
-            elif role == "assistant":
-                prefix = "[Claude]"
-            else:
-                prefix = "[Message]"
-
-            # Add timestamp if available
-            if msg_timestamp:
-                prefix += f" ({msg_timestamp})"
-
-            message_parts.append(f"{prefix}: {content}")
-
-        concatenated_text = "\n\n".join(message_parts)
-
-        # Create final document content
-        doc_content = f"""Conversation: {title}
-Date: {timestamp}
-Messages ({len(messages)} messages):
-
-{concatenated_text}
-"""
-        return doc_content
-
-    def load_data(self, input_dir: str | None = None, **load_kwargs: Any) -> list[Document]:
-        """
-        Load Claude export data.
-
-        Args:
-            input_dir: Directory containing Claude export files or path to specific file
-            **load_kwargs:
-                max_count (int): Maximum number of conversations to process
-                claude_export_path (str): Specific path to Claude export file/directory
-                include_metadata (bool): Whether to include metadata in documents
-        """
-        docs: list[Document] = []
-        max_count = load_kwargs.get("max_count", -1)
-        claude_export_path = load_kwargs.get("claude_export_path", input_dir)
-        include_metadata = load_kwargs.get("include_metadata", True)
-
-        if not claude_export_path:
-            print("No Claude export path provided")
-            return docs
-
-        export_path = Path(claude_export_path)
-
-        if not export_path.exists():
-            print(f"Claude export path not found: {export_path}")
-            return docs
-
-        json_contents = []
-
-        # Handle different input types
-        if export_path.is_file():
-            if export_path.suffix.lower() == ".zip":
-                # Extract JSON from zip file
-                json_contents = self._extract_json_from_zip(export_path)
-            elif export_path.suffix.lower() == ".json":
-                # Read JSON file directly
-                try:
-                    with open(export_path, encoding="utf-8", errors="ignore") as f:
-                        json_contents.append(f.read())
-                except Exception as e:
-                    print(f"Error reading JSON file {export_path}: {e}")
-                    return docs
-            else:
-                print(f"Unsupported file type: {export_path.suffix}")
-                return docs
-
-        elif export_path.is_dir():
-            # Look for JSON files in directory
-            json_files = list(export_path.glob("*.json"))
-            zip_files = list(export_path.glob("*.zip"))
-
-            if json_files:
-                print(f"Found {len(json_files)} JSON files in directory")
-                for json_file in json_files:
-                    try:
-                        with open(json_file, encoding="utf-8", errors="ignore") as f:
-                            json_contents.append(f.read())
-                    except Exception as e:
-                        print(f"Error reading JSON file {json_file}: {e}")
-                        continue
-
-            if zip_files:
-                print(f"Found {len(zip_files)} ZIP files in directory")
-                for zip_file in zip_files:
-                    zip_contents = self._extract_json_from_zip(zip_file)
-                    json_contents.extend(zip_contents)
-
-            if not json_files and not zip_files:
-                print(f"No JSON or ZIP files found in {export_path}")
-                return docs
-
-        if not json_contents:
-            print("No JSON content found to process")
-            return docs
-
-        # Parse conversations from JSON content
-        print("Parsing Claude conversations from JSON...")
-        all_conversations = []
-        for json_content in json_contents:
-            conversations = self._parse_claude_json(json_content)
-            all_conversations.extend(conversations)
-
-        if not all_conversations:
-            print("No conversations found in JSON content")
-            return docs
-
-        print(f"Found {len(all_conversations)} conversations")
-
-        # Process conversations into documents
-        count = 0
-        for conversation in all_conversations:
-            if max_count > 0 and count >= max_count:
-                break
-
-            if self.concatenate_conversations:
-                # Create one document per conversation with concatenated messages
-                doc_content = self._create_concatenated_content(conversation)
-
-                metadata = {}
-                if include_metadata:
-                    metadata = {
-                        "title": conversation.get("title", "Claude Conversation"),
-                        "timestamp": conversation.get("timestamp", "Unknown"),
-                        "message_count": len(conversation.get("messages", [])),
-                        "source": "Claude Export",
-                    }
-
-                doc = Document(text=doc_content, metadata=metadata)
-                docs.append(doc)
-                count += 1
-
-            else:
-                # Create separate documents for each message
-                for message in conversation.get("messages", []):
-                    if max_count > 0 and count >= max_count:
-                        break
-
-                    role = message.get("role", "mixed")
-                    content = message.get("content", "")
-                    msg_timestamp = message.get("timestamp", "")
-
-                    if not content.strip():
-                        continue
-
-                    # Create document content with context
-                    doc_content = f"""Conversation: {conversation.get("title", "Claude Conversation")}
-Role: {role}
-Timestamp: {msg_timestamp or conversation.get("timestamp", "Unknown")}
-Message: {content}
-"""
-
-                    metadata = {}
-                    if include_metadata:
-                        metadata = {
-                            "conversation_title": conversation.get("title", "Claude Conversation"),
-                            "role": role,
-                            "timestamp": msg_timestamp or conversation.get("timestamp", "Unknown"),
-                            "source": "Claude Export",
-                        }
-
-                    doc = Document(text=doc_content, metadata=metadata)
-                    docs.append(doc)
-                    count += 1
-
-        print(f"Created {len(docs)} documents from Claude export")
-        return docs

apps/claude_rag.py

@@ -1,189 +0,0 @@
-"""
-Claude RAG example using the unified interface.
-Supports Claude export data from JSON files.
-"""
-
-import sys
-from pathlib import Path
-
-# Add parent directory to path for imports
-sys.path.insert(0, str(Path(__file__).parent))
-
-from base_rag_example import BaseRAGExample
-from chunking import create_text_chunks
-
-from .claude_data.claude_reader import ClaudeReader
-
-
-class ClaudeRAG(BaseRAGExample):
-    """RAG example for Claude conversation data."""
-
-    def __init__(self):
-        # Set default values BEFORE calling super().__init__
-        self.max_items_default = -1  # Process all conversations by default
-        self.embedding_model_default = (
-            "sentence-transformers/all-MiniLM-L6-v2"  # Fast 384-dim model
-        )
-
-        super().__init__(
-            name="Claude",
-            description="Process and query Claude conversation exports with LEANN",
-            default_index_name="claude_conversations_index",
-        )
-
-    def _add_specific_arguments(self, parser):
-        """Add Claude-specific arguments."""
-        claude_group = parser.add_argument_group("Claude Parameters")
-        claude_group.add_argument(
-            "--export-path",
-            type=str,
-            default="./claude_export",
-            help="Path to Claude export file (.json or .zip) or directory containing exports (default: ./claude_export)",
-        )
-        claude_group.add_argument(
-            "--concatenate-conversations",
-            action="store_true",
-            default=True,
-            help="Concatenate messages within conversations for better context (default: True)",
-        )
-        claude_group.add_argument(
-            "--separate-messages",
-            action="store_true",
-            help="Process each message as a separate document (overrides --concatenate-conversations)",
-        )
-        claude_group.add_argument(
-            "--chunk-size", type=int, default=512, help="Text chunk size (default: 512)"
-        )
-        claude_group.add_argument(
-            "--chunk-overlap", type=int, default=128, help="Text chunk overlap (default: 128)"
-        )
-
-    def _find_claude_exports(self, export_path: Path) -> list[Path]:
-        """
-        Find Claude export files in the given path.
-
-        Args:
-            export_path: Path to search for exports
-
-        Returns:
-            List of paths to Claude export files
-        """
-        export_files = []
-
-        if export_path.is_file():
-            if export_path.suffix.lower() in [".zip", ".json"]:
-                export_files.append(export_path)
-        elif export_path.is_dir():
-            # Look for zip and json files
-            export_files.extend(export_path.glob("*.zip"))
-            export_files.extend(export_path.glob("*.json"))
-
-        return export_files
-
-    async def load_data(self, args) -> list[str]:
-        """Load Claude export data and convert to text chunks."""
-        export_path = Path(args.export_path)
-
-        if not export_path.exists():
-            print(f"Claude export path not found: {export_path}")
-            print(
-                "Please ensure you have exported your Claude data and placed it in the correct location."
-            )
-            print("\nTo export your Claude data:")
-            print("1. Open Claude in your browser")
-            print("2. Look for export/download options in settings or conversation menu")
-            print("3. Download the conversation data (usually in JSON format)")
-            print("4. Place the file/directory at the specified path")
-            print(
-                "\nNote: Claude export methods may vary. Check Claude's help documentation for current instructions."
-            )
-            return []
-
-        # Find export files
-        export_files = self._find_claude_exports(export_path)
-
-        if not export_files:
-            print(f"No Claude export files (.json or .zip) found in: {export_path}")
-            return []
-
-        print(f"Found {len(export_files)} Claude export files")
-
-        # Create reader with appropriate settings
-        concatenate = args.concatenate_conversations and not args.separate_messages
-        reader = ClaudeReader(concatenate_conversations=concatenate)
-
-        # Process each export file
-        all_documents = []
-        total_processed = 0
-
-        for i, export_file in enumerate(export_files):
-            print(f"\nProcessing export file {i + 1}/{len(export_files)}: {export_file.name}")
-
-            try:
-                # Apply max_items limit per file
-                max_per_file = -1
-                if args.max_items > 0:
-                    remaining = args.max_items - total_processed
-                    if remaining <= 0:
-                        break
-                    max_per_file = remaining
-
-                # Load conversations
-                documents = reader.load_data(
-                    claude_export_path=str(export_file),
-                    max_count=max_per_file,
-                    include_metadata=True,
-                )
-
-                if documents:
-                    all_documents.extend(documents)
-                    total_processed += len(documents)
-                    print(f"Processed {len(documents)} conversations from this file")
-                else:
-                    print(f"No conversations loaded from {export_file}")
-
-            except Exception as e:
-                print(f"Error processing {export_file}: {e}")
-                continue
-
-        if not all_documents:
-            print("No conversations found to process!")
-            print("\nTroubleshooting:")
-            print("- Ensure the export file is a valid Claude export")
-            print("- Check that the JSON file contains conversation data")
-            print("- Try using a different export format or method")
-            print("- Check Claude's documentation for current export procedures")
-            return []
-
-        print(f"\nTotal conversations processed: {len(all_documents)}")
-        print("Now starting to split into text chunks... this may take some time")
-
-        # Convert to text chunks
-        all_texts = create_text_chunks(
-            all_documents, chunk_size=args.chunk_size, chunk_overlap=args.chunk_overlap
-        )
-
-        print(f"Created {len(all_texts)} text chunks from {len(all_documents)} conversations")
-        return all_texts
-
-
-if __name__ == "__main__":
-    import asyncio
-
-    # Example queries for Claude RAG
-    print("\n🤖 Claude RAG Example")
-    print("=" * 50)
-    print("\nExample queries you can try:")
-    print("- 'What did I ask Claude about Python programming?'")
-    print("- 'Show me conversations about machine learning'")
-    print("- 'Find discussions about code optimization'")
-    print("- 'What advice did Claude give me about software design?'")
-    print("- 'Search for conversations about debugging techniques'")
-    print("\nTo get started:")
-    print("1. Export your Claude conversation data")
-    print("2. Place the JSON/ZIP file in ./claude_export/")
-    print("3. Run this script to build your personal Claude knowledge base!")
-    print("\nOr run without --query for interactive mode\n")
-
-    rag = ClaudeRAG()
-    asyncio.run(rag.run())

apps/imessage_data/__init__.py

@@ -1 +0,0 @@
-"""iMessage data processing module."""

apps/imessage_data/imessage_reader.py

@@ -1,342 +0,0 @@
-"""
-iMessage data reader.
-
-Reads and processes iMessage conversation data from the macOS Messages database.
-"""
-
-import sqlite3
-from datetime import datetime
-from pathlib import Path
-from typing import Any
-
-from llama_index.core import Document
-from llama_index.core.readers.base import BaseReader
-
-
-class IMessageReader(BaseReader):
-    """
-    iMessage data reader.
-
-    Reads iMessage conversation data from the macOS Messages database (chat.db).
-    Processes conversations into structured documents with metadata.
-    """
-
-    def __init__(self, concatenate_conversations: bool = True) -> None:
-        """
-        Initialize.
-
-        Args:
-            concatenate_conversations: Whether to concatenate messages within conversations for better context
-        """
-        self.concatenate_conversations = concatenate_conversations
-
-    def _get_default_chat_db_path(self) -> Path:
-        """
-        Get the default path to the iMessage chat database.
-
-        Returns:
-            Path to the chat.db file
-        """
-        home = Path.home()
-        return home / "Library" / "Messages" / "chat.db"
-
-    def _convert_cocoa_timestamp(self, cocoa_timestamp: int) -> str:
-        """
-        Convert Cocoa timestamp to readable format.
-
-        Args:
-            cocoa_timestamp: Timestamp in Cocoa format (nanoseconds since 2001-01-01)
-
-        Returns:
-            Formatted timestamp string
-        """
-        if cocoa_timestamp == 0:
-            return "Unknown"
-
-        try:
-            # Cocoa timestamp is nanoseconds since 2001-01-01 00:00:00 UTC
-            # Convert to seconds and add to Unix epoch
-            cocoa_epoch = datetime(2001, 1, 1)
-            unix_timestamp = cocoa_timestamp / 1_000_000_000  # Convert nanoseconds to seconds
-            message_time = cocoa_epoch.timestamp() + unix_timestamp
-            return datetime.fromtimestamp(message_time).strftime("%Y-%m-%d %H:%M:%S")
-        except (ValueError, OSError):
-            return "Unknown"
-
-    def _get_contact_name(self, handle_id: str) -> str:
-        """
-        Get a readable contact name from handle ID.
-
-        Args:
-            handle_id: The handle ID (phone number or email)
-
-        Returns:
-            Formatted contact name
-        """
-        if not handle_id:
-            return "Unknown"
-
-        # Clean up phone numbers and emails for display
-        if "@" in handle_id:
-            return handle_id  # Email address
-        elif handle_id.startswith("+"):
-            return handle_id  # International phone number
-        else:
-            # Try to format as phone number
-            digits = "".join(filter(str.isdigit, handle_id))
-            if len(digits) == 10:
-                return f"({digits[:3]}) {digits[3:6]}-{digits[6:]}"
-            elif len(digits) == 11 and digits[0] == "1":
-                return f"+1 ({digits[1:4]}) {digits[4:7]}-{digits[7:]}"
-            else:
-                return handle_id
-
-    def _read_messages_from_db(self, db_path: Path) -> list[dict]:
-        """
-        Read messages from the iMessage database.
-
-        Args:
-            db_path: Path to the chat.db file
-
-        Returns:
-            List of message dictionaries
-        """
-        if not db_path.exists():
-            print(f"iMessage database not found at: {db_path}")
-            return []
-
-        try:
-            # Connect to the database
-            conn = sqlite3.connect(str(db_path))
-            cursor = conn.cursor()
-
-            # Query to get messages with chat and handle information
-            query = """
-            SELECT
-                m.ROWID as message_id,
-                m.text,
-                m.date,
-                m.is_from_me,
-                m.service,
-                c.chat_identifier,
-                c.display_name as chat_display_name,
-                h.id as handle_id,
-                c.ROWID as chat_id
-            FROM message m
-            LEFT JOIN chat_message_join cmj ON m.ROWID = cmj.message_id
-            LEFT JOIN chat c ON cmj.chat_id = c.ROWID
-            LEFT JOIN handle h ON m.handle_id = h.ROWID
-            WHERE m.text IS NOT NULL AND m.text != ''
-            ORDER BY c.ROWID, m.date
-            """
-
-            cursor.execute(query)
-            rows = cursor.fetchall()
-
-            messages = []
-            for row in rows:
-                (
-                    message_id,
-                    text,
-                    date,
-                    is_from_me,
-                    service,
-                    chat_identifier,
-                    chat_display_name,
-                    handle_id,
-                    chat_id,
-                ) = row
-
-                message = {
-                    "message_id": message_id,
-                    "text": text,
-                    "timestamp": self._convert_cocoa_timestamp(date),
-                    "is_from_me": bool(is_from_me),
-                    "service": service or "iMessage",
-                    "chat_identifier": chat_identifier or "Unknown",
-                    "chat_display_name": chat_display_name or "Unknown Chat",
-                    "handle_id": handle_id or "Unknown",
-                    "contact_name": self._get_contact_name(handle_id or ""),
-                    "chat_id": chat_id,
-                }
-                messages.append(message)
-
-            conn.close()
-            print(f"Found {len(messages)} messages in database")
-            return messages
-
-        except sqlite3.Error as e:
-            print(f"Error reading iMessage database: {e}")
-            return []
-        except Exception as e:
-            print(f"Unexpected error reading iMessage database: {e}")
-            return []
-
-    def _group_messages_by_chat(self, messages: list[dict]) -> dict[int, list[dict]]:
-        """
-        Group messages by chat ID.
-
-        Args:
-            messages: List of message dictionaries
-
-        Returns:
-            Dictionary mapping chat_id to list of messages
-        """
-        chats = {}
-        for message in messages:
-            chat_id = message["chat_id"]
-            if chat_id not in chats:
-                chats[chat_id] = []
-            chats[chat_id].append(message)
-
-        return chats
-
-    def _create_concatenated_content(self, chat_id: int, messages: list[dict]) -> str:
-        """
-        Create concatenated content from chat messages.
-
-        Args:
-            chat_id: The chat ID
-            messages: List of messages in the chat
-
-        Returns:
-            Concatenated text content
-        """
-        if not messages:
-            return ""
-
-        # Get chat info from first message
-        first_msg = messages[0]
-        chat_name = first_msg["chat_display_name"]
-        chat_identifier = first_msg["chat_identifier"]
-
-        # Build message content
-        message_parts = []
-        for message in messages:
-            timestamp = message["timestamp"]
-            is_from_me = message["is_from_me"]
-            text = message["text"]
-            contact_name = message["contact_name"]
-
-            if is_from_me:
-                prefix = "[You]"
-            else:
-                prefix = f"[{contact_name}]"
-
-            if timestamp != "Unknown":
-                prefix += f" ({timestamp})"
-
-            message_parts.append(f"{prefix}: {text}")
-
-        concatenated_text = "\n\n".join(message_parts)
-
-        doc_content = f"""Chat: {chat_name}
-Identifier: {chat_identifier}
-Messages ({len(messages)} messages):
-
-{concatenated_text}
-"""
-        return doc_content
-
-    def _create_individual_content(self, message: dict) -> str:
-        """
-        Create content for individual message.
-
-        Args:
-            message: Message dictionary
-
-        Returns:
-            Formatted message content
-        """
-        timestamp = message["timestamp"]
-        is_from_me = message["is_from_me"]
-        text = message["text"]
-        contact_name = message["contact_name"]
-        chat_name = message["chat_display_name"]
-
-        sender = "You" if is_from_me else contact_name
-
-        return f"""Message from {sender} in chat "{chat_name}"
-Time: {timestamp}
-Content: {text}
-"""
-
-    def load_data(self, input_dir: str | None = None, **load_kwargs: Any) -> list[Document]:
-        """
-        Load iMessage data and return as documents.
-
-        Args:
-            input_dir: Optional path to directory containing chat.db file.
-                      If not provided, uses default macOS location.
-            **load_kwargs: Additional arguments (unused)
-
-        Returns:
-            List of Document objects containing iMessage data
-        """
-        docs = []
-
-        # Determine database path
-        if input_dir:
-            db_path = Path(input_dir) / "chat.db"
-        else:
-            db_path = self._get_default_chat_db_path()
-
-        print(f"Reading iMessage database from: {db_path}")
-
-        # Read messages from database
-        messages = self._read_messages_from_db(db_path)
-        if not messages:
-            return docs
-
-        if self.concatenate_conversations:
-            # Group messages by chat and create concatenated documents
-            chats = self._group_messages_by_chat(messages)
-
-            for chat_id, chat_messages in chats.items():
-                if not chat_messages:
-                    continue
-
-                content = self._create_concatenated_content(chat_id, chat_messages)
-
-                # Create metadata
-                first_msg = chat_messages[0]
-                last_msg = chat_messages[-1]
-
-                metadata = {
-                    "source": "iMessage",
-                    "chat_id": chat_id,
-                    "chat_name": first_msg["chat_display_name"],
-                    "chat_identifier": first_msg["chat_identifier"],
-                    "message_count": len(chat_messages),
-                    "first_message_date": first_msg["timestamp"],
-                    "last_message_date": last_msg["timestamp"],
-                    "participants": list(
-                        {msg["contact_name"] for msg in chat_messages if not msg["is_from_me"]}
-                    ),
-                }
-
-                doc = Document(text=content, metadata=metadata)
-                docs.append(doc)
-
-        else:
-            # Create individual documents for each message
-            for message in messages:
-                content = self._create_individual_content(message)
-
-                metadata = {
-                    "source": "iMessage",
-                    "message_id": message["message_id"],
-                    "chat_id": message["chat_id"],
-                    "chat_name": message["chat_display_name"],
-                    "chat_identifier": message["chat_identifier"],
-                    "timestamp": message["timestamp"],
-                    "is_from_me": message["is_from_me"],
-                    "contact_name": message["contact_name"],
-                    "service": message["service"],
-                }
-
-                doc = Document(text=content, metadata=metadata)
-                docs.append(doc)
-
-        print(f"Created {len(docs)} documents from iMessage data")
-        return docs

apps/imessage_rag.py

@@ -1,125 +0,0 @@
-"""
-iMessage RAG Example.
-
-This example demonstrates how to build a RAG system on your iMessage conversation history.
-"""
-
-import asyncio
-from pathlib import Path
-
-from leann.chunking_utils import create_text_chunks
-
-from apps.base_rag_example import BaseRAGExample
-from apps.imessage_data.imessage_reader import IMessageReader
-
-
-class IMessageRAG(BaseRAGExample):
-    """RAG example for iMessage conversation history."""
-
-    def __init__(self):
-        super().__init__(
-            name="iMessage",
-            description="RAG on your iMessage conversation history",
-            default_index_name="imessage_index",
-        )
-
-    def _add_specific_arguments(self, parser):
-        """Add iMessage-specific arguments."""
-        imessage_group = parser.add_argument_group("iMessage Parameters")
-        imessage_group.add_argument(
-            "--db-path",
-            type=str,
-            default=None,
-            help="Path to iMessage chat.db file (default: ~/Library/Messages/chat.db)",
-        )
-        imessage_group.add_argument(
-            "--concatenate-conversations",
-            action="store_true",
-            default=True,
-            help="Concatenate messages within conversations for better context (default: True)",
-        )
-        imessage_group.add_argument(
-            "--no-concatenate-conversations",
-            action="store_true",
-            help="Process each message individually instead of concatenating by conversation",
-        )
-        imessage_group.add_argument(
-            "--chunk-size",
-            type=int,
-            default=1000,
-            help="Maximum characters per text chunk (default: 1000)",
-        )
-        imessage_group.add_argument(
-            "--chunk-overlap",
-            type=int,
-            default=200,
-            help="Overlap between text chunks (default: 200)",
-        )
-
-    async def load_data(self, args) -> list[str]:
-        """Load iMessage history and convert to text chunks."""
-        print("Loading iMessage conversation history...")
-
-        # Determine concatenation setting
-        concatenate = args.concatenate_conversations and not args.no_concatenate_conversations
-
-        # Initialize iMessage reader
-        reader = IMessageReader(concatenate_conversations=concatenate)
-
-        # Load documents
-        try:
-            if args.db_path:
-                # Use custom database path
-                db_dir = str(Path(args.db_path).parent)
-                documents = reader.load_data(input_dir=db_dir)
-            else:
-                # Use default macOS location
-                documents = reader.load_data()
-
-        except Exception as e:
-            print(f"Error loading iMessage data: {e}")
-            print("\nTroubleshooting tips:")
-            print("1. Make sure you have granted Full Disk Access to your terminal/IDE")
-            print("2. Check that the iMessage database exists at ~/Library/Messages/chat.db")
-            print("3. Try specifying a custom path with --db-path if you have a backup")
-            return []
-
-        if not documents:
-            print("No iMessage conversations found!")
-            return []
-
-        print(f"Loaded {len(documents)} iMessage documents")
-
-        # Show some statistics
-        total_messages = sum(doc.metadata.get("message_count", 1) for doc in documents)
-        print(f"Total messages: {total_messages}")
-
-        if concatenate:
-            # Show chat statistics
-            chat_names = [doc.metadata.get("chat_name", "Unknown") for doc in documents]
-            unique_chats = len(set(chat_names))
-            print(f"Unique conversations: {unique_chats}")
-
-        # Convert to text chunks
-        all_texts = create_text_chunks(
-            documents,
-            chunk_size=args.chunk_size,
-            chunk_overlap=args.chunk_overlap,
-        )
-
-        # Apply max_items limit if specified
-        if args.max_items > 0:
-            all_texts = all_texts[: args.max_items]
-            print(f"Limited to {len(all_texts)} text chunks (max_items={args.max_items})")
-
-        return all_texts
-
-
-async def main():
-    """Main entry point."""
-    app = IMessageRAG()
-    await app.run()
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

apps/multimodal/vision-based-pdf-multi-vector/README.md

@@ -1,113 +0,0 @@
-## Vision-based PDF Multi-Vector Demos (macOS/MPS)
-
-This folder contains two demos to index PDF pages as images and run multi-vector retrieval with ColPali/ColQwen2, plus optional similarity map visualization and answer generation.
-
-### What you’ll run
-- `multi-vector-leann-paper-example.py`: local PDF → pages → embed → build HNSW index → search.
-- `multi-vector-leann-similarity-map.py`: HF dataset (default) or local pages → embed → index → retrieve → similarity maps → optional Qwen-VL answer.
-
-## Prerequisites (macOS)
-
-### 1) Homebrew poppler (for pdf2image)
-```bash
-brew install poppler
-which pdfinfo && pdfinfo -v
-```
-
-### 2) Python environment
-Use uv (recommended) or pip. Python 3.9+.
-
-Using uv:
-```bash
-uv pip install \
-  colpali_engine \
-  pdf2image \
-  pillow \
-  matplotlib qwen_vl_utils \
-  einops \
-  seaborn
-```
-
-Notes:
-- On first run, models download from Hugging Face. Login/config if needed.
-- The scripts auto-select device: CUDA > MPS > CPU. Verify MPS:
-```bash
-python -c "import torch; print('MPS available:', bool(getattr(torch.backends, 'mps', None) and torch.backends.mps.is_available()))"
-```
-
-## Run the demos
-
-### A) Local PDF example
-Converts a local PDF into page images, embeds them, builds an index, and searches.
-
-```bash
-cd apps/multimodal/vision-based-pdf-multi-vector
-# If you don't have the sample PDF locally, download it (ignored by Git)
-mkdir -p pdfs
-curl -L -o pdfs/2004.12832v2.pdf https://arxiv.org/pdf/2004.12832.pdf
-ls pdfs/2004.12832v2.pdf
-# Ensure output dir exists
-mkdir -p pages
-python multi-vector-leann-paper-example.py
-```
-Expected:
-- Page images in `pages/`.
-- Console prints like `Using device=mps, dtype=...` and retrieved file paths for queries.
-
-To use your own PDF: edit `pdf_path` near the top of the script.
-
-### B) Similarity map + answer demo
-Uses HF dataset `weaviate/arXiv-AI-papers-multi-vector` by default; can switch to local pages.
-
-```bash
-cd apps/multimodal/vision-based-pdf-multi-vector
-python multi-vector-leann-similarity-map.py
-```
-Artifacts (when enabled):
-- Retrieved pages: `./figures/retrieved_page_rank{K}.png`
-- Similarity maps: `./figures/similarity_map_rank{K}.png`
-
-Key knobs in the script (top of file):
-- `QUERY`: your question
-- `MODEL`: `"colqwen2"` or `"colpali"`
-- `USE_HF_DATASET`: set `False` to use local pages
-- `PDF`, `PAGES_DIR`: for local mode
-- `INDEX_PATH`, `TOPK`, `FIRST_STAGE_K`, `REBUILD_INDEX`
-- `SIMILARITY_MAP`, `SIM_TOKEN_IDX`, `SIM_OUTPUT`
-- `ANSWER`, `MAX_NEW_TOKENS` (Qwen-VL)
-
-## Troubleshooting
-- pdf2image errors on macOS: ensure `brew install poppler` and `pdfinfo` works in terminal.
-- Slow or OOM on MPS: reduce dataset size (e.g., set `MAX_DOCS`) or switch to CPU.
-- NaNs on MPS: keep fp32 on MPS (default in similarity-map script); avoid fp16 there.
-- First-run model downloads can be large; ensure network access (HF mirrors if needed).
-
-## Notes
-- Index files are under `./indexes/`. Delete or set `REBUILD_INDEX=True` to rebuild.
-- For local PDFs, page images go to `./pages/`.
-
-
-### Retrieval and Visualization Example
-
-Example settings in `multi-vector-leann-similarity-map.py`:
-- `QUERY = "How does DeepSeek-V2 compare against the LLaMA family of LLMs?"`
-- `SIMILARITY_MAP = True` (to generate heatmaps)
-- `TOPK = 1` (save the top retrieved page and its similarity map)
-
-Run:
-```bash
-cd apps/multimodal/vision-based-pdf-multi-vector
-python multi-vector-leann-similarity-map.py
-```
-
-Outputs (by default):
-- Retrieved page: `./figures/retrieved_page_rank1.png`
-- Similarity map: `./figures/similarity_map_rank1.png`
-
-Sample visualization (example result, and the query is "QUERY = "How does Vim model performance and efficiency compared to other models?"
-"):
-![Similarity map example](fig/image.png)
-
-Notes:
-- Set `SIM_TOKEN_IDX` to visualize a specific token index; set `-1` to auto-select the most salient token.
-- If you change `SIM_OUTPUT` to a file path (e.g., `./figures/my_map.png`), multiple ranks are saved as `my_map_rank{K}.png`.

apps/multimodal/vision-based-pdf-multi-vector/multi-vector-leann-similarity-map.py

@@ -169,7 +169,7 @@ def _embed_images(model, processor, images: list[Image.Image]) -> list[Any]:
     )
 
     doc_vecs: list[Any] = []
-    for batch_doc in tqdm(dataloader, desc="Embedding images"):
+    for batch_doc in dataloader:
         with torch.no_grad():
             batch_doc = {k: v.to(model.device) for k, v in batch_doc.items()}
             # autocast on CUDA for bf16/fp16; on CPU/MPS stay in fp32
@@ -200,7 +200,7 @@ def _embed_queries(model, processor, queries: list[str]) -> list[Any]:
     )
 
     q_vecs: list[Any] = []
-    for batch_query in tqdm(dataloader, desc="Embedding queries"):
+    for batch_query in dataloader:
         with torch.no_grad():
             batch_query = {k: v.to(model.device) for k, v in batch_query.items()}
             if model.device.type == "cuda":
@@ -362,7 +362,7 @@ if USE_HF_DATASET:
     N = len(dataset) if MAX_DOCS is None else min(MAX_DOCS, len(dataset))
     filepaths: list[str] = []
     images: list[Image.Image] = []
-    for i in tqdm(range(N), desc="Loading dataset", total=N ):
+    for i in tqdm(range(N), desc="Loading dataset"):
         p = dataset[i]
         # Compose a descriptive identifier for printing later
         identifier = f"arXiv:{p['paper_arxiv_id']}|title:{p['paper_title']}|page:{int(p['page_number'])}|id:{p['page_id']}"

apps/multimodal/vision-based-pdf-multi-vector/multi-vector-leann.py

@@ -4,24 +4,39 @@
 # pip install tqdm
 # pip install pillow
 
+# %%
+from pdf2image import convert_from_path
+
+pdf_path = "pdfs/2004.12832v2.pdf"
+images = convert_from_path(pdf_path)
+
+for i, image in enumerate(images):
+    image.save(f"pages/page_{i + 1}.png", "PNG")
+
+# %%
 import os
-import re
-import sys
 from pathlib import Path
-from typing import cast
 
-from PIL import Image
-from tqdm import tqdm
-
-# Ensure local leann packages are importable before importing them
+# Make local leann packages importable without installing
 _repo_root = Path(__file__).resolve().parents[3]
 _leann_core_src = _repo_root / "packages" / "leann-core" / "src"
 _leann_hnsw_pkg = _repo_root / "packages" / "leann-backend-hnsw"
+import sys
+
 if str(_leann_core_src) not in sys.path:
     sys.path.append(str(_leann_core_src))
 if str(_leann_hnsw_pkg) not in sys.path:
     sys.path.append(str(_leann_hnsw_pkg))
 
+from leann_multi_vector import LeannMultiVector
+
+
+class LeannRetriever(LeannMultiVector):
+    pass
+
+
+# %%
+from typing import cast
 
 import torch
 from colpali_engine.models import ColPali
@@ -73,6 +88,13 @@ for batch_query in dataloader:
     qs.extend(list(torch.unbind(embeddings_query.to("cpu"))))
 print(qs[0].shape)
 # %%
+
+
+import re
+
+from PIL import Image
+from tqdm import tqdm
+
 page_filenames = sorted(os.listdir("./pages"), key=lambda n: int(re.search(r"\d+", n).group()))
 images = [Image.open(os.path.join("./pages", name)) for name in page_filenames]
 

apps/semantic_file_search/leann-plus-temporal-search.py

@@ -1,183 +0,0 @@
-#!/usr/bin/env python3
-import re
-import sys
-from datetime import datetime, timedelta
-from pathlib import Path
-
-from leann import LeannSearcher
-
-INDEX_PATH = str(Path("./").resolve() / "demo.leann")
-
-
-class TimeParser:
-    def __init__(self):
-        # Main pattern: captures optional fuzzy modifier, number, unit, and optional "ago"
-        self.pattern = r"(?:(around|about|roughly|approximately)\s+)?(\d+)\s+(hour|day|week|month|year)s?(?:\s+ago)?"
-
-        # Compile for performance
-        self.regex = re.compile(self.pattern, re.IGNORECASE)
-
-        # Stop words to remove before regex parsing
-        self.stop_words = {
-            "in",
-            "at",
-            "of",
-            "by",
-            "as",
-            "me",
-            "the",
-            "a",
-            "an",
-            "and",
-            "any",
-            "find",
-            "search",
-            "list",
-            "ago",
-            "back",
-            "past",
-            "earlier",
-        }
-
-    def clean_text(self, text):
-        """Remove stop words from text"""
-        words = text.split()
-        cleaned = " ".join(word for word in words if word.lower() not in self.stop_words)
-        return cleaned
-
-    def parse(self, text):
-        """Extract all time expressions from text"""
-        # Clean text first
-        cleaned_text = self.clean_text(text)
-
-        matches = []
-        for match in self.regex.finditer(cleaned_text):
-            fuzzy = match.group(1)  # "around", "about", etc.
-            number = int(match.group(2))
-            unit = match.group(3).lower()
-
-            matches.append(
-                {
-                    "full_match": match.group(0),
-                    "fuzzy": bool(fuzzy),
-                    "number": number,
-                    "unit": unit,
-                    "range": self.calculate_range(number, unit, bool(fuzzy)),
-                }
-            )
-
-        return matches
-
-    def calculate_range(self, number, unit, is_fuzzy):
-        """Convert to actual datetime range and return ISO format strings"""
-        units = {
-            "hour": timedelta(hours=number),
-            "day": timedelta(days=number),
-            "week": timedelta(weeks=number),
-            "month": timedelta(days=number * 30),
-            "year": timedelta(days=number * 365),
-        }
-
-        delta = units[unit]
-        now = datetime.now()
-        target = now - delta
-
-        if is_fuzzy:
-            buffer = delta * 0.2  # 20% buffer for fuzzy
-            start = (target - buffer).isoformat()
-            end = (target + buffer).isoformat()
-        else:
-            start = target.isoformat()
-            end = now.isoformat()
-
-        return (start, end)
-
-
-def search_files(query, top_k=15):
-    """Search the index and return results"""
-    # Parse time expressions
-    parser = TimeParser()
-    time_matches = parser.parse(query)
-
-    # Remove time expressions from query for semantic search
-    clean_query = query
-    if time_matches:
-        for match in time_matches:
-            clean_query = clean_query.replace(match["full_match"], "").strip()
-
-    # Check if clean_query is less than 4 characters
-    if len(clean_query) < 4:
-        print("Error: add more input for accurate results.")
-        return
-
-    # Single query to vector DB
-    searcher = LeannSearcher(INDEX_PATH)
-    results = searcher.search(
-        clean_query if clean_query else query, top_k=top_k, recompute_embeddings=False
-    )
-
-    # Filter by time if time expression found
-    if time_matches:
-        time_range = time_matches[0]["range"]  # Use first time expression
-        start_time, end_time = time_range
-
-        filtered_results = []
-        for result in results:
-            # Access metadata attribute directly (not .get())
-            metadata = result.metadata if hasattr(result, "metadata") else {}
-
-            if metadata:
-                # Check modification date first, fall back to creation date
-                date_str = metadata.get("modification_date") or metadata.get("creation_date")
-
-                if date_str:
-                    # Convert strings to datetime objects for proper comparison
-                    try:
-                        file_date = datetime.fromisoformat(date_str)
-                        start_dt = datetime.fromisoformat(start_time)
-                        end_dt = datetime.fromisoformat(end_time)
-
-                        # Compare dates properly
-                        if start_dt <= file_date <= end_dt:
-                            filtered_results.append(result)
-                    except (ValueError, TypeError):
-                        # Handle invalid date formats
-                        print(f"Warning: Invalid date format in metadata: {date_str}")
-                        continue
-
-        results = filtered_results
-
-    # Print results
-    print(f"\nSearch results for: '{query}'")
-    if time_matches:
-        print(
-            f"Time filter: {time_matches[0]['number']} {time_matches[0]['unit']}(s) {'(fuzzy)' if time_matches[0]['fuzzy'] else ''}"
-        )
-        print(
-            f"Date range: {time_matches[0]['range'][0][:10]} to {time_matches[0]['range'][1][:10]}"
-        )
-    print("-" * 80)
-
-    for i, result in enumerate(results, 1):
-        print(f"\n[{i}] Score: {result.score:.4f}")
-        print(f"Content: {result.text}")
-
-        # Show metadata if present
-        metadata = result.metadata if hasattr(result, "metadata") else None
-        if metadata:
-            if "creation_date" in metadata:
-                print(f"Created: {metadata['creation_date']}")
-            if "modification_date" in metadata:
-                print(f"Modified: {metadata['modification_date']}")
-        print("-" * 80)
-
-
-if __name__ == "__main__":
-    if len(sys.argv) < 2:
-        print('Usage: python search_index.py "<search query>" [top_k]')
-        sys.exit(1)
-
-    query = sys.argv[1]
-    top_k = int(sys.argv[2]) if len(sys.argv) > 2 else 15
-
-    search_files(query, top_k)

apps/semantic_file_search/leann_index_builder.py

@@ -1,82 +0,0 @@
-#!/usr/bin/env python3
-import json
-import sys
-from pathlib import Path
-
-from leann import LeannBuilder
-
-
-def process_json_items(json_file_path):
-    """Load and process JSON file with metadata items"""
-
-    with open(json_file_path, encoding="utf-8") as f:
-        items = json.load(f)
-
-    # Guard against empty JSON
-    if not items:
-        print("⚠️  No items found in the JSON file. Exiting gracefully.")
-        return
-
-    INDEX_PATH = str(Path("./").resolve() / "demo.leann")
-    builder = LeannBuilder(backend_name="hnsw", is_recompute=False)
-
-    total_items = len(items)
-    items_added = 0
-    print(f"Processing {total_items} items...")
-
-    for idx, item in enumerate(items):
-        try:
-            # Create embedding text sentence
-            embedding_text = f"{item.get('Name', 'unknown')} located at {item.get('Path', 'unknown')} and size {item.get('Size', 'unknown')} bytes with content type {item.get('ContentType', 'unknown')} and kind {item.get('Kind', 'unknown')}"
-
-            # Prepare metadata with dates
-            metadata = {}
-            if "CreationDate" in item:
-                metadata["creation_date"] = item["CreationDate"]
-            if "ContentChangeDate" in item:
-                metadata["modification_date"] = item["ContentChangeDate"]
-
-            # Add to builder
-            builder.add_text(embedding_text, metadata=metadata)
-            items_added += 1
-
-        except Exception as e:
-            print(f"\n⚠️  Warning: Failed to process item {idx}: {e}")
-            continue
-
-        # Show progress
-        progress = (idx + 1) / total_items * 100
-        sys.stdout.write(f"\rProgress: {idx + 1}/{total_items} ({progress:.1f}%)")
-        sys.stdout.flush()
-
-    print()  # New line after progress
-
-    # Guard against no successfully added items
-    if items_added == 0:
-        print("⚠️  No items were successfully added to the index. Exiting gracefully.")
-        return
-
-    print(f"\n✅ Successfully processed {items_added}/{total_items} items")
-    print("Building index...")
-
-    try:
-        builder.build_index(INDEX_PATH)
-        print(f"✓ Index saved to {INDEX_PATH}")
-    except ValueError as e:
-        if "No chunks added" in str(e):
-            print("⚠️  No chunks were added to the builder. Index not created.")
-        else:
-            raise
-
-
-if __name__ == "__main__":
-    if len(sys.argv) != 2:
-        print("Usage: python build_index.py <json_file>")
-        sys.exit(1)
-
-    json_file = sys.argv[1]
-    if not Path(json_file).exists():
-        print(f"Error: File {json_file} not found")
-        sys.exit(1)
-
-    process_json_items(json_file)

apps/semantic_file_search/spotlight_index_dump.py

@@ -1,265 +0,0 @@
-#!/usr/bin/env python3
-"""
-Spotlight Metadata Dumper for Vector DB
-Extracts only essential metadata for semantic search embeddings
-Output is optimized for vector database storage with minimal fields
-"""
-
-import json
-import sys
-from datetime import datetime
-
-# Check platform before importing macOS-specific modules
-if sys.platform != "darwin":
-    print("This script requires macOS (uses Spotlight)")
-    sys.exit(1)
-
-from Foundation import NSDate, NSMetadataQuery, NSPredicate, NSRunLoop
-
-# EDIT THIS LIST: Add or remove folders to search
-# Can be either:
-# - Folder names relative to home directory (e.g., "Desktop", "Downloads")
-# - Absolute paths (e.g., "/Applications", "/System/Library")
-SEARCH_FOLDERS = [
-    "Desktop",
-    "Downloads",
-    "Documents",
-    "Music",
-    "Pictures",
-    "Movies",
-    # "Library",  # Uncomment to include
-    # "/Applications",  # Absolute path example
-    # "Code/Projects",  # Subfolder example
-    # Add any other folders here
-]
-
-
-def convert_to_serializable(obj):
-    """Convert NS objects to Python serializable types"""
-    if obj is None:
-        return None
-
-    # Handle NSDate
-    if hasattr(obj, "timeIntervalSince1970"):
-        return datetime.fromtimestamp(obj.timeIntervalSince1970()).isoformat()
-
-    # Handle NSArray
-    if hasattr(obj, "count") and hasattr(obj, "objectAtIndex_"):
-        return [convert_to_serializable(obj.objectAtIndex_(i)) for i in range(obj.count())]
-
-    # Convert to string
-    try:
-        return str(obj)
-    except Exception:
-        return repr(obj)
-
-
-def dump_spotlight_data(max_items=10, output_file="spotlight_dump.json"):
-    """
-    Dump Spotlight data using public.item predicate
-    """
-    # Build full paths from SEARCH_FOLDERS
-    import os
-
-    home_dir = os.path.expanduser("~")
-    search_paths = []
-
-    print("Search locations:")
-    for folder in SEARCH_FOLDERS:
-        # Check if it's an absolute path or relative
-        if folder.startswith("/"):
-            full_path = folder
-        else:
-            full_path = os.path.join(home_dir, folder)
-
-        if os.path.exists(full_path):
-            search_paths.append(full_path)
-            print(f"  ✓ {full_path}")
-        else:
-            print(f"  ✗ {full_path} (not found)")
-
-    if not search_paths:
-        print("No valid search paths found!")
-        return []
-
-    print(f"\nDumping {max_items} items from Spotlight (public.item)...")
-
-    # Create query with public.item predicate
-    query = NSMetadataQuery.alloc().init()
-    predicate = NSPredicate.predicateWithFormat_("kMDItemContentTypeTree CONTAINS 'public.item'")
-    query.setPredicate_(predicate)
-
-    # Set search scopes to our specific folders
-    query.setSearchScopes_(search_paths)
-
-    print("Starting query...")
-    query.startQuery()
-
-    # Wait for gathering to complete
-    run_loop = NSRunLoop.currentRunLoop()
-    print("Gathering results...")
-
-    # Let it gather for a few seconds
-    for i in range(50):  # 5 seconds max
-        run_loop.runMode_beforeDate_(
-            "NSDefaultRunLoopMode", NSDate.dateWithTimeIntervalSinceNow_(0.1)
-        )
-        # Check gathering status periodically
-        if i % 10 == 0:
-            current_count = query.resultCount()
-            if current_count > 0:
-                print(f"  Found {current_count} items so far...")
-
-    # Continue while still gathering (up to 2 more seconds)
-    timeout = NSDate.dateWithTimeIntervalSinceNow_(2.0)
-    while query.isGathering() and timeout.timeIntervalSinceNow() > 0:
-        run_loop.runMode_beforeDate_(
-            "NSDefaultRunLoopMode", NSDate.dateWithTimeIntervalSinceNow_(0.1)
-        )
-
-    query.stopQuery()
-
-    total_results = query.resultCount()
-    print(f"Found {total_results} total items")
-
-    if total_results == 0:
-        print("No results found")
-        return []
-
-    # Process items
-    items_to_process = min(total_results, max_items)
-    results = []
-
-    # ONLY relevant attributes for vector embeddings
-    # These provide essential context for semantic search without bloat
-    attributes = [
-        "kMDItemPath",  # Full path for file retrieval
-        "kMDItemFSName",  # Filename for display & embedding
-        "kMDItemFSSize",  # Size for filtering/ranking
-        "kMDItemContentType",  # File type for categorization
-        "kMDItemKind",  # Human-readable type for embedding
-        "kMDItemFSCreationDate",  # Temporal context
-        "kMDItemFSContentChangeDate",  # Recency for ranking
-    ]
-
-    print(f"Processing {items_to_process} items...")
-
-    for i in range(items_to_process):
-        try:
-            item = query.resultAtIndex_(i)
-            metadata = {}
-
-            # Extract ONLY the relevant attributes
-            for attr in attributes:
-                try:
-                    value = item.valueForAttribute_(attr)
-                    if value is not None:
-                        # Keep the attribute name clean (remove kMDItem prefix for cleaner JSON)
-                        clean_key = attr.replace("kMDItem", "").replace("FS", "")
-                        metadata[clean_key] = convert_to_serializable(value)
-                except (AttributeError, ValueError, TypeError):
-                    continue
-
-            # Only add if we have at least a path
-            if metadata.get("Path"):
-                results.append(metadata)
-
-        except Exception as e:
-            print(f"Error processing item {i}: {e}")
-            continue
-
-    # Save to JSON
-    with open(output_file, "w", encoding="utf-8") as f:
-        json.dump(results, f, indent=2, ensure_ascii=False)
-
-    print(f"\n✓ Saved {len(results)} items to {output_file}")
-
-    # Show summary
-    print("\nSample items:")
-    import os
-
-    home_dir = os.path.expanduser("~")
-
-    for i, item in enumerate(results[:3]):
-        print(f"\n[Item {i + 1}]")
-        print(f"  Path: {item.get('Path', 'N/A')}")
-        print(f"  Name: {item.get('Name', 'N/A')}")
-        print(f"  Type: {item.get('ContentType', 'N/A')}")
-        print(f"  Kind: {item.get('Kind', 'N/A')}")
-
-        # Handle size properly
-        size = item.get("Size")
-        if size:
-            try:
-                size_int = int(size)
-                if size_int > 1024 * 1024:
-                    print(f"  Size: {size_int / (1024 * 1024):.2f} MB")
-                elif size_int > 1024:
-                    print(f"  Size: {size_int / 1024:.2f} KB")
-                else:
-                    print(f"  Size: {size_int} bytes")
-            except (ValueError, TypeError):
-                print(f"  Size: {size}")
-
-        # Show dates
-        if "CreationDate" in item:
-            print(f"  Created: {item['CreationDate']}")
-        if "ContentChangeDate" in item:
-            print(f"  Modified: {item['ContentChangeDate']}")
-
-    # Count by type
-    type_counts = {}
-    for item in results:
-        content_type = item.get("ContentType", "unknown")
-        type_counts[content_type] = type_counts.get(content_type, 0) + 1
-
-    print(f"\nTotal items saved: {len(results)}")
-
-    if type_counts:
-        print("\nTop content types:")
-        for ct, count in sorted(type_counts.items(), key=lambda x: x[1], reverse=True)[:5]:
-            print(f"  {ct}: {count} items")
-
-    # Count by folder
-    folder_counts = {}
-    for item in results:
-        path = item.get("Path", "")
-        for folder in SEARCH_FOLDERS:
-            # Build the full folder path
-            if folder.startswith("/"):
-                folder_path = folder
-            else:
-                folder_path = os.path.join(home_dir, folder)
-
-            if path.startswith(folder_path):
-                folder_counts[folder] = folder_counts.get(folder, 0) + 1
-                break
-
-    if folder_counts:
-        print("\nItems by location:")
-        for folder, count in sorted(folder_counts.items(), key=lambda x: x[1], reverse=True):
-            print(f"  {folder}: {count} items")
-
-    return results
-
-
-def main():
-    # Parse arguments
-    if len(sys.argv) > 1:
-        try:
-            max_items = int(sys.argv[1])
-        except ValueError:
-            print("Usage: python spot.py [number_of_items]")
-            print("Default: 10 items")
-            sys.exit(1)
-    else:
-        max_items = 10
-
-    output_file = sys.argv[2] if len(sys.argv) > 2 else "spotlight_dump.json"
-
-    # Run dump
-    dump_spotlight_data(max_items=max_items, output_file=output_file)
-
-
-if __name__ == "__main__":
-    main()

apps/slack_data/__init__.py

@@ -1 +0,0 @@
-# Slack MCP data integration for LEANN

apps/slack_data/slack_mcp_reader.py

@@ -1,510 +0,0 @@
-#!/usr/bin/env python3
-"""
-Slack MCP Reader for LEANN
-
-This module provides functionality to connect to Slack MCP servers and fetch message data
-for indexing in LEANN. It supports various Slack MCP server implementations and provides
-flexible message processing options.
-"""
-
-import asyncio
-import json
-import logging
-from typing import Any, Optional
-
-logger = logging.getLogger(__name__)
-
-
-class SlackMCPReader:
-    """
-    Reader for Slack data via MCP (Model Context Protocol) servers.
-
-    This class connects to Slack MCP servers to fetch message data and convert it
-    into a format suitable for LEANN indexing.
-    """
-
-    def __init__(
-        self,
-        mcp_server_command: str,
-        workspace_name: Optional[str] = None,
-        concatenate_conversations: bool = True,
-        max_messages_per_conversation: int = 100,
-        max_retries: int = 5,
-        retry_delay: float = 2.0,
-    ):
-        """
-        Initialize the Slack MCP Reader.
-
-        Args:
-            mcp_server_command: Command to start the MCP server (e.g., 'slack-mcp-server')
-            workspace_name: Optional workspace name to filter messages
-            concatenate_conversations: Whether to group messages by channel/thread
-            max_messages_per_conversation: Maximum messages to include per conversation
-            max_retries: Maximum number of retries for failed operations
-            retry_delay: Initial delay between retries in seconds
-        """
-        self.mcp_server_command = mcp_server_command
-        self.workspace_name = workspace_name
-        self.concatenate_conversations = concatenate_conversations
-        self.max_messages_per_conversation = max_messages_per_conversation
-        self.max_retries = max_retries
-        self.retry_delay = retry_delay
-        self.mcp_process = None
-
-    async def start_mcp_server(self):
-        """Start the MCP server process."""
-        try:
-            self.mcp_process = await asyncio.create_subprocess_exec(
-                *self.mcp_server_command.split(),
-                stdin=asyncio.subprocess.PIPE,
-                stdout=asyncio.subprocess.PIPE,
-                stderr=asyncio.subprocess.PIPE,
-            )
-            logger.info(f"Started MCP server: {self.mcp_server_command}")
-        except Exception as e:
-            logger.error(f"Failed to start MCP server: {e}")
-            raise
-
-    async def stop_mcp_server(self):
-        """Stop the MCP server process."""
-        if self.mcp_process:
-            self.mcp_process.terminate()
-            await self.mcp_process.wait()
-            logger.info("Stopped MCP server")
-
-    async def send_mcp_request(self, request: dict[str, Any]) -> dict[str, Any]:
-        """Send a request to the MCP server and get response."""
-        if not self.mcp_process:
-            raise RuntimeError("MCP server not started")
-
-        request_json = json.dumps(request) + "\n"
-        self.mcp_process.stdin.write(request_json.encode())
-        await self.mcp_process.stdin.drain()
-
-        response_line = await self.mcp_process.stdout.readline()
-        if not response_line:
-            raise RuntimeError("No response from MCP server")
-
-        return json.loads(response_line.decode().strip())
-
-    async def initialize_mcp_connection(self):
-        """Initialize the MCP connection."""
-        init_request = {
-            "jsonrpc": "2.0",
-            "id": 1,
-            "method": "initialize",
-            "params": {
-                "protocolVersion": "2024-11-05",
-                "capabilities": {},
-                "clientInfo": {"name": "leann-slack-reader", "version": "1.0.0"},
-            },
-        }
-
-        response = await self.send_mcp_request(init_request)
-        if "error" in response:
-            raise RuntimeError(f"MCP initialization failed: {response['error']}")
-
-        logger.info("MCP connection initialized successfully")
-
-    async def list_available_tools(self) -> list[dict[str, Any]]:
-        """List available tools from the MCP server."""
-        list_request = {"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}}
-
-        response = await self.send_mcp_request(list_request)
-        if "error" in response:
-            raise RuntimeError(f"Failed to list tools: {response['error']}")
-
-        return response.get("result", {}).get("tools", [])
-
-    def _is_cache_sync_error(self, error: dict) -> bool:
-        """Check if the error is related to users cache not being ready."""
-        if isinstance(error, dict):
-            message = error.get("message", "").lower()
-            return (
-                "users cache is not ready" in message or "sync process is still running" in message
-            )
-        return False
-
-    async def _retry_with_backoff(self, func, *args, **kwargs):
-        """Retry a function with exponential backoff, especially for cache sync issues."""
-        last_exception = None
-
-        for attempt in range(self.max_retries + 1):
-            try:
-                return await func(*args, **kwargs)
-            except Exception as e:
-                last_exception = e
-
-                # Check if this is a cache sync error
-                error_dict = {}
-                if hasattr(e, "args") and e.args and isinstance(e.args[0], dict):
-                    error_dict = e.args[0]
-                elif "Failed to fetch messages" in str(e):
-                    # Try to extract error from the exception message
-                    import re
-
-                    match = re.search(r"'error':\s*(\{[^}]+\})", str(e))
-                    if match:
-                        try:
-                            error_dict = eval(match.group(1))
-                        except (ValueError, SyntaxError, NameError):
-                            pass
-                    else:
-                        # Try alternative format
-                        match = re.search(r"Failed to fetch messages:\s*(\{[^}]+\})", str(e))
-                        if match:
-                            try:
-                                error_dict = eval(match.group(1))
-                            except (ValueError, SyntaxError, NameError):
-                                pass
-
-                if self._is_cache_sync_error(error_dict):
-                    if attempt < self.max_retries:
-                        delay = self.retry_delay * (2**attempt)  # Exponential backoff
-                        logger.info(
-                            f"Cache sync not ready, waiting {delay:.1f}s before retry {attempt + 1}/{self.max_retries}"
-                        )
-                        await asyncio.sleep(delay)
-                        continue
-                    else:
-                        logger.warning(
-                            f"Cache sync still not ready after {self.max_retries} retries, giving up"
-                        )
-                        break
-                else:
-                    # Not a cache sync error, don't retry
-                    break
-
-        # If we get here, all retries failed or it's not a retryable error
-        raise last_exception
-
-    async def fetch_slack_messages(
-        self, channel: Optional[str] = None, limit: int = 100
-    ) -> list[dict[str, Any]]:
-        """
-        Fetch Slack messages using MCP tools with retry logic for cache sync issues.
-
-        Args:
-            channel: Optional channel name to filter messages
-            limit: Maximum number of messages to fetch
-
-        Returns:
-            List of message dictionaries
-        """
-        return await self._retry_with_backoff(self._fetch_slack_messages_impl, channel, limit)
-
-    async def _fetch_slack_messages_impl(
-        self, channel: Optional[str] = None, limit: int = 100
-    ) -> list[dict[str, Any]]:
-        """
-        Internal implementation of fetch_slack_messages without retry logic.
-        """
-        # This is a generic implementation - specific MCP servers may have different tool names
-        # Common tool names might be: 'get_messages', 'list_messages', 'fetch_channel_history'
-
-        tools = await self.list_available_tools()
-        logger.info(f"Available tools: {[tool.get('name') for tool in tools]}")
-        message_tool = None
-
-        # Look for a tool that can fetch messages - prioritize conversations_history
-        message_tool = None
-
-        # First, try to find conversations_history specifically
-        for tool in tools:
-            tool_name = tool.get("name", "").lower()
-            if "conversations_history" in tool_name:
-                message_tool = tool
-                logger.info(f"Found conversations_history tool: {tool}")
-                break
-
-        # If not found, look for other message-fetching tools
-        if not message_tool:
-            for tool in tools:
-                tool_name = tool.get("name", "").lower()
-                if any(
-                    keyword in tool_name
-                    for keyword in ["conversations_search", "message", "history"]
-                ):
-                    message_tool = tool
-                    break
-
-        if not message_tool:
-            raise RuntimeError("No message fetching tool found in MCP server")
-
-        # Prepare tool call parameters
-        tool_params = {"limit": "180d"}  # Use 180 days to get older messages
-        if channel:
-            # For conversations_history, use channel_id parameter
-            if message_tool["name"] == "conversations_history":
-                tool_params["channel_id"] = channel
-            else:
-                # Try common parameter names for channel specification
-                for param_name in ["channel", "channel_id", "channel_name"]:
-                    tool_params[param_name] = channel
-                    break
-
-        logger.info(f"Tool parameters: {tool_params}")
-
-        fetch_request = {
-            "jsonrpc": "2.0",
-            "id": 3,
-            "method": "tools/call",
-            "params": {"name": message_tool["name"], "arguments": tool_params},
-        }
-
-        response = await self.send_mcp_request(fetch_request)
-        if "error" in response:
-            raise RuntimeError(f"Failed to fetch messages: {response['error']}")
-
-        # Extract messages from response - format may vary by MCP server
-        result = response.get("result", {})
-        if "content" in result and isinstance(result["content"], list):
-            # Some MCP servers return content as a list
-            content = result["content"][0] if result["content"] else {}
-            if "text" in content:
-                try:
-                    messages = json.loads(content["text"])
-                except json.JSONDecodeError:
-                    # If not JSON, try to parse as CSV format (Slack MCP server format)
-                    messages = self._parse_csv_messages(content["text"], channel)
-            else:
-                messages = result["content"]
-        else:
-            # Direct message format
-            messages = result.get("messages", [result])
-
-        return messages if isinstance(messages, list) else [messages]
-
-    def _parse_csv_messages(self, csv_text: str, channel: str) -> list[dict[str, Any]]:
-        """Parse CSV format messages from Slack MCP server."""
-        import csv
-        import io
-
-        messages = []
-        try:
-            # Split by lines and process each line as a CSV row
-            lines = csv_text.strip().split("\n")
-            if not lines:
-                return messages
-
-            # Skip header line if it exists
-            start_idx = 0
-            if lines[0].startswith("MsgID,UserID,UserName"):
-                start_idx = 1
-
-            for line in lines[start_idx:]:
-                if not line.strip():
-                    continue
-
-                # Parse CSV line
-                reader = csv.reader(io.StringIO(line))
-                try:
-                    row = next(reader)
-                    if len(row) >= 7:  # Ensure we have enough columns
-                        message = {
-                            "ts": row[0],
-                            "user": row[1],
-                            "username": row[2],
-                            "real_name": row[3],
-                            "channel": row[4],
-                            "thread_ts": row[5],
-                            "text": row[6],
-                            "time": row[7] if len(row) > 7 else "",
-                            "reactions": row[8] if len(row) > 8 else "",
-                            "cursor": row[9] if len(row) > 9 else "",
-                        }
-                        messages.append(message)
-                except Exception as e:
-                    logger.warning(f"Failed to parse CSV line: {line[:100]}... Error: {e}")
-                    continue
-
-        except Exception as e:
-            logger.warning(f"Failed to parse CSV messages: {e}")
-            # Fallback: treat entire text as one message
-            messages = [{"text": csv_text, "channel": channel or "unknown"}]
-
-        return messages
-
-    def _format_message(self, message: dict[str, Any]) -> str:
-        """Format a single message for indexing."""
-        text = message.get("text", "")
-        user = message.get("user", message.get("username", "Unknown"))
-        channel = message.get("channel", message.get("channel_name", "Unknown"))
-        timestamp = message.get("ts", message.get("timestamp", ""))
-
-        # Format timestamp if available
-        formatted_time = ""
-        if timestamp:
-            try:
-                import datetime
-
-                if isinstance(timestamp, str) and "." in timestamp:
-                    dt = datetime.datetime.fromtimestamp(float(timestamp))
-                    formatted_time = dt.strftime("%Y-%m-%d %H:%M:%S")
-                elif isinstance(timestamp, (int, float)):
-                    dt = datetime.datetime.fromtimestamp(timestamp)
-                    formatted_time = dt.strftime("%Y-%m-%d %H:%M:%S")
-                else:
-                    formatted_time = str(timestamp)
-            except (ValueError, TypeError):
-                formatted_time = str(timestamp)
-
-        # Build formatted message
-        parts = []
-        if channel:
-            parts.append(f"Channel: #{channel}")
-        if user:
-            parts.append(f"User: {user}")
-        if formatted_time:
-            parts.append(f"Time: {formatted_time}")
-        if text:
-            parts.append(f"Message: {text}")
-
-        return "\n".join(parts)
-
-    def _create_concatenated_content(self, messages: list[dict[str, Any]], channel: str) -> str:
-        """Create concatenated content from multiple messages in a channel."""
-        if not messages:
-            return ""
-
-        # Sort messages by timestamp if available
-        try:
-            messages.sort(key=lambda x: float(x.get("ts", x.get("timestamp", 0))))
-        except (ValueError, TypeError):
-            pass  # Keep original order if timestamps aren't numeric
-
-        # Limit messages per conversation
-        if len(messages) > self.max_messages_per_conversation:
-            messages = messages[-self.max_messages_per_conversation :]
-
-        # Create header
-        content_parts = [
-            f"Slack Channel: #{channel}",
-            f"Message Count: {len(messages)}",
-            f"Workspace: {self.workspace_name or 'Unknown'}",
-            "=" * 50,
-            "",
-        ]
-
-        # Add messages
-        for message in messages:
-            formatted_msg = self._format_message(message)
-            if formatted_msg.strip():
-                content_parts.append(formatted_msg)
-                content_parts.append("-" * 30)
-                content_parts.append("")
-
-        return "\n".join(content_parts)
-
-    async def get_all_channels(self) -> list[str]:
-        """Get list of all available channels."""
-        try:
-            channels_list_request = {
-                "jsonrpc": "2.0",
-                "id": 4,
-                "method": "tools/call",
-                "params": {"name": "channels_list", "arguments": {}},
-            }
-            channels_response = await self.send_mcp_request(channels_list_request)
-            if "result" in channels_response:
-                result = channels_response["result"]
-                if "content" in result and isinstance(result["content"], list):
-                    content = result["content"][0] if result["content"] else {}
-                    if "text" in content:
-                        # Parse the channels from the response
-                        channels = []
-                        lines = content["text"].split("\n")
-                        for line in lines:
-                            if line.strip() and ("#" in line or "C" in line[:10]):
-                                # Extract channel ID or name
-                                parts = line.split()
-                                for part in parts:
-                                    if part.startswith("C") and len(part) > 5:
-                                        channels.append(part)
-                                    elif part.startswith("#"):
-                                        channels.append(part[1:])  # Remove #
-                        logger.info(f"Found {len(channels)} channels: {channels}")
-                        return channels
-            return []
-        except Exception as e:
-            logger.warning(f"Failed to get channels list: {e}")
-            return []
-
-    async def read_slack_data(self, channels: Optional[list[str]] = None) -> list[str]:
-        """
-        Read Slack data and return formatted text chunks.
-
-        Args:
-            channels: Optional list of channel names to fetch. If None, fetches from all available channels.
-
-        Returns:
-            List of formatted text chunks ready for LEANN indexing
-        """
-        try:
-            await self.start_mcp_server()
-            await self.initialize_mcp_connection()
-
-            all_texts = []
-
-            if channels:
-                # Fetch specific channels
-                for channel in channels:
-                    try:
-                        messages = await self.fetch_slack_messages(channel=channel, limit=1000)
-                        if messages:
-                            if self.concatenate_conversations:
-                                text_content = self._create_concatenated_content(messages, channel)
-                                if text_content.strip():
-                                    all_texts.append(text_content)
-                            else:
-                                # Process individual messages
-                                for message in messages:
-                                    formatted_msg = self._format_message(message)
-                                    if formatted_msg.strip():
-                                        all_texts.append(formatted_msg)
-                    except Exception as e:
-                        logger.warning(f"Failed to fetch messages from channel {channel}: {e}")
-                        continue
-            else:
-                # Fetch from all available channels
-                logger.info("Fetching from all available channels...")
-                all_channels = await self.get_all_channels()
-
-                if not all_channels:
-                    # Fallback to common channel names if we can't get the list
-                    all_channels = ["general", "random", "announcements", "C0GN5BX0F"]
-                    logger.info(f"Using fallback channels: {all_channels}")
-
-                for channel in all_channels:
-                    try:
-                        logger.info(f"Searching channel: {channel}")
-                        messages = await self.fetch_slack_messages(channel=channel, limit=1000)
-                        if messages:
-                            if self.concatenate_conversations:
-                                text_content = self._create_concatenated_content(messages, channel)
-                                if text_content.strip():
-                                    all_texts.append(text_content)
-                            else:
-                                # Process individual messages
-                                for message in messages:
-                                    formatted_msg = self._format_message(message)
-                                    if formatted_msg.strip():
-                                        all_texts.append(formatted_msg)
-                    except Exception as e:
-                        logger.warning(f"Failed to fetch messages from channel {channel}: {e}")
-                        continue
-
-            return all_texts
-
-        finally:
-            await self.stop_mcp_server()
-
-    async def __aenter__(self):
-        """Async context manager entry."""
-        await self.start_mcp_server()
-        await self.initialize_mcp_connection()
-        return self
-
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
-        """Async context manager exit."""
-        await self.stop_mcp_server()

apps/slack_rag.py

@@ -1,227 +0,0 @@
-#!/usr/bin/env python3
-"""
-Slack RAG Application with MCP Support
-
-This application enables RAG (Retrieval-Augmented Generation) on Slack messages
-by connecting to Slack MCP servers to fetch live data and index it in LEANN.
-
-Usage:
-    python -m apps.slack_rag --mcp-server "slack-mcp-server" --query "What did the team discuss about the project?"
-"""
-
-import argparse
-import asyncio
-
-from apps.base_rag_example import BaseRAGExample
-from apps.slack_data.slack_mcp_reader import SlackMCPReader
-
-
-class SlackMCPRAG(BaseRAGExample):
-    """
-    RAG application for Slack messages via MCP servers.
-
-    This class provides a complete RAG pipeline for Slack data, including
-    MCP server connection, data fetching, indexing, and interactive chat.
-    """
-
-    def __init__(self):
-        super().__init__(
-            name="Slack MCP RAG",
-            description="RAG application for Slack messages via MCP servers",
-            default_index_name="slack_messages",
-        )
-
-    def _add_specific_arguments(self, parser: argparse.ArgumentParser):
-        """Add Slack MCP-specific arguments."""
-        parser.add_argument(
-            "--mcp-server",
-            type=str,
-            required=True,
-            help="Command to start the Slack MCP server (e.g., 'slack-mcp-server' or 'npx slack-mcp-server')",
-        )
-
-        parser.add_argument(
-            "--workspace-name",
-            type=str,
-            help="Slack workspace name for better organization and filtering",
-        )
-
-        parser.add_argument(
-            "--channels",
-            nargs="+",
-            help="Specific Slack channels to index (e.g., general random). If not specified, fetches from all available channels",
-        )
-
-        parser.add_argument(
-            "--concatenate-conversations",
-            action="store_true",
-            default=True,
-            help="Group messages by channel/thread for better context (default: True)",
-        )
-
-        parser.add_argument(
-            "--no-concatenate-conversations",
-            action="store_true",
-            help="Process individual messages instead of grouping by channel",
-        )
-
-        parser.add_argument(
-            "--max-messages-per-channel",
-            type=int,
-            default=100,
-            help="Maximum number of messages to include per channel (default: 100)",
-        )
-
-        parser.add_argument(
-            "--test-connection",
-            action="store_true",
-            help="Test MCP server connection and list available tools without indexing",
-        )
-
-        parser.add_argument(
-            "--max-retries",
-            type=int,
-            default=5,
-            help="Maximum number of retries for failed operations (default: 5)",
-        )
-
-        parser.add_argument(
-            "--retry-delay",
-            type=float,
-            default=2.0,
-            help="Initial delay between retries in seconds (default: 2.0)",
-        )
-
-    async def test_mcp_connection(self, args) -> bool:
-        """Test the MCP server connection and display available tools."""
-        print(f"Testing connection to MCP server: {args.mcp_server}")
-
-        try:
-            reader = SlackMCPReader(
-                mcp_server_command=args.mcp_server,
-                workspace_name=args.workspace_name,
-                concatenate_conversations=not args.no_concatenate_conversations,
-                max_messages_per_conversation=args.max_messages_per_channel,
-                max_retries=args.max_retries,
-                retry_delay=args.retry_delay,
-            )
-
-            async with reader:
-                tools = await reader.list_available_tools()
-
-                print("Successfully connected to MCP server!")
-                print(f"Available tools ({len(tools)}):")
-
-                for i, tool in enumerate(tools, 1):
-                    name = tool.get("name", "Unknown")
-                    description = tool.get("description", "No description available")
-                    print(f"\n{i}. {name}")
-                    print(
-                        f"   Description: {description[:100]}{'...' if len(description) > 100 else ''}"
-                    )
-
-                    # Show input schema if available
-                    schema = tool.get("inputSchema", {})
-                    if schema.get("properties"):
-                        props = list(schema["properties"].keys())[:3]  # Show first 3 properties
-                        print(
-                            f"   Parameters: {', '.join(props)}{'...' if len(schema['properties']) > 3 else ''}"
-                        )
-
-                return True
-
-        except Exception as e:
-            print(f"Failed to connect to MCP server: {e}")
-            print("\nTroubleshooting tips:")
-            print("1. Make sure the MCP server is installed and accessible")
-            print("2. Check if the server command is correct")
-            print("3. Ensure you have proper authentication/credentials configured")
-            print("4. Try running the MCP server command directly to test it")
-            return False
-
-    async def load_data(self, args) -> list[str]:
-        """Load Slack messages via MCP server."""
-        print(f"Connecting to Slack MCP server: {args.mcp_server}")
-
-        if args.workspace_name:
-            print(f"Workspace: {args.workspace_name}")
-
-        # Filter out empty strings from channels
-        channels = [ch for ch in args.channels if ch.strip()] if args.channels else None
-
-        if channels:
-            print(f"Channels: {', '.join(channels)}")
-        else:
-            print("Fetching from all available channels")
-
-        concatenate = not args.no_concatenate_conversations
-        print(
-            f"Processing mode: {'Concatenated conversations' if concatenate else 'Individual messages'}"
-        )
-
-        try:
-            reader = SlackMCPReader(
-                mcp_server_command=args.mcp_server,
-                workspace_name=args.workspace_name,
-                concatenate_conversations=concatenate,
-                max_messages_per_conversation=args.max_messages_per_channel,
-                max_retries=args.max_retries,
-                retry_delay=args.retry_delay,
-            )
-
-            texts = await reader.read_slack_data(channels=channels)
-
-            if not texts:
-                print("No messages found! This could mean:")
-                print("- The MCP server couldn't fetch messages")
-                print("- The specified channels don't exist or are empty")
-                print("- Authentication issues with the Slack workspace")
-                return []
-
-            print(f"Successfully loaded {len(texts)} text chunks from Slack")
-
-            # Show sample of what was loaded
-            if texts:
-                sample_text = texts[0][:200] + "..." if len(texts[0]) > 200 else texts[0]
-                print("\nSample content:")
-                print("-" * 40)
-                print(sample_text)
-                print("-" * 40)
-
-            return texts
-
-        except Exception as e:
-            print(f"Error loading Slack data: {e}")
-            print("\nThis might be due to:")
-            print("- MCP server connection issues")
-            print("- Authentication problems")
-            print("- Network connectivity issues")
-            print("- Incorrect channel names")
-            raise
-
-    async def run(self):
-        """Main entry point with MCP connection testing."""
-        args = self.parser.parse_args()
-
-        # Test connection if requested
-        if args.test_connection:
-            success = await self.test_mcp_connection(args)
-            if not success:
-                return
-            print(
-                "MCP server is working! You can now run without --test-connection to start indexing."
-            )
-            return
-
-        # Run the standard RAG pipeline
-        await super().run()
-
-
-async def main():
-    """Main entry point for the Slack MCP RAG application."""
-    app = SlackMCPRAG()
-    await app.run()
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

apps/twitter_data/__init__.py

@@ -1 +0,0 @@
-# Twitter MCP data integration for LEANN

apps/twitter_data/twitter_mcp_reader.py

@@ -1,295 +0,0 @@
-#!/usr/bin/env python3
-"""
-Twitter MCP Reader for LEANN
-
-This module provides functionality to connect to Twitter MCP servers and fetch bookmark data
-for indexing in LEANN. It supports various Twitter MCP server implementations and provides
-flexible bookmark processing options.
-"""
-
-import asyncio
-import json
-import logging
-from typing import Any, Optional
-
-logger = logging.getLogger(__name__)
-
-
-class TwitterMCPReader:
-    """
-    Reader for Twitter bookmark data via MCP (Model Context Protocol) servers.
-
-    This class connects to Twitter MCP servers to fetch bookmark data and convert it
-    into a format suitable for LEANN indexing.
-    """
-
-    def __init__(
-        self,
-        mcp_server_command: str,
-        username: Optional[str] = None,
-        include_tweet_content: bool = True,
-        include_metadata: bool = True,
-        max_bookmarks: int = 1000,
-    ):
-        """
-        Initialize the Twitter MCP Reader.
-
-        Args:
-            mcp_server_command: Command to start the MCP server (e.g., 'twitter-mcp-server')
-            username: Optional Twitter username to filter bookmarks
-            include_tweet_content: Whether to include full tweet content
-            include_metadata: Whether to include tweet metadata (likes, retweets, etc.)
-            max_bookmarks: Maximum number of bookmarks to fetch
-        """
-        self.mcp_server_command = mcp_server_command
-        self.username = username
-        self.include_tweet_content = include_tweet_content
-        self.include_metadata = include_metadata
-        self.max_bookmarks = max_bookmarks
-        self.mcp_process = None
-
-    async def start_mcp_server(self):
-        """Start the MCP server process."""
-        try:
-            self.mcp_process = await asyncio.create_subprocess_exec(
-                *self.mcp_server_command.split(),
-                stdin=asyncio.subprocess.PIPE,
-                stdout=asyncio.subprocess.PIPE,
-                stderr=asyncio.subprocess.PIPE,
-            )
-            logger.info(f"Started MCP server: {self.mcp_server_command}")
-        except Exception as e:
-            logger.error(f"Failed to start MCP server: {e}")
-            raise
-
-    async def stop_mcp_server(self):
-        """Stop the MCP server process."""
-        if self.mcp_process:
-            self.mcp_process.terminate()
-            await self.mcp_process.wait()
-            logger.info("Stopped MCP server")
-
-    async def send_mcp_request(self, request: dict[str, Any]) -> dict[str, Any]:
-        """Send a request to the MCP server and get response."""
-        if not self.mcp_process:
-            raise RuntimeError("MCP server not started")
-
-        request_json = json.dumps(request) + "\n"
-        self.mcp_process.stdin.write(request_json.encode())
-        await self.mcp_process.stdin.drain()
-
-        response_line = await self.mcp_process.stdout.readline()
-        if not response_line:
-            raise RuntimeError("No response from MCP server")
-
-        return json.loads(response_line.decode().strip())
-
-    async def initialize_mcp_connection(self):
-        """Initialize the MCP connection."""
-        init_request = {
-            "jsonrpc": "2.0",
-            "id": 1,
-            "method": "initialize",
-            "params": {
-                "protocolVersion": "2024-11-05",
-                "capabilities": {},
-                "clientInfo": {"name": "leann-twitter-reader", "version": "1.0.0"},
-            },
-        }
-
-        response = await self.send_mcp_request(init_request)
-        if "error" in response:
-            raise RuntimeError(f"MCP initialization failed: {response['error']}")
-
-        logger.info("MCP connection initialized successfully")
-
-    async def list_available_tools(self) -> list[dict[str, Any]]:
-        """List available tools from the MCP server."""
-        list_request = {"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}}
-
-        response = await self.send_mcp_request(list_request)
-        if "error" in response:
-            raise RuntimeError(f"Failed to list tools: {response['error']}")
-
-        return response.get("result", {}).get("tools", [])
-
-    async def fetch_twitter_bookmarks(self, limit: Optional[int] = None) -> list[dict[str, Any]]:
-        """
-        Fetch Twitter bookmarks using MCP tools.
-
-        Args:
-            limit: Maximum number of bookmarks to fetch
-
-        Returns:
-            List of bookmark dictionaries
-        """
-        tools = await self.list_available_tools()
-        bookmark_tool = None
-
-        # Look for a tool that can fetch bookmarks
-        for tool in tools:
-            tool_name = tool.get("name", "").lower()
-            if any(keyword in tool_name for keyword in ["bookmark", "saved", "favorite"]):
-                bookmark_tool = tool
-                break
-
-        if not bookmark_tool:
-            raise RuntimeError("No bookmark fetching tool found in MCP server")
-
-        # Prepare tool call parameters
-        tool_params = {}
-        if limit or self.max_bookmarks:
-            tool_params["limit"] = limit or self.max_bookmarks
-        if self.username:
-            tool_params["username"] = self.username
-
-        fetch_request = {
-            "jsonrpc": "2.0",
-            "id": 3,
-            "method": "tools/call",
-            "params": {"name": bookmark_tool["name"], "arguments": tool_params},
-        }
-
-        response = await self.send_mcp_request(fetch_request)
-        if "error" in response:
-            raise RuntimeError(f"Failed to fetch bookmarks: {response['error']}")
-
-        # Extract bookmarks from response
-        result = response.get("result", {})
-        if "content" in result and isinstance(result["content"], list):
-            content = result["content"][0] if result["content"] else {}
-            if "text" in content:
-                try:
-                    bookmarks = json.loads(content["text"])
-                except json.JSONDecodeError:
-                    # If not JSON, treat as plain text
-                    bookmarks = [{"text": content["text"], "source": "twitter"}]
-            else:
-                bookmarks = result["content"]
-        else:
-            bookmarks = result.get("bookmarks", result.get("tweets", [result]))
-
-        return bookmarks if isinstance(bookmarks, list) else [bookmarks]
-
-    def _format_bookmark(self, bookmark: dict[str, Any]) -> str:
-        """Format a single bookmark for indexing."""
-        # Extract tweet information
-        text = bookmark.get("text", bookmark.get("content", ""))
-        author = bookmark.get(
-            "author", bookmark.get("username", bookmark.get("user", {}).get("username", "Unknown"))
-        )
-        timestamp = bookmark.get("created_at", bookmark.get("timestamp", ""))
-        url = bookmark.get("url", bookmark.get("tweet_url", ""))
-
-        # Extract metadata if available
-        likes = bookmark.get("likes", bookmark.get("favorite_count", 0))
-        retweets = bookmark.get("retweets", bookmark.get("retweet_count", 0))
-        replies = bookmark.get("replies", bookmark.get("reply_count", 0))
-
-        # Build formatted bookmark
-        parts = []
-
-        # Header
-        parts.append("=== Twitter Bookmark ===")
-
-        if author:
-            parts.append(f"Author: @{author}")
-
-        if timestamp:
-            # Format timestamp if it's a standard format
-            try:
-                import datetime
-
-                if "T" in str(timestamp):  # ISO format
-                    dt = datetime.datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
-                    formatted_time = dt.strftime("%Y-%m-%d %H:%M:%S")
-                else:
-                    formatted_time = str(timestamp)
-                parts.append(f"Date: {formatted_time}")
-            except (ValueError, TypeError):
-                parts.append(f"Date: {timestamp}")
-
-        if url:
-            parts.append(f"URL: {url}")
-
-        # Tweet content
-        if text and self.include_tweet_content:
-            parts.append("")
-            parts.append("Content:")
-            parts.append(text)
-
-        # Metadata
-        if self.include_metadata and any([likes, retweets, replies]):
-            parts.append("")
-            parts.append("Engagement:")
-            if likes:
-                parts.append(f"  Likes: {likes}")
-            if retweets:
-                parts.append(f"  Retweets: {retweets}")
-            if replies:
-                parts.append(f"  Replies: {replies}")
-
-        # Extract hashtags and mentions if available
-        hashtags = bookmark.get("hashtags", [])
-        mentions = bookmark.get("mentions", [])
-
-        if hashtags or mentions:
-            parts.append("")
-            if hashtags:
-                parts.append(f"Hashtags: {', '.join(hashtags)}")
-            if mentions:
-                parts.append(f"Mentions: {', '.join(mentions)}")
-
-        return "\n".join(parts)
-
-    async def read_twitter_bookmarks(self) -> list[str]:
-        """
-        Read Twitter bookmark data and return formatted text chunks.
-
-        Returns:
-            List of formatted text chunks ready for LEANN indexing
-        """
-        try:
-            await self.start_mcp_server()
-            await self.initialize_mcp_connection()
-
-            print(f"Fetching up to {self.max_bookmarks} bookmarks...")
-            if self.username:
-                print(f"Filtering for user: @{self.username}")
-
-            bookmarks = await self.fetch_twitter_bookmarks()
-
-            if not bookmarks:
-                print("No bookmarks found")
-                return []
-
-            print(f"Processing {len(bookmarks)} bookmarks...")
-
-            all_texts = []
-            processed_count = 0
-
-            for bookmark in bookmarks:
-                try:
-                    formatted_bookmark = self._format_bookmark(bookmark)
-                    if formatted_bookmark.strip():
-                        all_texts.append(formatted_bookmark)
-                        processed_count += 1
-                except Exception as e:
-                    logger.warning(f"Failed to format bookmark: {e}")
-                    continue
-
-            print(f"Successfully processed {processed_count} bookmarks")
-            return all_texts
-
-        finally:
-            await self.stop_mcp_server()
-
-    async def __aenter__(self):
-        """Async context manager entry."""
-        await self.start_mcp_server()
-        await self.initialize_mcp_connection()
-        return self
-
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
-        """Async context manager exit."""
-        await self.stop_mcp_server()

apps/twitter_rag.py

@@ -1,195 +0,0 @@
-#!/usr/bin/env python3
-"""
-Twitter RAG Application with MCP Support
-
-This application enables RAG (Retrieval-Augmented Generation) on Twitter bookmarks
-by connecting to Twitter MCP servers to fetch live data and index it in LEANN.
-
-Usage:
-    python -m apps.twitter_rag --mcp-server "twitter-mcp-server" --query "What articles did I bookmark about AI?"
-"""
-
-import argparse
-import asyncio
-
-from apps.base_rag_example import BaseRAGExample
-from apps.twitter_data.twitter_mcp_reader import TwitterMCPReader
-
-
-class TwitterMCPRAG(BaseRAGExample):
-    """
-    RAG application for Twitter bookmarks via MCP servers.
-
-    This class provides a complete RAG pipeline for Twitter bookmark data, including
-    MCP server connection, data fetching, indexing, and interactive chat.
-    """
-
-    def __init__(self):
-        super().__init__(
-            name="Twitter MCP RAG",
-            description="RAG application for Twitter bookmarks via MCP servers",
-            default_index_name="twitter_bookmarks",
-        )
-
-    def _add_specific_arguments(self, parser: argparse.ArgumentParser):
-        """Add Twitter MCP-specific arguments."""
-        parser.add_argument(
-            "--mcp-server",
-            type=str,
-            required=True,
-            help="Command to start the Twitter MCP server (e.g., 'twitter-mcp-server' or 'npx twitter-mcp-server')",
-        )
-
-        parser.add_argument(
-            "--username", type=str, help="Twitter username to filter bookmarks (without @)"
-        )
-
-        parser.add_argument(
-            "--max-bookmarks",
-            type=int,
-            default=1000,
-            help="Maximum number of bookmarks to fetch (default: 1000)",
-        )
-
-        parser.add_argument(
-            "--no-tweet-content",
-            action="store_true",
-            help="Exclude tweet content, only include metadata",
-        )
-
-        parser.add_argument(
-            "--no-metadata",
-            action="store_true",
-            help="Exclude engagement metadata (likes, retweets, etc.)",
-        )
-
-        parser.add_argument(
-            "--test-connection",
-            action="store_true",
-            help="Test MCP server connection and list available tools without indexing",
-        )
-
-    async def test_mcp_connection(self, args) -> bool:
-        """Test the MCP server connection and display available tools."""
-        print(f"Testing connection to MCP server: {args.mcp_server}")
-
-        try:
-            reader = TwitterMCPReader(
-                mcp_server_command=args.mcp_server,
-                username=args.username,
-                include_tweet_content=not args.no_tweet_content,
-                include_metadata=not args.no_metadata,
-                max_bookmarks=args.max_bookmarks,
-            )
-
-            async with reader:
-                tools = await reader.list_available_tools()
-
-                print("\n✅ Successfully connected to MCP server!")
-                print(f"Available tools ({len(tools)}):")
-
-                for i, tool in enumerate(tools, 1):
-                    name = tool.get("name", "Unknown")
-                    description = tool.get("description", "No description available")
-                    print(f"\n{i}. {name}")
-                    print(
-                        f"   Description: {description[:100]}{'...' if len(description) > 100 else ''}"
-                    )
-
-                    # Show input schema if available
-                    schema = tool.get("inputSchema", {})
-                    if schema.get("properties"):
-                        props = list(schema["properties"].keys())[:3]  # Show first 3 properties
-                        print(
-                            f"   Parameters: {', '.join(props)}{'...' if len(schema['properties']) > 3 else ''}"
-                        )
-
-                return True
-
-        except Exception as e:
-            print(f"\n❌ Failed to connect to MCP server: {e}")
-            print("\nTroubleshooting tips:")
-            print("1. Make sure the Twitter MCP server is installed and accessible")
-            print("2. Check if the server command is correct")
-            print("3. Ensure you have proper Twitter API credentials configured")
-            print("4. Verify your Twitter account has bookmarks to fetch")
-            print("5. Try running the MCP server command directly to test it")
-            return False
-
-    async def load_data(self, args) -> list[str]:
-        """Load Twitter bookmarks via MCP server."""
-        print(f"Connecting to Twitter MCP server: {args.mcp_server}")
-
-        if args.username:
-            print(f"Username filter: @{args.username}")
-
-        print(f"Max bookmarks: {args.max_bookmarks}")
-        print(f"Include tweet content: {not args.no_tweet_content}")
-        print(f"Include metadata: {not args.no_metadata}")
-
-        try:
-            reader = TwitterMCPReader(
-                mcp_server_command=args.mcp_server,
-                username=args.username,
-                include_tweet_content=not args.no_tweet_content,
-                include_metadata=not args.no_metadata,
-                max_bookmarks=args.max_bookmarks,
-            )
-
-            texts = await reader.read_twitter_bookmarks()
-
-            if not texts:
-                print("❌ No bookmarks found! This could mean:")
-                print("- You don't have any bookmarks on Twitter")
-                print("- The MCP server couldn't access your bookmarks")
-                print("- Authentication issues with Twitter API")
-                print("- The username filter didn't match any bookmarks")
-                return []
-
-            print(f"✅ Successfully loaded {len(texts)} bookmarks from Twitter")
-
-            # Show sample of what was loaded
-            if texts:
-                sample_text = texts[0][:300] + "..." if len(texts[0]) > 300 else texts[0]
-                print("\nSample bookmark:")
-                print("-" * 50)
-                print(sample_text)
-                print("-" * 50)
-
-            return texts
-
-        except Exception as e:
-            print(f"❌ Error loading Twitter bookmarks: {e}")
-            print("\nThis might be due to:")
-            print("- MCP server connection issues")
-            print("- Twitter API authentication problems")
-            print("- Network connectivity issues")
-            print("- Rate limiting from Twitter API")
-            raise
-
-    async def run(self):
-        """Main entry point with MCP connection testing."""
-        args = self.parser.parse_args()
-
-        # Test connection if requested
-        if args.test_connection:
-            success = await self.test_mcp_connection(args)
-            if not success:
-                return
-            print(
-                "\n🎉 MCP server is working! You can now run without --test-connection to start indexing."
-            )
-            return
-
-        # Run the standard RAG pipeline
-        await super().run()
-
-
-async def main():
-    """Main entry point for the Twitter MCP RAG application."""
-    app = TwitterMCPRAG()
-    await app.run()
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

benchmarks/llm_utils.py

@@ -54,51 +54,29 @@ def extract_thinking_answer(response):
     return response.strip()
 
 
-def load_hf_model(model_name="Qwen/Qwen3-8B", trust_remote_code=False):
-    """Load HuggingFace model
-
-    Args:
-        model_name (str): Name of the model to load
-        trust_remote_code (bool): Whether to allow execution of code from the model repository.
-            Defaults to False for security. Only enable for trusted models.
-    """
+def load_hf_model(model_name="Qwen/Qwen3-8B"):
+    """Load HuggingFace model"""
     if not HF_AVAILABLE:
         raise ImportError("transformers not available")
 
-    if trust_remote_code:
-        print(
-            "⚠️  WARNING: Loading model with trust_remote_code=True. This can execute arbitrary code."
-        )
-
     print(f"Loading HF: {model_name}")
-    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=trust_remote_code)
+    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
     model = AutoModelForCausalLM.from_pretrained(
         model_name,
         torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
         device_map="auto",
-        trust_remote_code=trust_remote_code,
+        trust_remote_code=True,
     )
     return tokenizer, model
 
 
-def load_vllm_model(model_name="Qwen/Qwen3-8B", trust_remote_code=False):
-    """Load vLLM model
-
-    Args:
-        model_name (str): Name of the model to load
-        trust_remote_code (bool): Whether to allow execution of code from the model repository.
-            Defaults to False for security. Only enable for trusted models.
-    """
+def load_vllm_model(model_name="Qwen/Qwen3-8B"):
+    """Load vLLM model"""
     if not VLLM_AVAILABLE:
         raise ImportError("vllm not available")
 
-    if trust_remote_code:
-        print(
-            "⚠️  WARNING: Loading model with trust_remote_code=True. This can execute arbitrary code."
-        )
-
     print(f"Loading vLLM: {model_name}")
-    llm = LLM(model=model_name, trust_remote_code=trust_remote_code)
+    llm = LLM(model=model_name, trust_remote_code=True)
 
     # Qwen3 specific config
     if is_qwen3_model(model_name):
@@ -200,33 +178,19 @@ def evaluate_rag(searcher, llm_func, queries, domain="default", top_k=3, complex
     }
 
 
-def load_qwen_vl_model(model_name="Qwen/Qwen2.5-VL-7B-Instruct", trust_remote_code=False):
-    """Load Qwen2.5-VL multimodal model
-
-    Args:
-        model_name (str): Name of the model to load
-        trust_remote_code (bool): Whether to allow execution of code from the model repository.
-            Defaults to False for security. Only enable for trusted models.
-    """
+def load_qwen_vl_model(model_name="Qwen/Qwen2.5-VL-7B-Instruct"):
+    """Load Qwen2.5-VL multimodal model"""
     if not HF_AVAILABLE:
         raise ImportError("transformers not available")
 
-    if trust_remote_code:
-        print(
-            "⚠️  WARNING: Loading model with trust_remote_code=True. This can execute arbitrary code."
-        )
-
     print(f"Loading Qwen2.5-VL: {model_name}")
 
     try:
         from transformers import AutoModelForVision2Seq, AutoProcessor
 
-        processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=trust_remote_code)
+        processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
         model = AutoModelForVision2Seq.from_pretrained(
-            model_name,
-            torch_dtype=torch.bfloat16,
-            device_map="auto",
-            trust_remote_code=trust_remote_code,
+            model_name, torch_dtype=torch.bfloat16, device_map="auto", trust_remote_code=True
         )
 
         return processor, model
@@ -238,14 +202,9 @@ def load_qwen_vl_model(model_name="Qwen/Qwen2.5-VL-7B-Instruct", trust_remote_co
         try:
             from transformers import AutoProcessor, Qwen2VLForConditionalGeneration
 
-            processor = AutoProcessor.from_pretrained(
-                model_name, trust_remote_code=trust_remote_code
-            )
+            processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
             model = Qwen2VLForConditionalGeneration.from_pretrained(
-                model_name,
-                torch_dtype=torch.bfloat16,
-                device_map="auto",
-                trust_remote_code=trust_remote_code,
+                model_name, torch_dtype=torch.bfloat16, device_map="auto", trust_remote_code=True
             )
 
             return processor, model

docs/configuration-guide.md

@@ -455,5 +455,5 @@ Conclusion:
 
 - [Lessons Learned Developing LEANN](https://yichuan-w.github.io/blog/lessons_learned_in_dev_leann/)
 - [LEANN Technical Paper](https://arxiv.org/abs/2506.08276)
-- [DiskANN Original Paper](https://suhasjs.github.io/files/diskann_neurips19.pdf)
+- [DiskANN Original Paper](https://papers.nips.cc/paper/2019/file/09853c7fb1d3f8ee67a61b6bf4a7f8e6-Paper.pdf)
 - [SSD-based Graph Partitioning](https://github.com/SonglinLife/SSD_BASED_PLAN)

docs/slack-setup-guide.md

@@ -1,395 +0,0 @@
-# Slack Integration Setup Guide
-
-This guide provides step-by-step instructions for setting up Slack integration with LEANN.
-
-## Overview
-
-LEANN's Slack integration uses MCP (Model Context Protocol) servers to fetch and index your Slack messages for RAG (Retrieval-Augmented Generation). This allows you to search through your Slack conversations using natural language queries.
-
-## Prerequisites
-
-1. **Slack Workspace Access**: You need admin or owner permissions in your Slack workspace to create apps and configure OAuth tokens.
-
-2. **Slack MCP Server**: Install a Slack MCP server (e.g., `slack-mcp-server` via npm)
-
-3. **LEANN**: Ensure you have LEANN installed and working
-
-## Step 1: Create a Slack App
-
-### 1.1 Go to Slack API Dashboard
-
-1. Visit [https://api.slack.com/apps](https://api.slack.com/apps)
-2. Click **"Create New App"**
-3. Choose **"From scratch"**
-4. Enter your app name (e.g., "LEANN Slack Integration")
-5. Select your workspace
-6. Click **"Create App"**
-
-### 1.2 Configure App Permissions
-
-#### Token Scopes
-
-1. In your app dashboard, go to **"OAuth & Permissions"** in the left sidebar
-2. Scroll down to **"Scopes"** section
-3. Under **"Bot Token Scopes & OAuth Scope"**, click **"Add an OAuth Scope"**
-4. Add the following scopes:
-   - `channels:read` - Read public channel information
-   - `channels:history` - Read messages in public channels
-   - `groups:read` - Read private channel information
-   - `groups:history` - Read messages in private channels
-   - `im:read` - Read direct message information
-   - `im:history` - Read direct messages
-   - `mpim:read` - Read group direct message information
-   - `mpim:history` - Read group direct messages
-   - `users:read` - Read user information
-   - `team:read` - Read workspace information
-
-#### App-Level Tokens (Optional)
-
-Some MCP servers may require app-level tokens:
-
-1. Go to **"Basic Information"** in the left sidebar
-2. Scroll down to **"App-Level Tokens"**
-3. Click **"Generate Token and Scopes"**
-4. Enter a name (e.g., "LEANN Integration")
-5. Add the `connections:write` scope
-6. Click **"Generate"**
-7. Copy the token (starts with `xapp-`)
-
-### 1.3 Install App to Workspace
-
-1. Go to **"OAuth & Permissions"** in the left sidebar
-2. Click **"Install to Workspace"**
-3. Review the permissions and click **"Allow"**
-4. Copy the **"Bot User OAuth Token"** (starts with `xoxb-`)
-5. Copy the **"User OAuth Token"** (starts with `xoxp-`)
-
-## Step 2: Install Slack MCP Server
-
-### Option A: Using npm (Recommended)
-
-```bash
-# Install globally
-npm install -g slack-mcp-server
-
-# Or install locally
-npm install slack-mcp-server
-```
-
-### Option B: Using npx (No installation required)
-
-```bash
-# Use directly without installation
-npx slack-mcp-server
-```
-
-## Step 3: Install and Configure Ollama (for Real LLM Responses)
-
-### 3.1 Install Ollama
-
-```bash
-# Install Ollama using Homebrew (macOS)
-brew install ollama
-
-# Or download from https://ollama.ai/
-```
-
-### 3.2 Start Ollama Service
-
-```bash
-# Start Ollama as a service
-brew services start ollama
-
-# Or start manually
-ollama serve
-```
-
-### 3.3 Pull a Model
-
-```bash
-# Pull a lightweight model for testing
-ollama pull llama3.2:1b
-
-# Verify the model is available
-ollama list
-```
-
-## Step 4: Configure Environment Variables
-
-Create a `.env` file or set environment variables:
-
-```bash
-# Required: User OAuth Token
-SLACK_OAUTH_TOKEN=xoxp-your-user-oauth-token-here
-
-# Optional: App-Level Token (if your MCP server requires it)
-SLACK_APP_TOKEN=xapp-your-app-token-here
-
-# Optional: Workspace-specific settings
-SLACK_WORKSPACE_ID=T1234567890  # Your workspace ID (optional)
-```
-
-## Step 5: Test the Setup
-
-### 5.1 Test MCP Server Connection
-
-```bash
-python -m apps.slack_rag \
-  --mcp-server "slack-mcp-server" \
-  --test-connection \
-  --workspace-name "Your Workspace Name"
-```
-
-This will test the connection and list available tools without indexing any data.
-
-### 5.2 Index a Specific Channel
-
-```bash
-python -m apps.slack_rag \
-  --mcp-server "slack-mcp-server" \
-  --workspace-name "Your Workspace Name" \
-  --channels general \
-  --query "What did we discuss about the project?"
-```
-
-### 5.3 Real RAG Query Examples
-
-This section demonstrates successful Slack RAG integration queries against the Sky Lab Computing workspace's "random" channel. The system successfully retrieves actual conversation messages and performs semantic search with high relevance scores, including finding specific research paper announcements and technical discussions.
-
-### Example 1: Advisor Models Query
-
-**Query:** "train black-box models to adopt to your personal data"
-
-This query demonstrates the system's ability to find specific research announcements about training black-box models for personal data adaptation.
-
-![Advisor Models Query - Command Setup](videos/slack_integration_1.1.png)
-
-![Advisor Models Query - Search Results](videos/slack_integration_1.2.png)
-
-![Advisor Models Query - LLM Response](videos/slack_integration_1.3.png)
-
-### Example 2: Barbarians at the Gate Query
-
-**Query:** "AI-driven research systems ADRS"
-
-This query demonstrates the system's ability to find specific research announcements about AI-driven research systems and algorithm discovery.
-
-![Barbarians Query - Command Setup](videos/slack_integration_2.1.png)
-
-![Barbarians Query - Search Results](videos/slack_integration_2.2.png)
-
-![Barbarians Query - LLM Response](videos/slack_integration_2.3.png)
-
-### Prerequisites
-
-- Bot is installed in the Sky Lab Computing workspace and invited to the target channel (run `/invite @YourBotName` in the channel if needed)
-- Bot token available and exported in the same terminal session
-
-### Commands
-
-1) Set the workspace token for this shell
-
-```bash
-export SLACK_MCP_XOXP_TOKEN="xoxp-***-redacted-***"
-```
-
-2) Run queries against the "random" channel by channel ID (C0GN5BX0F)
-
-**Advisor Models Query:**
-```bash
-python -m apps.slack_rag \
-  --mcp-server "slack-mcp-server" \
-  --workspace-name "Sky Lab Computing" \
-  --channels C0GN5BX0F \
-  --max-messages-per-channel 100000 \
-  --query "train black-box models to adopt to your personal data" \
-  --llm ollama \
-  --llm-model "llama3.2:1b" \
-  --llm-host "http://localhost:11434" \
-  --no-concatenate-conversations
-```
-
-**Barbarians at the Gate Query:**
-```bash
-python -m apps.slack_rag \
-  --mcp-server "slack-mcp-server" \
-  --workspace-name "Sky Lab Computing" \
-  --channels C0GN5BX0F \
-  --max-messages-per-channel 100000 \
-  --query "AI-driven research systems ADRS" \
-  --llm ollama \
-  --llm-model "llama3.2:1b" \
-  --llm-host "http://localhost:11434" \
-  --no-concatenate-conversations
-```
-
-These examples demonstrate the system's ability to find and retrieve specific research announcements and technical discussions from the conversation history, showcasing the power of semantic search in Slack data.
-
-3) Optional: Ask a broader question
-
-```bash
-python test_channel_by_id_or_name.py \
-  --channel-id C0GN5BX0F \
-  --workspace-name "Sky Lab Computing" \
-  --query "What is LEANN about?"
-```
-
-Notes:
-- If you see `not_in_channel`, invite the bot to the channel and re-run.
-- If you see `channel_not_found`, confirm the channel ID and workspace.
-- Deep search via server-side “search” tools may require additional Slack scopes; the example above performs client-side filtering over retrieved history.
-
-## Common Issues and Solutions
-
-### Issue 1: "users cache is not ready yet" Error
-
-**Problem**: You see this warning:
-```
-WARNING - Failed to fetch messages from channel random: Failed to fetch messages: {'code': -32603, 'message': 'users cache is not ready yet, sync process is still running... please wait'}
-```
-
-**Solution**: This is a common timing issue. The LEANN integration now includes automatic retry logic:
-
-1. **Wait and Retry**: The system will automatically retry with exponential backoff (2s, 4s, 8s, etc.)
-2. **Increase Retry Parameters**: If needed, you can customize retry behavior:
-   ```bash
-   python -m apps.slack_rag \
-     --mcp-server "slack-mcp-server" \
-     --max-retries 10 \
-     --retry-delay 3.0 \
-     --channels general \
-     --query "Your query here"
-   ```
-3. **Keep MCP Server Running**: Start the MCP server separately and keep it running:
-   ```bash
-   # Terminal 1: Start MCP server
-   slack-mcp-server
-
-   # Terminal 2: Run LEANN (it will connect to the running server)
-   python -m apps.slack_rag --mcp-server "slack-mcp-server" --channels general --query "test"
-   ```
-
-### Issue 2: "No message fetching tool found"
-
-**Problem**: The MCP server doesn't have the expected tools.
-
-**Solution**:
-1. Check if your MCP server is properly installed and configured
-2. Verify your Slack tokens are correct
-3. Try a different MCP server implementation
-4. Check the MCP server documentation for required configuration
-
-### Issue 3: Permission Denied Errors
-
-**Problem**: You get permission errors when trying to access channels.
-
-**Solutions**:
-1. **Check Bot Permissions**: Ensure your bot has been added to the channels you want to access
-2. **Verify Token Scopes**: Make sure you have all required scopes configured
-3. **Channel Access**: For private channels, the bot needs to be explicitly invited
-4. **Workspace Permissions**: Ensure your Slack app has the necessary workspace permissions
-
-### Issue 4: Empty Results
-
-**Problem**: No messages are returned even though the channel has messages.
-
-**Solutions**:
-1. **Check Channel Names**: Ensure channel names are correct (without the # symbol)
-2. **Verify Bot Access**: Make sure the bot can access the channels
-3. **Check Date Ranges**: Some MCP servers have limitations on message history
-4. **Increase Message Limits**: Try increasing the message limit:
-   ```bash
-   python -m apps.slack_rag \
-     --mcp-server "slack-mcp-server" \
-     --channels general \
-     --max-messages-per-channel 1000 \
-     --query "test"
-   ```
-
-## Advanced Configuration
-
-### Custom MCP Server Commands
-
-If you need to pass additional parameters to your MCP server:
-
-```bash
-python -m apps.slack_rag \
-  --mcp-server "slack-mcp-server --token-file /path/to/tokens.json" \
-  --workspace-name "Your Workspace" \
-  --channels general \
-  --query "Your query"
-```
-
-### Multiple Workspaces
-
-To work with multiple Slack workspaces, you can:
-
-1. Create separate apps for each workspace
-2. Use different environment variables
-3. Run separate instances with different configurations
-
-### Performance Optimization
-
-For better performance with large workspaces:
-
-```bash
-python -m apps.slack_rag \
-  --mcp-server "slack-mcp-server" \
-  --workspace-name "Your Workspace" \
-  --max-messages-per-channel 500 \
-  --no-concatenate-conversations \
-  --query "Your query"
-```
----
-
-## Troubleshooting Checklist
-
-- [ ] Slack app created with proper permissions
-- [ ] Bot token (xoxb-) copied correctly
-- [ ] App-level token (xapp-) created if needed
-- [ ] MCP server installed and accessible
-- [ ] Ollama installed and running (`brew services start ollama`)
-- [ ] Ollama model pulled (`ollama pull llama3.2:1b`)
-- [ ] Environment variables set correctly
-- [ ] Bot invited to relevant channels
-- [ ] Channel names specified without # symbol
-- [ ] Sufficient retry attempts configured
-- [ ] Network connectivity to Slack APIs
-
-## Getting Help
-
-If you continue to have issues:
-
-1. **Check Logs**: Look for detailed error messages in the console output
-2. **Test MCP Server**: Use `--test-connection` to verify the MCP server is working
-3. **Verify Tokens**: Double-check that your Slack tokens are valid and have the right scopes
-4. **Check Ollama**: Ensure Ollama is running (`ollama serve`) and the model is available (`ollama list`)
-5. **Community Support**: Reach out to the LEANN community for help
-
-## Example Commands
-
-### Basic Usage
-```bash
-# Test connection
-python -m apps.slack_rag --mcp-server "slack-mcp-server" --test-connection
-
-# Index specific channels
-python -m apps.slack_rag \
-  --mcp-server "slack-mcp-server" \
-  --workspace-name "My Company" \
-  --channels general random \
-  --query "What did we decide about the project timeline?"
-```
-
-### Advanced Usage
-```bash
-# With custom retry settings
-python -m apps.slack_rag \
-  --mcp-server "slack-mcp-server" \
-  --workspace-name "My Company" \
-  --channels general \
-  --max-retries 10 \
-  --retry-delay 5.0 \
-  --max-messages-per-channel 2000 \
-  --query "Show me all decisions made in the last month"
-```

examples/dynamic_update_no_recompute.py

@@ -43,11 +43,7 @@ from apps.chunking import create_text_chunks
 REPO_ROOT = Path(__file__).resolve().parents[1]
 
 DEFAULT_QUERY = "What's LEANN?"
-DEFAULT_INITIAL_FILES = [
-    REPO_ROOT / "data" / "2501.14312v1 (1).pdf",
-    REPO_ROOT / "data" / "huawei_pangu.md",
-    REPO_ROOT / "data" / "PrideandPrejudice.txt",
-]
+DEFAULT_INITIAL_FILES = [REPO_ROOT / "data" / "2501.14312v1 (1).pdf"]
 DEFAULT_UPDATE_FILES = [REPO_ROOT / "data" / "2506.08276v1.pdf"]
 
 
@@ -186,7 +182,6 @@ def run_workflow(
     is_recompute: bool,
     query: str,
     top_k: int,
-    skip_search: bool,
 ) -> dict[str, Any]:
     prefix = f"[{label}] " if label else ""
 
@@ -203,15 +198,12 @@ def run_workflow(
     )
 
     initial_size = index_file_size(index_path)
-    if not skip_search:
-        before_results = run_search(
-            index_path,
-            query,
-            top_k,
-            recompute_embeddings=is_recompute,
-        )
-    else:
-        before_results = None
+    before_results = run_search(
+        index_path,
+        query,
+        top_k,
+        recompute_embeddings=is_recompute,
+    )
 
     print(f"\n{prefix}Updating index with additional passages...")
     update_index(
@@ -223,23 +215,20 @@ def run_workflow(
         is_recompute=is_recompute,
     )
 
-    if not skip_search:
-        after_results = run_search(
-            index_path,
-            query,
-            top_k,
-            recompute_embeddings=is_recompute,
-        )
-    else:
-        after_results = None
+    after_results = run_search(
+        index_path,
+        query,
+        top_k,
+        recompute_embeddings=is_recompute,
+    )
     updated_size = index_file_size(index_path)
 
     return {
         "initial_size": initial_size,
         "updated_size": updated_size,
         "delta": updated_size - initial_size,
-        "before_results": before_results if not skip_search else None,
-        "after_results": after_results if not skip_search else None,
+        "before_results": before_results,
+        "after_results": after_results,
         "metadata": load_metadata_snapshot(index_path),
     }
 
@@ -325,12 +314,6 @@ def main() -> None:
         action="store_false",
         help="Skip building the no-recompute baseline.",
     )
-    parser.add_argument(
-        "--skip-search",
-        dest="skip_search",
-        action="store_true",
-        help="Skip the search step.",
-    )
     parser.set_defaults(compare_no_recompute=True)
     args = parser.parse_args()
 
@@ -367,13 +350,10 @@ def main() -> None:
         is_recompute=True,
         query=args.query,
         top_k=args.top_k,
-        skip_search=args.skip_search,
     )
 
-    if not args.skip_search:
-        print_results("initial search", recompute_stats["before_results"])
-    if not args.skip_search:
-        print_results("after update", recompute_stats["after_results"])
+    print_results("initial search", recompute_stats["before_results"])
+    print_results("after update", recompute_stats["after_results"])
     print(
         f"\n[recompute] Index file size change: {recompute_stats['initial_size']} -> {recompute_stats['updated_size']} bytes"
         f" (Δ {recompute_stats['delta']})"
@@ -398,7 +378,6 @@ def main() -> None:
             is_recompute=False,
             query=args.query,
             top_k=args.top_k,
-            skip_search=args.skip_search,
         )
 
         print(
@@ -406,12 +385,8 @@ def main() -> None:
             f" (Δ {baseline_stats['delta']})"
         )
 
-        after_texts = (
-            [res.text for res in recompute_stats["after_results"]] if not args.skip_search else None
-        )
-        baseline_after_texts = (
-            [res.text for res in baseline_stats["after_results"]] if not args.skip_search else None
-        )
+        after_texts = [res.text for res in recompute_stats["after_results"]]
+        baseline_after_texts = [res.text for res in baseline_stats["after_results"]]
         if after_texts == baseline_after_texts:
             print(
                 "[no-recompute] Search results match recompute baseline; see above for the shared output."

examples/mcp_integration_demo.py

@@ -1,178 +0,0 @@
-#!/usr/bin/env python3
-"""
-MCP Integration Examples for LEANN
-
-This script demonstrates how to use LEANN with different MCP servers for
-RAG on various platforms like Slack and Twitter.
-
-Examples:
-1. Slack message RAG via MCP
-2. Twitter bookmark RAG via MCP
-3. Testing MCP server connections
-"""
-
-import asyncio
-import sys
-from pathlib import Path
-
-# Add the parent directory to the path so we can import from apps
-sys.path.append(str(Path(__file__).parent.parent))
-
-
-async def demo_slack_mcp():
-    """Demonstrate Slack MCP integration."""
-    print("=" * 60)
-    print("🔥 Slack MCP RAG Demo")
-    print("=" * 60)
-
-    print("\n1. Testing Slack MCP server connection...")
-
-    # This would typically use a real MCP server command
-    # For demo purposes, we show what the command would look like
-    # slack_app = SlackMCPRAG()  # Would be used for actual testing
-
-    # Simulate command line arguments for testing
-    class MockArgs:
-        mcp_server = "slack-mcp-server"  # This would be the actual MCP server command
-        workspace_name = "my-workspace"
-        channels = ["general", "random", "dev-team"]
-        no_concatenate_conversations = False
-        max_messages_per_channel = 50
-        test_connection = True
-
-    print(f"MCP Server Command: {MockArgs.mcp_server}")
-    print(f"Workspace: {MockArgs.workspace_name}")
-    print(f"Channels: {', '.join(MockArgs.channels)}")
-
-    # In a real scenario, you would run:
-    # success = await slack_app.test_mcp_connection(MockArgs)
-
-    print("\n📝 Example usage:")
-    print("python -m apps.slack_rag \\")
-    print("  --mcp-server 'slack-mcp-server' \\")
-    print("  --workspace-name 'my-team' \\")
-    print("  --channels general dev-team \\")
-    print("  --test-connection")
-
-    print("\n🔍 After indexing, you could query:")
-    print("- 'What did the team discuss about the project deadline?'")
-    print("- 'Find messages about the new feature launch'")
-    print("- 'Show me conversations about budget planning'")
-
-
-async def demo_twitter_mcp():
-    """Demonstrate Twitter MCP integration."""
-    print("\n" + "=" * 60)
-    print("🐦 Twitter MCP RAG Demo")
-    print("=" * 60)
-
-    print("\n1. Testing Twitter MCP server connection...")
-
-    # twitter_app = TwitterMCPRAG()  # Would be used for actual testing
-
-    class MockArgs:
-        mcp_server = "twitter-mcp-server"
-        username = None  # Fetch all bookmarks
-        max_bookmarks = 500
-        no_tweet_content = False
-        no_metadata = False
-        test_connection = True
-
-    print(f"MCP Server Command: {MockArgs.mcp_server}")
-    print(f"Max Bookmarks: {MockArgs.max_bookmarks}")
-    print(f"Include Content: {not MockArgs.no_tweet_content}")
-    print(f"Include Metadata: {not MockArgs.no_metadata}")
-
-    print("\n📝 Example usage:")
-    print("python -m apps.twitter_rag \\")
-    print("  --mcp-server 'twitter-mcp-server' \\")
-    print("  --max-bookmarks 1000 \\")
-    print("  --test-connection")
-
-    print("\n🔍 After indexing, you could query:")
-    print("- 'What AI articles did I bookmark last month?'")
-    print("- 'Find tweets about machine learning techniques'")
-    print("- 'Show me bookmarked threads about startup advice'")
-
-
-async def show_mcp_server_setup():
-    """Show how to set up MCP servers."""
-    print("\n" + "=" * 60)
-    print("⚙️  MCP Server Setup Guide")
-    print("=" * 60)
-
-    print("\n🔧 Setting up Slack MCP Server:")
-    print("1. Install a Slack MCP server (example commands):")
-    print("   npm install -g slack-mcp-server")
-    print("   # OR")
-    print("   pip install slack-mcp-server")
-
-    print("\n2. Configure Slack credentials:")
-    print("   export SLACK_BOT_TOKEN='xoxb-your-bot-token'")
-    print("   export SLACK_APP_TOKEN='xapp-your-app-token'")
-
-    print("\n3. Test the server:")
-    print("   slack-mcp-server --help")
-
-    print("\n🔧 Setting up Twitter MCP Server:")
-    print("1. Install a Twitter MCP server:")
-    print("   npm install -g twitter-mcp-server")
-    print("   # OR")
-    print("   pip install twitter-mcp-server")
-
-    print("\n2. Configure Twitter API credentials:")
-    print("   export TWITTER_API_KEY='your-api-key'")
-    print("   export TWITTER_API_SECRET='your-api-secret'")
-    print("   export TWITTER_ACCESS_TOKEN='your-access-token'")
-    print("   export TWITTER_ACCESS_TOKEN_SECRET='your-access-token-secret'")
-
-    print("\n3. Test the server:")
-    print("   twitter-mcp-server --help")
-
-
-async def show_integration_benefits():
-    """Show the benefits of MCP integration."""
-    print("\n" + "=" * 60)
-    print("🌟 Benefits of MCP Integration")
-    print("=" * 60)
-
-    benefits = [
-        ("🔄 Live Data Access", "Fetch real-time data from platforms without manual exports"),
-        ("🔌 Standardized Protocol", "Use any MCP-compatible server with minimal code changes"),
-        ("🚀 Easy Extension", "Add new platforms by implementing MCP readers"),
-        ("🔒 Secure Access", "MCP servers handle authentication and API management"),
-        ("📊 Rich Metadata", "Access full platform metadata (timestamps, engagement, etc.)"),
-        ("⚡ Efficient Processing", "Stream data directly into LEANN without intermediate files"),
-    ]
-
-    for title, description in benefits:
-        print(f"\n{title}")
-        print(f"   {description}")
-
-
-async def main():
-    """Main demo function."""
-    print("🎯 LEANN MCP Integration Examples")
-    print("This demo shows how to integrate LEANN with MCP servers for various platforms.")
-
-    await demo_slack_mcp()
-    await demo_twitter_mcp()
-    await show_mcp_server_setup()
-    await show_integration_benefits()
-
-    print("\n" + "=" * 60)
-    print("✨ Next Steps")
-    print("=" * 60)
-    print("1. Install and configure MCP servers for your platforms")
-    print("2. Test connections using --test-connection flag")
-    print("3. Run indexing to build your RAG knowledge base")
-    print("4. Start querying your personal data!")
-
-    print("\n📚 For more information:")
-    print("- Check the README for detailed setup instructions")
-    print("- Look at the apps/slack_rag.py and apps/twitter_rag.py for implementation details")
-    print("- Explore other MCP servers for additional platforms")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

packages/leann-core/pyproject.toml

@@ -18,16 +18,14 @@ dependencies = [
     "pyzmq>=23.0.0",
     "msgpack>=1.0.0",
     "torch>=2.0.0",
-    "sentence-transformers>=3.0.0",
+    "sentence-transformers>=2.2.0",
     "llama-index-core>=0.12.0",
     "llama-index-readers-file>=0.4.0",  # Essential for document reading
     "llama-index-embeddings-huggingface>=0.5.5",  # For embeddings
     "python-dotenv>=1.0.0",
     "openai>=1.0.0",
     "huggingface-hub>=0.20.0",
-    # Keep transformers below 4.46: 4.46.0 adds Python 3.10-only return type syntax and
-    # breaks Python 3.9 environments.
-    "transformers>=4.30.0,<4.46",
+    "transformers>=4.30.0",
     "requests>=2.25.0",
     "accelerate>=0.20.0",
     "PyPDF2>=3.0.0",
@@ -42,7 +40,7 @@ dependencies = [
 [project.optional-dependencies]
 colab = [
     "torch>=2.0.0,<3.0.0",  # Limit torch version to avoid conflicts
-    "transformers>=4.30.0,<4.46",  # 4.46.0 switches to PEP 604 typing (int | None), breaks Py3.9
+    "transformers>=4.30.0,<5.0.0",  # Limit transformers version
     "accelerate>=0.20.0,<1.0.0",  # Limit accelerate version
 ]
 

packages/leann-core/src/leann/api.py

@@ -5,7 +5,6 @@ with the correct, original embedding logic from the user's reference code.
 
 import json
 import logging
-import os
 import pickle
 import re
 import subprocess
@@ -18,11 +17,9 @@ from typing import Any, Literal, Optional, Union
 import numpy as np
 from leann_backend_hnsw.convert_to_csr import prune_hnsw_embeddings_inplace
 
-from leann.interactive_utils import create_api_session
 from leann.interface import LeannBackendSearcherInterface
 
 from .chat import get_llm
-from .embedding_server_manager import EmbeddingServerManager
 from .interface import LeannBackendFactoryInterface
 from .metadata_filter import MetadataFilterEngine
 from .registry import BACKEND_REGISTRY
@@ -731,7 +728,6 @@ class LeannBuilder:
         index = faiss.read_index(str(index_file))
         if hasattr(index, "is_recompute"):
             index.is_recompute = needs_recompute
-            print(f"index.is_recompute: {index.is_recompute}")
         if getattr(index, "storage", None) is None:
             if index.metric_type == faiss.METRIC_INNER_PRODUCT:
                 storage_index = faiss.IndexFlatIP(index.d)
@@ -739,112 +735,37 @@ class LeannBuilder:
                 storage_index = faiss.IndexFlatL2(index.d)
             index.storage = storage_index
             index.own_fields = True
-            # Faiss expects storage.ntotal to reflect the existing graph's
-            # population (even if the vectors themselves were pruned from disk
-            # for recompute mode).  When we attach a fresh IndexFlat here its
-            # ntotal starts at zero, which later causes IndexHNSW::add to
-            # believe new "preset" levels were provided and trips the
-            # `n0 + n == levels.size()` assertion.  Seed the temporary storage
-            # with the current ntotal so Faiss maintains the proper offset for
-            # incoming vectors.
-            try:
-                storage_index.ntotal = index.ntotal
-            except AttributeError:
-                # Older Faiss builds may not expose ntotal as a writable
-                # attribute; in that case we fall back to the default behaviour.
-                pass
         if index.d != embedding_dim:
             raise ValueError(
                 f"Existing index dimension ({index.d}) does not match new embeddings ({embedding_dim})."
             )
 
-        passage_meta_mode = meta.get("embedding_mode", self.embedding_mode)
-        passage_provider_options = meta.get("embedding_options", self.embedding_options)
-
         base_id = index.ntotal
         for offset, chunk in enumerate(valid_chunks):
             new_id = str(base_id + offset)
             chunk.setdefault("metadata", {})["id"] = new_id
             chunk["id"] = new_id
 
-        # Append passages/offsets before we attempt index.add so the ZMQ server
-        # can resolve newly assigned IDs during recompute. Keep rollback hooks
-        # so we can restore files if the update fails mid-way.
-        rollback_passages_size = passages_file.stat().st_size if passages_file.exists() else 0
-        offset_map_backup = offset_map.copy()
+        index.add(embeddings.shape[0], faiss.swig_ptr(embeddings))
+        faiss.write_index(index, str(index_file))
 
-        try:
-            with open(passages_file, "a", encoding="utf-8") as f:
-                for chunk in valid_chunks:
-                    offset = f.tell()
-                    json.dump(
-                        {
-                            "id": chunk["id"],
-                            "text": chunk["text"],
-                            "metadata": chunk.get("metadata", {}),
-                        },
-                        f,
-                        ensure_ascii=False,
-                    )
-                    f.write("\n")
-                    offset_map[chunk["id"]] = offset
+        with open(passages_file, "a", encoding="utf-8") as f:
+            for chunk in valid_chunks:
+                offset = f.tell()
+                json.dump(
+                    {
+                        "id": chunk["id"],
+                        "text": chunk["text"],
+                        "metadata": chunk.get("metadata", {}),
+                    },
+                    f,
+                    ensure_ascii=False,
+                )
+                f.write("\n")
+                offset_map[chunk["id"]] = offset
 
-            with open(offset_file, "wb") as f:
-                pickle.dump(offset_map, f)
-
-            server_manager: Optional[EmbeddingServerManager] = None
-            server_started = False
-            requested_zmq_port = int(os.getenv("LEANN_UPDATE_ZMQ_PORT", "5557"))
-
-            try:
-                if needs_recompute:
-                    server_manager = EmbeddingServerManager(
-                        backend_module_name="leann_backend_hnsw.hnsw_embedding_server"
-                    )
-                    server_started, actual_port = server_manager.start_server(
-                        port=requested_zmq_port,
-                        model_name=self.embedding_model,
-                        embedding_mode=passage_meta_mode,
-                        passages_file=str(meta_path),
-                        distance_metric=distance_metric,
-                        provider_options=passage_provider_options,
-                    )
-                    if not server_started:
-                        raise RuntimeError(
-                            "Failed to start HNSW embedding server for recompute update."
-                        )
-                    if actual_port != requested_zmq_port:
-                        logger.warning(
-                            "Embedding server started on port %s instead of requested %s. "
-                            "Using reassigned port.",
-                            actual_port,
-                            requested_zmq_port,
-                        )
-                    try:
-                        index.hnsw.zmq_port = actual_port
-                    except AttributeError:
-                        pass
-
-                if needs_recompute:
-                    for i in range(embeddings.shape[0]):
-                        print(f"add {i} embeddings")
-                        index.add(1, faiss.swig_ptr(embeddings[i : i + 1]))
-                else:
-                    index.add(embeddings.shape[0], faiss.swig_ptr(embeddings))
-                faiss.write_index(index, str(index_file))
-            finally:
-                if server_started and server_manager is not None:
-                    server_manager.stop_server()
-
-        except Exception:
-            # Roll back appended passages/offset map to keep files consistent.
-            if passages_file.exists():
-                with open(passages_file, "rb+") as f:
-                    f.truncate(rollback_passages_size)
-            offset_map = offset_map_backup
-            with open(offset_file, "wb") as f:
-                pickle.dump(offset_map, f)
-            raise
+        with open(offset_file, "wb") as f:
+            pickle.dump(offset_map, f)
 
         meta["total_passages"] = len(offset_map)
         with open(meta_path, "w", encoding="utf-8") as f:
@@ -1243,14 +1164,19 @@ class LeannChat:
         return ans
 
     def start_interactive(self):
-        """Start interactive chat session."""
-        session = create_api_session()
-
-        def handle_query(user_input: str):
-            response = self.ask(user_input)
-            print(f"Leann: {response}")
-
-        session.run_interactive_loop(handle_query)
+        print("\nLeann Chat started (type 'quit' to exit)")
+        while True:
+            try:
+                user_input = input("You: ").strip()
+                if user_input.lower() in ["quit", "exit"]:
+                    break
+                if not user_input:
+                    continue
+                response = self.ask(user_input)
+                print(f"Leann: {response}")
+            except (KeyboardInterrupt, EOFError):
+                print("\nGoodbye!")
+                break
 
     def cleanup(self):
         """Explicitly cleanup embedding server resources.

packages/leann-core/src/leann/chat.py

@@ -12,8 +12,6 @@ from typing import Any, Optional
 
 import torch
 
-from .settings import resolve_ollama_host, resolve_openai_api_key, resolve_openai_base_url
-
 # Configure logging
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -312,12 +310,11 @@ def search_hf_models(query: str, limit: int = 10) -> list[str]:
 
 
 def validate_model_and_suggest(
-    model_name: str, llm_type: str, host: Optional[str] = None
+    model_name: str, llm_type: str, host: str = "http://localhost:11434"
 ) -> Optional[str]:
     """Validate model name and provide suggestions if invalid"""
     if llm_type == "ollama":
-        resolved_host = resolve_ollama_host(host)
-        available_models = check_ollama_models(resolved_host)
+        available_models = check_ollama_models(host)
         if available_models and model_name not in available_models:
             error_msg = f"Model '{model_name}' not found in your local Ollama installation."
 
@@ -460,19 +457,19 @@ class LLMInterface(ABC):
 class OllamaChat(LLMInterface):
     """LLM interface for Ollama models."""
 
-    def __init__(self, model: str = "llama3:8b", host: Optional[str] = None):
+    def __init__(self, model: str = "llama3:8b", host: str = "http://localhost:11434"):
         self.model = model
-        self.host = resolve_ollama_host(host)
-        logger.info(f"Initializing OllamaChat with model='{model}' and host='{self.host}'")
+        self.host = host
+        logger.info(f"Initializing OllamaChat with model='{model}' and host='{host}'")
         try:
             import requests
 
             # Check if the Ollama server is responsive
-            if self.host:
-                requests.get(self.host)
+            if host:
+                requests.get(host)
 
             # Pre-check model availability with helpful suggestions
-            model_error = validate_model_and_suggest(model, "ollama", self.host)
+            model_error = validate_model_and_suggest(model, "ollama", host)
             if model_error:
                 raise ValueError(model_error)
 
@@ -481,11 +478,9 @@ class OllamaChat(LLMInterface):
                 "The 'requests' library is required for Ollama. Please install it with 'pip install requests'."
             )
         except requests.exceptions.ConnectionError:
-            logger.error(
-                f"Could not connect to Ollama at {self.host}. Please ensure Ollama is running."
-            )
+            logger.error(f"Could not connect to Ollama at {host}. Please ensure Ollama is running.")
             raise ConnectionError(
-                f"Could not connect to Ollama at {self.host}. Please ensure Ollama is running."
+                f"Could not connect to Ollama at {host}. Please ensure Ollama is running."
             )
 
     def ask(self, prompt: str, **kwargs) -> str:
@@ -546,30 +541,11 @@ class OllamaChat(LLMInterface):
 
 
 class HFChat(LLMInterface):
-    """LLM interface for local Hugging Face Transformers models with proper chat templates.
+    """LLM interface for local Hugging Face Transformers models with proper chat templates."""
 
-    Args:
-        model_name (str): Name of the Hugging Face model to load.
-        trust_remote_code (bool): Whether to allow execution of code from the model repository.
-            Defaults to False for security. Only enable for trusted models as this can pose
-            a security risk if the model repository is compromised.
-    """
-
-    def __init__(
-        self, model_name: str = "deepseek-ai/deepseek-llm-7b-chat", trust_remote_code: bool = False
-    ):
+    def __init__(self, model_name: str = "deepseek-ai/deepseek-llm-7b-chat"):
         logger.info(f"Initializing HFChat with model='{model_name}'")
 
-        # Security warning when trust_remote_code is enabled
-        if trust_remote_code:
-            logger.warning(
-                "SECURITY WARNING: trust_remote_code=True allows execution of arbitrary code from the model repository. "
-                "Only enable this for models from trusted sources. This creates a potential security risk if the model "
-                "repository is compromised."
-            )
-
-        self.trust_remote_code = trust_remote_code
-
         # Pre-check model availability with helpful suggestions
         model_error = validate_model_and_suggest(model_name, "hf")
         if model_error:
@@ -601,22 +577,35 @@ class HFChat(LLMInterface):
             def timeout_handler(signum, frame):
                 raise TimeoutError("Model download/loading timed out")
 
-            # Set timeout for model loading (60 seconds)
+            # Set timeout for model loading (increase to 300s for large models)
             old_handler = signal.signal(signal.SIGALRM, timeout_handler)
-            signal.alarm(60)
+            signal.alarm(300)
 
             try:
                 logger.info(f"Loading tokenizer for {model_name}...")
-                self.tokenizer = AutoTokenizer.from_pretrained(
-                    model_name, trust_remote_code=self.trust_remote_code
-                )
+                self.tokenizer = AutoTokenizer.from_pretrained(model_name)
 
                 logger.info(f"Loading model {model_name}...")
+                # Choose a numerically stable dtype per device
+                if self.device == "cuda":
+                    # Prefer bfloat16 when available; otherwise fall back to float16
+                    try:
+                        bf16_ok = torch.cuda.is_bf16_supported()
+                    except Exception:
+                        bf16_ok = False
+                    load_dtype = torch.bfloat16 if bf16_ok else torch.float16
+                elif self.device == "mps":
+                    # On Apple MPS, float16 often causes NaNs/INFs during sampling.
+                    # Use float32 for stability, even if it increases memory.
+                    load_dtype = torch.float32
+                else:
+                    load_dtype = torch.float32
+
                 self.model = AutoModelForCausalLM.from_pretrained(
                     model_name,
-                    torch_dtype=torch.float16 if self.device != "cpu" else torch.float32,
+                    torch_dtype=load_dtype,
                     device_map="auto" if self.device != "cpu" else None,
-                    trust_remote_code=self.trust_remote_code,
+                    trust_remote_code=True,
                 )
                 logger.info(f"Successfully loaded {model_name}")
             finally:
@@ -632,8 +621,12 @@ class HFChat(LLMInterface):
             logger.error(f"Failed to load model {model_name}: {e}")
             raise
 
-        # Move model to device if not using device_map
-        if self.device != "cpu" and "device_map" not in str(self.model):
+        # Move model to device only if not managed by accelerate (no device_map)
+        try:
+            has_device_map = getattr(self.model, "hf_device_map", None) is not None
+        except Exception:
+            has_device_map = False
+        if self.device != "cpu" and not has_device_map:
             self.model = self.model.to(self.device)
 
         # Set pad token if not present
@@ -665,13 +658,15 @@ class HFChat(LLMInterface):
             # Fallback for models without chat template
             formatted_prompt = prompt
 
-        # Tokenize input
+        # Tokenize input (respect model context length when available)
         inputs = self.tokenizer(
             formatted_prompt,
             return_tensors="pt",
             padding=True,
             truncation=True,
-            max_length=2048,
+            max_length=getattr(
+                getattr(self.model, "config", None), "max_position_embeddings", 2048
+            ),
         )
 
         # Move inputs to device
@@ -686,6 +681,8 @@ class HFChat(LLMInterface):
             "do_sample": kwargs.get("temperature", 0.7) > 0,
             "pad_token_id": self.tokenizer.eos_token_id,
             "eos_token_id": self.tokenizer.eos_token_id,
+            # Helps avoid numerical issues in sampling when logits processors are used
+            "renormalize_logits": True,
         }
 
         # Handle temperature=0 for greedy decoding
@@ -695,11 +692,39 @@ class HFChat(LLMInterface):
 
         logger.info(f"Generating with HuggingFace model, config: {generation_config}")
 
-        # Generate
+        # Streaming support (optional)
+        stream = bool(kwargs.get("stream", False))
+        if stream:
+            try:
+                from threading import Thread
+
+                from transformers import TextIteratorStreamer
+
+                streamer = TextIteratorStreamer(
+                    self.tokenizer, skip_prompt=True, skip_special_tokens=True
+                )
+
+                def _gen():
+                    with torch.no_grad():
+                        self.model.generate(**inputs, **generation_config, streamer=streamer)
+
+                t = Thread(target=_gen)
+                t.start()
+
+                pieces = []
+                for new_text in streamer:
+                    print(new_text, end="", flush=True)
+                    pieces.append(new_text)
+                t.join()
+                print("")  # newline after streaming
+                return ("".join(pieces)).strip()
+            except Exception as e:
+                logger.warning(f"Streaming failed, falling back to non-streaming: {e}")
+
+        # Non-streaming path
         with torch.no_grad():
             outputs = self.model.generate(**inputs, **generation_config)
 
-        # Decode response
         generated_tokens = outputs[0][inputs["input_ids"].shape[1] :]
         response = self.tokenizer.decode(generated_tokens, skip_special_tokens=True)
 
@@ -763,31 +788,21 @@ class GeminiChat(LLMInterface):
 class OpenAIChat(LLMInterface):
     """LLM interface for OpenAI models."""
 
-    def __init__(
-        self,
-        model: str = "gpt-4o",
-        api_key: Optional[str] = None,
-        base_url: Optional[str] = None,
-    ):
+    def __init__(self, model: str = "gpt-4o", api_key: Optional[str] = None):
         self.model = model
-        self.base_url = resolve_openai_base_url(base_url)
-        self.api_key = resolve_openai_api_key(api_key)
+        self.api_key = api_key or os.getenv("OPENAI_API_KEY")
 
         if not self.api_key:
             raise ValueError(
                 "OpenAI API key is required. Set OPENAI_API_KEY environment variable or pass api_key parameter."
             )
 
-        logger.info(
-            "Initializing OpenAI Chat with model='%s' and base_url='%s'",
-            model,
-            self.base_url,
-        )
+        logger.info(f"Initializing OpenAI Chat with model='{model}'")
 
         try:
             import openai
 
-            self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url)
+            self.client = openai.OpenAI(api_key=self.api_key)
         except ImportError:
             raise ImportError(
                 "The 'openai' library is required for OpenAI models. Please install it with 'pip install openai'."
@@ -877,19 +892,12 @@ def get_llm(llm_config: Optional[dict[str, Any]] = None) -> LLMInterface:
     if llm_type == "ollama":
         return OllamaChat(
             model=model or "llama3:8b",
-            host=llm_config.get("host"),
+            host=llm_config.get("host", "http://localhost:11434"),
         )
     elif llm_type == "hf":
-        return HFChat(
-            model_name=model or "deepseek-ai/deepseek-llm-7b-chat",
-            trust_remote_code=llm_config.get("trust_remote_code", False),
-        )
+        return HFChat(model_name=model or "deepseek-ai/deepseek-llm-7b-chat")
     elif llm_type == "openai":
-        return OpenAIChat(
-            model=model or "gpt-4o",
-            api_key=llm_config.get("api_key"),
-            base_url=llm_config.get("base_url"),
-        )
+        return OpenAIChat(model=model or "gpt-4o", api_key=llm_config.get("api_key"))
     elif llm_type == "gemini":
         return GeminiChat(model=model or "gemini-2.5-flash", api_key=llm_config.get("api_key"))
     elif llm_type == "simulated":

packages/leann-core/src/leann/cli.py

@@ -8,7 +8,6 @@ from llama_index.core.node_parser import SentenceSplitter
 from tqdm import tqdm
 
 from .api import LeannBuilder, LeannChat, LeannSearcher
-from .interactive_utils import create_cli_session
 from .registry import register_project_directory
 from .settings import resolve_ollama_host, resolve_openai_api_key, resolve_openai_base_url
 
@@ -1557,13 +1556,22 @@ Examples:
         initial_query = (args.query or "").strip()
 
         if args.interactive:
-            # Create interactive session
-            session = create_cli_session(index_name)
-
             if initial_query:
                 _ask_once(initial_query)
 
-            session.run_interactive_loop(_ask_once)
+            print("LEANN Assistant ready! Type 'quit' to exit")
+            print("=" * 40)
+
+            while True:
+                user_input = input("\nYou: ").strip()
+                if user_input.lower() in ["quit", "exit", "q"]:
+                    print("Goodbye!")
+                    break
+
+                if not user_input:
+                    continue
+
+                _ask_once(user_input)
         else:
             query = initial_query or input("Enter your question: ").strip()
             if not query:

packages/leann-core/src/leann/embedding_compute.py

@@ -183,73 +183,32 @@ def compute_embeddings_sentence_transformers(
         }
 
         try:
-            # Try loading with advanced parameters first (newer versions)
-            local_model_kwargs = model_kwargs.copy()
-            local_tokenizer_kwargs = tokenizer_kwargs.copy()
-            local_model_kwargs["local_files_only"] = True
-            local_tokenizer_kwargs["local_files_only"] = True
+            # Try local loading first
+            model_kwargs["local_files_only"] = True
+            tokenizer_kwargs["local_files_only"] = True
 
             model = SentenceTransformer(
                 model_name,
                 device=device,
-                model_kwargs=local_model_kwargs,
-                tokenizer_kwargs=local_tokenizer_kwargs,
+                model_kwargs=model_kwargs,
+                tokenizer_kwargs=tokenizer_kwargs,
                 local_files_only=True,
             )
             logger.info("Model loaded successfully! (local + optimized)")
-        except TypeError as e:
-            if "model_kwargs" in str(e) or "tokenizer_kwargs" in str(e):
-                logger.warning(
-                    f"Advanced parameters not supported ({e}), using basic initialization..."
-                )
-                # Fallback to basic initialization for older versions
-                try:
-                    model = SentenceTransformer(
-                        model_name,
-                        device=device,
-                        local_files_only=True,
-                    )
-                    logger.info("Model loaded successfully! (local + basic)")
-                except Exception as e2:
-                    logger.warning(f"Local loading failed ({e2}), trying network download...")
-                    model = SentenceTransformer(
-                        model_name,
-                        device=device,
-                        local_files_only=False,
-                    )
-                    logger.info("Model loaded successfully! (network + basic)")
-            else:
-                raise
         except Exception as e:
             logger.warning(f"Local loading failed ({e}), trying network download...")
-            # Fallback to network loading with advanced parameters
-            try:
-                network_model_kwargs = model_kwargs.copy()
-                network_tokenizer_kwargs = tokenizer_kwargs.copy()
-                network_model_kwargs["local_files_only"] = False
-                network_tokenizer_kwargs["local_files_only"] = False
+            # Fallback to network loading
+            model_kwargs["local_files_only"] = False
+            tokenizer_kwargs["local_files_only"] = False
 
-                model = SentenceTransformer(
-                    model_name,
-                    device=device,
-                    model_kwargs=network_model_kwargs,
-                    tokenizer_kwargs=network_tokenizer_kwargs,
-                    local_files_only=False,
-                )
-                logger.info("Model loaded successfully! (network + optimized)")
-            except TypeError as e2:
-                if "model_kwargs" in str(e2) or "tokenizer_kwargs" in str(e2):
-                    logger.warning(
-                        f"Advanced parameters not supported ({e2}), using basic network loading..."
-                    )
-                    model = SentenceTransformer(
-                        model_name,
-                        device=device,
-                        local_files_only=False,
-                    )
-                    logger.info("Model loaded successfully! (network + basic)")
-                else:
-                    raise
+            model = SentenceTransformer(
+                model_name,
+                device=device,
+                model_kwargs=model_kwargs,
+                tokenizer_kwargs=tokenizer_kwargs,
+                local_files_only=False,
+            )
+            logger.info("Model loaded successfully! (network + optimized)")
 
         # Apply additional optimizations based on mode
         if use_fp16 and device in ["cuda", "mps"]:

packages/leann-core/src/leann/embedding_server_manager.py

@@ -1,5 +1,4 @@
 import atexit
-import json
 import logging
 import os
 import socket
@@ -49,85 +48,6 @@ def _check_port(port: int) -> bool:
 # Note: All cross-process scanning helpers removed for simplicity
 
 
-def _safe_resolve(path: Path) -> str:
-    """Resolve paths safely even if the target does not yet exist."""
-    try:
-        return str(path.resolve(strict=False))
-    except Exception:
-        return str(path)
-
-
-def _safe_stat_signature(path: Path) -> dict:
-    """Return a lightweight signature describing the current state of a path."""
-    signature: dict[str, object] = {"path": _safe_resolve(path)}
-    try:
-        stat = path.stat()
-    except FileNotFoundError:
-        signature["missing"] = True
-    except Exception as exc:  # pragma: no cover - unexpected filesystem errors
-        signature["error"] = str(exc)
-    else:
-        signature["mtime_ns"] = stat.st_mtime_ns
-        signature["size"] = stat.st_size
-    return signature
-
-
-def _build_passages_signature(passages_file: Optional[str]) -> Optional[dict]:
-    """Collect modification signatures for metadata and referenced passage files."""
-    if not passages_file:
-        return None
-
-    meta_path = Path(passages_file)
-    signature: dict[str, object] = {"meta": _safe_stat_signature(meta_path)}
-
-    try:
-        with meta_path.open(encoding="utf-8") as fh:
-            meta = json.load(fh)
-    except FileNotFoundError:
-        signature["meta_missing"] = True
-        signature["sources"] = []
-        return signature
-    except json.JSONDecodeError as exc:
-        signature["meta_error"] = f"json_error:{exc}"
-        signature["sources"] = []
-        return signature
-    except Exception as exc:  # pragma: no cover - unexpected errors
-        signature["meta_error"] = str(exc)
-        signature["sources"] = []
-        return signature
-
-    base_dir = meta_path.parent
-    seen_paths: set[str] = set()
-    source_signatures: list[dict[str, object]] = []
-
-    for source in meta.get("passage_sources", []):
-        for key, kind in (
-            ("path", "passages"),
-            ("path_relative", "passages"),
-            ("index_path", "index"),
-            ("index_path_relative", "index"),
-        ):
-            raw_path = source.get(key)
-            if not raw_path:
-                continue
-            candidate = Path(raw_path)
-            if not candidate.is_absolute():
-                candidate = base_dir / candidate
-            resolved = _safe_resolve(candidate)
-            if resolved in seen_paths:
-                continue
-            seen_paths.add(resolved)
-            sig = _safe_stat_signature(candidate)
-            sig["kind"] = kind
-            source_signatures.append(sig)
-
-    signature["sources"] = source_signatures
-    return signature
-
-
-# Note: All cross-process scanning helpers removed for simplicity
-
-
 class EmbeddingServerManager:
     """
     A simplified manager for embedding server processes that avoids complex update mechanisms.
@@ -165,14 +85,13 @@ class EmbeddingServerManager:
         """Start the embedding server."""
         # passages_file may be present in kwargs for server CLI, but we don't need it here
         provider_options = kwargs.pop("provider_options", None)
-        passages_file = kwargs.get("passages_file", "")
 
-        config_signature = self._build_config_signature(
-            model_name=model_name,
-            embedding_mode=embedding_mode,
-            provider_options=provider_options,
-            passages_file=passages_file,
-        )
+        config_signature = {
+            "model_name": model_name,
+            "passages_file": kwargs.get("passages_file", ""),
+            "embedding_mode": embedding_mode,
+            "provider_options": provider_options or {},
+        }
 
         # If this manager already has a live server, just reuse it
         if (
@@ -196,7 +115,6 @@ class EmbeddingServerManager:
                 port,
                 model_name,
                 embedding_mode,
-                config_signature=config_signature,
                 provider_options=provider_options,
                 **kwargs,
             )
@@ -218,30 +136,11 @@ class EmbeddingServerManager:
             **kwargs,
         )
 
-    def _build_config_signature(
-        self,
-        *,
-        model_name: str,
-        embedding_mode: str,
-        provider_options: Optional[dict],
-        passages_file: Optional[str],
-    ) -> dict:
-        """Create a signature describing the current server configuration."""
-        return {
-            "model_name": model_name,
-            "passages_file": passages_file or "",
-            "embedding_mode": embedding_mode,
-            "provider_options": provider_options or {},
-            "passages_signature": _build_passages_signature(passages_file),
-        }
-
     def _start_server_colab(
         self,
         port: int,
         model_name: str,
         embedding_mode: str = "sentence-transformers",
-        *,
-        config_signature: Optional[dict] = None,
         provider_options: Optional[dict] = None,
         **kwargs,
     ) -> tuple[bool, int]:
@@ -264,11 +163,10 @@ class EmbeddingServerManager:
                 command,
                 actual_port,
                 provider_options=provider_options,
-                config_signature=config_signature,
             )
             started, ready_port = self._wait_for_server_ready_colab(actual_port)
             if started:
-                self._server_config = config_signature or {
+                self._server_config = {
                     "model_name": model_name,
                     "passages_file": kwargs.get("passages_file", ""),
                     "embedding_mode": embedding_mode,
@@ -300,7 +198,6 @@ class EmbeddingServerManager:
                 command,
                 port,
                 provider_options=provider_options,
-                config_signature=config_signature,
             )
             started, ready_port = self._wait_for_server_ready(port)
             if started:
@@ -344,9 +241,7 @@ class EmbeddingServerManager:
         self,
         command: list,
         port: int,
-        *,
         provider_options: Optional[dict] = None,
-        config_signature: Optional[dict] = None,
     ) -> None:
         """Launch the server process."""
         project_root = Path(__file__).parent.parent.parent.parent.parent
@@ -381,29 +276,26 @@ class EmbeddingServerManager:
         )
         self.server_port = port
         # Record config for in-process reuse (best effort; refined later when ready)
-        if config_signature is not None:
-            self._server_config = config_signature
-        else:  # Fallback for unexpected code paths
-            try:
-                self._server_config = {
-                    "model_name": command[command.index("--model-name") + 1]
-                    if "--model-name" in command
-                    else "",
-                    "passages_file": command[command.index("--passages-file") + 1]
-                    if "--passages-file" in command
-                    else "",
-                    "embedding_mode": command[command.index("--embedding-mode") + 1]
-                    if "--embedding-mode" in command
-                    else "sentence-transformers",
-                    "provider_options": provider_options or {},
-                }
-            except Exception:
-                self._server_config = {
-                    "model_name": "",
-                    "passages_file": "",
-                    "embedding_mode": "sentence-transformers",
-                    "provider_options": provider_options or {},
-                }
+        try:
+            self._server_config = {
+                "model_name": command[command.index("--model-name") + 1]
+                if "--model-name" in command
+                else "",
+                "passages_file": command[command.index("--passages-file") + 1]
+                if "--passages-file" in command
+                else "",
+                "embedding_mode": command[command.index("--embedding-mode") + 1]
+                if "--embedding-mode" in command
+                else "sentence-transformers",
+                "provider_options": provider_options or {},
+            }
+        except Exception:
+            self._server_config = {
+                "model_name": "",
+                "passages_file": "",
+                "embedding_mode": "sentence-transformers",
+                "provider_options": provider_options or {},
+            }
         logger.info(f"Server process started with PID: {self.server_process.pid}")
 
         # Register atexit callback only when we actually start a process
@@ -511,9 +403,7 @@ class EmbeddingServerManager:
         self,
         command: list,
         port: int,
-        *,
         provider_options: Optional[dict] = None,
-        config_signature: Optional[dict] = None,
     ) -> None:
         """Launch the server process with Colab-specific settings."""
         logger.info(f"Colab Command: {' '.join(command)}")
@@ -539,15 +429,12 @@ class EmbeddingServerManager:
             atexit.register(self._finalize_process)
             self._atexit_registered = True
         # Record config for in-process reuse is best-effort in Colab mode
-        if config_signature is not None:
-            self._server_config = config_signature
-        else:
-            self._server_config = {
-                "model_name": "",
-                "passages_file": "",
-                "embedding_mode": "sentence-transformers",
-                "provider_options": provider_options or {},
-            }
+        self._server_config = {
+            "model_name": "",
+            "passages_file": "",
+            "embedding_mode": "sentence-transformers",
+            "provider_options": provider_options or {},
+        }
 
     def _wait_for_server_ready_colab(self, port: int) -> tuple[bool, int]:
         """Wait for the server to be ready with Colab-specific timeout."""

packages/leann-core/src/leann/interactive_utils.py

@@ -1,189 +0,0 @@
-"""
-Interactive session utilities for LEANN applications.
-
-Provides shared readline functionality and command handling across
-CLI, API, and RAG example interactive modes.
-"""
-
-import atexit
-import os
-from pathlib import Path
-from typing import Callable, Optional
-
-# Try to import readline with fallback for Windows
-try:
-    import readline
-
-    HAS_READLINE = True
-except ImportError:
-    # Windows doesn't have readline by default
-    HAS_READLINE = False
-    readline = None
-
-
-class InteractiveSession:
-    """Manages interactive session with optional readline support and common commands."""
-
-    def __init__(
-        self,
-        history_name: str,
-        prompt: str = "You: ",
-        welcome_message: str = "",
-    ):
-        """
-        Initialize interactive session with optional readline support.
-
-        Args:
-            history_name: Name for history file (e.g., "cli", "api_chat")
-                         (ignored if readline not available)
-            prompt: Input prompt to display
-            welcome_message: Message to show when starting session
-
-        Note:
-            On systems without readline (e.g., Windows), falls back to basic input()
-            with limited functionality (no history, no line editing).
-        """
-        self.history_name = history_name
-        self.prompt = prompt
-        self.welcome_message = welcome_message
-        self._setup_complete = False
-
-    def setup_readline(self):
-        """Setup readline with history support (if available)."""
-        if self._setup_complete:
-            return
-
-        if not HAS_READLINE:
-            # Readline not available (likely Windows), skip setup
-            self._setup_complete = True
-            return
-
-        # History file setup
-        history_dir = Path.home() / ".leann" / "history"
-        history_dir.mkdir(parents=True, exist_ok=True)
-        history_file = history_dir / f"{self.history_name}.history"
-
-        # Load history if exists
-        try:
-            readline.read_history_file(str(history_file))
-            readline.set_history_length(1000)
-        except FileNotFoundError:
-            pass
-
-        # Save history on exit
-        atexit.register(readline.write_history_file, str(history_file))
-
-        # Optional: Enable vi editing mode (commented out by default)
-        # readline.parse_and_bind("set editing-mode vi")
-
-        self._setup_complete = True
-
-    def _show_help(self):
-        """Show available commands."""
-        print("Commands:")
-        print("  quit/exit/q - Exit the chat")
-        print("  help - Show this help message")
-        print("  clear - Clear screen")
-        print("  history - Show command history")
-
-    def _show_history(self):
-        """Show command history."""
-        if not HAS_READLINE:
-            print("  History not available (readline not supported on this system)")
-            return
-
-        history_length = readline.get_current_history_length()
-        if history_length == 0:
-            print("  No history available")
-            return
-
-        for i in range(history_length):
-            item = readline.get_history_item(i + 1)
-            if item:
-                print(f"  {i + 1}: {item}")
-
-    def get_user_input(self) -> Optional[str]:
-        """
-        Get user input with readline support.
-
-        Returns:
-            User input string, or None if EOF (Ctrl+D)
-        """
-        try:
-            return input(self.prompt).strip()
-        except KeyboardInterrupt:
-            print("\n(Use 'quit' to exit)")
-            return ""  # Return empty string to continue
-        except EOFError:
-            print("\nGoodbye!")
-            return None
-
-    def run_interactive_loop(self, handler_func: Callable[[str], None]):
-        """
-        Run the interactive loop with a custom handler function.
-
-        Args:
-            handler_func: Function to handle user input that's not a built-in command
-                         Should accept a string and handle the user's query
-        """
-        self.setup_readline()
-
-        if self.welcome_message:
-            print(self.welcome_message)
-
-        while True:
-            user_input = self.get_user_input()
-
-            if user_input is None:  # EOF (Ctrl+D)
-                break
-
-            if not user_input:  # Empty input or KeyboardInterrupt
-                continue
-
-            # Handle built-in commands
-            command = user_input.lower()
-            if command in ["quit", "exit", "q"]:
-                print("Goodbye!")
-                break
-            elif command == "help":
-                self._show_help()
-            elif command == "clear":
-                os.system("clear" if os.name != "nt" else "cls")
-            elif command == "history":
-                self._show_history()
-            else:
-                # Regular user input - pass to handler
-                try:
-                    handler_func(user_input)
-                except Exception as e:
-                    print(f"Error: {e}")
-
-
-def create_cli_session(index_name: str) -> InteractiveSession:
-    """Create an interactive session for CLI usage."""
-    return InteractiveSession(
-        history_name=index_name,
-        prompt="\nYou: ",
-        welcome_message="LEANN Assistant ready! Type 'quit' to exit, 'help' for commands\n"
-        + "=" * 40,
-    )
-
-
-def create_api_session() -> InteractiveSession:
-    """Create an interactive session for API chat."""
-    return InteractiveSession(
-        history_name="api_chat",
-        prompt="You: ",
-        welcome_message="Leann Chat started (type 'quit' to exit, 'help' for commands)\n"
-        + "=" * 40,
-    )
-
-
-def create_rag_session(app_name: str, data_description: str) -> InteractiveSession:
-    """Create an interactive session for RAG examples."""
-    return InteractiveSession(
-        history_name=f"{app_name}_rag",
-        prompt="You: ",
-        welcome_message=f"[Interactive Mode] Chat with your {data_description} data!\nType 'quit' or 'exit' to stop, 'help' for commands.\n"
-        + "=" * 40,
-    )

pyproject.toml

@@ -22,10 +22,7 @@ dependencies = [
     "sglang",
     "ollama",
     "requests>=2.25.0",
-    "sentence-transformers>=3.0.0",
-    # Pin transformers below 4.46: 4.46.0 introduced Python 3.10-only typing (PEP 604) and
-    # breaks our Python 3.9 test matrix when pulled in by sentence-transformers.
-    "transformers<4.46",
+    "sentence-transformers>=2.2.0",
     "openai>=1.0.0",
     # PDF parsing dependencies - essential for document processing
     "PyPDF2>=3.0.0",
@@ -114,7 +111,7 @@ target-version = "py39"
 line-length = 100
 extend-exclude = [
     "third_party",
-    "apps/multimodal/vision-based-pdf-multi-vector/multi-vector-leann-paper-example.py",
+    "apps/multimodal/vision-based-pdf-multi-vector/multi-vector-leann.py",
     "apps/multimodal/vision-based-pdf-multi-vector/multi-vector-leann-similarity-map.py"
 ]
 

scripts/measure_generation_times.py

@@ -0,0 +1,324 @@
+#!/usr/bin/env python3
+"""Measure generation latency of a HuggingFace/OpenAI-compatible model over prompt files."""
+
+import argparse
+import contextlib
+import io
+import json
+import logging
+import time
+from pathlib import Path
+
+from leann.chat import get_llm
+
+PROMPT_PREFIX = "PROMPT #"
+logging.getLogger("leann.chat").setLevel(logging.ERROR)
+
+
+def load_prompts(path: Path) -> list[str]:
+    prompts: list[str] = []
+    buffer: list[str] = []
+    collecting = False
+
+    with path.open("r", encoding="utf-8") as handle:
+        for line in handle:
+            if line.startswith(PROMPT_PREFIX):
+                if buffer:
+                    prompts.append("".join(buffer).strip())
+                    buffer.clear()
+                collecting = True
+                continue
+
+            if collecting:
+                buffer.append(line)
+
+    if buffer:
+        prompts.append("".join(buffer).strip())
+
+    return prompts
+
+
+def measure_generation_times(
+    prompts: list[str],
+    llm,
+    generation_kwargs: dict[str, object],
+    allow_truncation: bool,
+    enable_qwen_thinking: bool,
+    verbose: bool,
+    per_call_timeout: int | None,
+):
+    timings: list[float] = []
+    tokenizer = getattr(llm, "tokenizer", None)
+    max_positions = None
+    if hasattr(llm, "model") and hasattr(llm.model, "config"):
+        max_positions = getattr(llm.model.config, "max_position_embeddings", None)
+
+    requested_new_tokens = None
+    if max_positions is not None:
+        if "max_new_tokens" in generation_kwargs:
+            requested_new_tokens = generation_kwargs["max_new_tokens"]
+        elif "max_tokens" in generation_kwargs:
+            requested_new_tokens = generation_kwargs["max_tokens"]
+
+    context_max_length = max_positions
+    if max_positions is not None and requested_new_tokens is not None:
+        if requested_new_tokens >= max_positions:
+            requested_new_tokens = max_positions - 1
+        context_max_length = max(max_positions - requested_new_tokens, 1)
+
+    suppress_buffer = io.StringIO()
+    # Log base config
+    if verbose:
+        device = getattr(llm, "device", None)
+        try:
+            dtype = getattr(getattr(llm, "model", None), "dtype", None)
+        except Exception:
+            dtype = None
+        print(
+            f"[dbg] device={device} dtype={dtype} max_positions={max_positions} requested_new_tokens={requested_new_tokens} context_max_length={context_max_length}"
+        )
+    total = len(prompts)
+    for idx, prompt in enumerate(prompts, start=1):
+        prompt_for_llm = prompt
+        if (
+            enable_qwen_thinking
+            and "/think" not in prompt_for_llm
+            and "/no_think" not in prompt_for_llm
+        ):
+            prompt_for_llm = f"{prompt_for_llm}\n/think"
+
+        if allow_truncation and tokenizer is not None and max_positions is not None:
+            tokenized = tokenizer(
+                prompt_for_llm,
+                truncation=True,
+                max_length=context_max_length,
+                return_tensors="pt",
+            )
+            prompt_for_llm = tokenizer.decode(tokenized["input_ids"][0], skip_special_tokens=True)
+
+        per_call_kwargs = dict(generation_kwargs)
+        if requested_new_tokens is not None:
+            per_call_kwargs["max_new_tokens"] = requested_new_tokens
+        # Enable streaming if requested (HF backend will print tokens)
+        if verbose:
+            # When verbose (or --stream propagated), enable streaming in HF backend
+            per_call_kwargs["stream"] = True
+
+        # Extra debug info about token lengths
+        if verbose and tokenizer is not None:
+            try:
+                toks = tokenizer(prompt_for_llm, return_tensors=None, truncation=False)
+                in_len = (
+                    len(toks["input_ids"])
+                    if isinstance(toks["input_ids"], list)
+                    else len(toks["input_ids"][0])
+                )
+            except Exception:
+                in_len = None
+            print(f"[dbg] prompt {idx}/{total} tokens={in_len}")
+            print(
+                f"[dbg] gen_cfg={{max_new_tokens:{per_call_kwargs.get('max_new_tokens')}, temp:{per_call_kwargs.get('temperature')}, top_p:{per_call_kwargs.get('top_p')}}}"
+            )
+
+        start = time.perf_counter()
+        # Optional per-call timeout using signal alarm
+        timeout_handler_installed = False
+        if per_call_timeout is not None:
+            import signal
+
+            def _timeout_handler(signum, frame):
+                raise TimeoutError("generation timed out")
+
+            old_handler = signal.signal(signal.SIGALRM, _timeout_handler)
+            signal.alarm(int(per_call_timeout))
+            timeout_handler_installed = True
+
+        try:
+            if verbose:
+                print("[dbg] generation_start")
+                llm.ask(prompt_for_llm, **per_call_kwargs)
+                print("[dbg] generation_done")
+            else:
+                with contextlib.redirect_stdout(suppress_buffer):
+                    llm.ask(prompt_for_llm, **per_call_kwargs)
+        except TimeoutError:
+            if verbose:
+                print("[dbg] generation_timeout")
+        finally:
+            if timeout_handler_installed:
+                import signal
+
+                signal.alarm(0)
+                signal.signal(signal.SIGALRM, old_handler)
+            end = time.perf_counter()
+        timings.append(end - start)
+        suppress_buffer.seek(0)
+        suppress_buffer.truncate(0)
+
+    return timings
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Measure generation timing for prompt files")
+    parser.add_argument(
+        "--max-prompts",
+        type=int,
+        default=None,
+        help="Optional limit on number of prompts to evaluate per file",
+    )
+    parser.add_argument(
+        "--allow-truncation",
+        action="store_true",
+        help="Allow truncating prompt context to respect model's max context",
+    )
+    parser.add_argument(
+        "--model",
+        type=str,
+        default="sshleifer/tiny-gpt2",
+        help="LLM model identifier (default: sshleifer/tiny-gpt2)",
+    )
+    parser.add_argument(
+        "--llm-type",
+        type=str,
+        default="hf",
+        choices=["hf", "openai", "ollama", "gemini", "simulated"],
+        help="LLM backend type (default: hf)",
+    )
+    parser.add_argument(
+        "--device",
+        type=str,
+        default="cpu",
+        choices=["cpu", "auto"],
+        help="Device override for HF models (default: cpu)",
+    )
+    parser.add_argument(
+        "--max-new-tokens",
+        type=int,
+        default=16,
+        help="Max new tokens per generation (default: 16)",
+    )
+    parser.add_argument(
+        "--temperature",
+        type=float,
+        default=0.2,
+        help="Sampling temperature (default: 0.2)",
+    )
+    parser.add_argument(
+        "--top-p",
+        type=float,
+        default=0.8,
+        help="Nucleus sampling top-p (default: 0.8)",
+    )
+    parser.add_argument(
+        "--qwen-thinking",
+        action="store_true",
+        help="Append /think to prompts for Qwen models",
+    )
+    parser.add_argument(
+        "--no-max-new-tokens",
+        action="store_true",
+        help="Do not set max_new_tokens in generation kwargs",
+    )
+    parser.add_argument(
+        "--per-call-timeout",
+        type=int,
+        default=None,
+        help="Optional timeout (seconds) per generation call; if hit, moves to next prompt",
+    )
+    parser.add_argument(
+        "--stream",
+        action="store_true",
+        help="Stream generated text to stdout during generation",
+    )
+    parser.add_argument(
+        "--datasets",
+        type=str,
+        default=None,
+        help=(
+            "Comma-separated subset of datasets to run. Options: gpqa_bm25,gpqa_diskann,gpqa_hnsw. "
+            "Default: all"
+        ),
+    )
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="Enable debug logging and show generation progress",
+    )
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    dataset_map = {
+        # "gpqa_bm25": Path("prompt_dump_gpqa_bm25.txt"),
+        # "gpqa_diskann": Path("prompt_dump_gpqa_diskann.txt"),
+        # "gpqa_hnsw": Path("prompt_dump_gpqa_hnsw.txt"),
+        # "nq_bm25": Path("prompt_dump_nq_bm25.txt"),
+        # # "nq_diskann": Path("prompt_dump_nq_diskann.txt"),
+        # "nq_hnsw": Path("prompt_dump_nq_hnsw.txt"),
+        "gpqa_bm25": Path("prompt_dump_hotpot_bm25.txt"),
+        "gpqa_diskann": Path("prompt_dump_hotpot_diskann.txt"),
+        # "gpqa_hnsw": Path("prompt_dump_hotpot_hnsw.txt"),
+        # "gpqa_bm25": Path("prompt_dump_trivia_bm25.txt"),
+        # "gpqa_diskann": Path("prompt_dump_trivia_diskann.txt"),
+    }
+    if args.datasets:
+        selected = [k.strip() for k in args.datasets.split(",") if k.strip()]
+        invalid = [k for k in selected if k not in dataset_map]
+        if invalid:
+            raise SystemExit(f"Invalid dataset names: {invalid}. Valid: {list(dataset_map)}")
+        dataset_files = [dataset_map[k] for k in selected]
+    else:
+        dataset_files = list(dataset_map.values())
+
+    generation_kwargs = {
+        "temperature": args.temperature,
+        "top_p": args.top_p,
+    }
+    if not args.no_max_new_tokens:
+        generation_kwargs["max_new_tokens"] = args.max_new_tokens
+
+    results: dict[str, dict[str, float | int]] = {}
+
+    llm_config = {"type": args.llm_type, "model": args.model}
+    try:
+        llm = get_llm(llm_config)
+    except Exception as exc:
+        print(f"Failed to initialize LLM: {exc}")
+        raise SystemExit(1) from exc
+
+    if args.llm_type == "hf" and hasattr(llm, "model") and args.device == "cpu":
+        llm.model = llm.model.to("cpu")
+        if hasattr(llm, "device"):
+            llm.device = "cpu"
+
+    for dataset_path in dataset_files:
+        print(f"Processing {dataset_path.name}...")
+        prompts = load_prompts(dataset_path)
+        if args.max_prompts is not None:
+            prompts = prompts[: args.max_prompts]
+        if args.verbose:
+            print(f"[dbg] loaded_prompts={len(prompts)} (showing up to --max-prompts)")
+        timings = measure_generation_times(
+            prompts,
+            llm,
+            generation_kwargs,
+            args.allow_truncation,
+            args.qwen_thinking,
+            args.verbose or args.stream,
+            args.per_call_timeout,
+        )
+        total_time = sum(timings)
+        count = len(timings)
+        average_time = total_time / count if count else 0.0
+        results[str(dataset_path.name)] = {
+            "total_prompts": count,
+            "total_time_seconds": total_time,
+            "average_time_seconds": average_time,
+        }
+
+    print(json.dumps(results, indent=2))
+
+
+if __name__ == "__main__":
+    main()

tests/test_embedding_server_manager.py

@@ -1,137 +0,0 @@
-import json
-import time
-
-import pytest
-from leann.embedding_server_manager import EmbeddingServerManager
-
-
-class DummyProcess:
-    def __init__(self):
-        self.pid = 12345
-        self._terminated = False
-
-    def poll(self):
-        return 0 if self._terminated else None
-
-    def terminate(self):
-        self._terminated = True
-
-    def kill(self):
-        self._terminated = True
-
-    def wait(self, timeout=None):
-        self._terminated = True
-        return 0
-
-
-@pytest.fixture
-def embedding_manager(monkeypatch):
-    manager = EmbeddingServerManager("leann_backend_hnsw.hnsw_embedding_server")
-
-    def fake_get_available_port(start_port):
-        return start_port
-
-    monkeypatch.setattr(
-        "leann.embedding_server_manager._get_available_port",
-        fake_get_available_port,
-    )
-
-    start_calls = []
-
-    def fake_start_new_server(self, port, model_name, embedding_mode, **kwargs):
-        config_signature = kwargs.get("config_signature")
-        start_calls.append(config_signature)
-        self.server_process = DummyProcess()
-        self.server_port = port
-        self._server_config = config_signature
-        return True, port
-
-    monkeypatch.setattr(
-        EmbeddingServerManager,
-        "_start_new_server",
-        fake_start_new_server,
-    )
-
-    # Ensure stop_server doesn't try to operate on real subprocesses
-    def fake_stop_server(self):
-        self.server_process = None
-        self.server_port = None
-        self._server_config = None
-
-    monkeypatch.setattr(EmbeddingServerManager, "stop_server", fake_stop_server)
-
-    return manager, start_calls
-
-
-def _write_meta(meta_path, passages_name, index_name, total):
-    meta_path.write_text(
-        json.dumps(
-            {
-                "backend_name": "hnsw",
-                "embedding_model": "test-model",
-                "embedding_mode": "sentence-transformers",
-                "dimensions": 3,
-                "backend_kwargs": {},
-                "passage_sources": [
-                    {
-                        "type": "jsonl",
-                        "path": passages_name,
-                        "index_path": index_name,
-                    }
-                ],
-                "total_passages": total,
-            }
-        ),
-        encoding="utf-8",
-    )
-
-
-def test_server_restarts_when_metadata_changes(tmp_path, embedding_manager):
-    manager, start_calls = embedding_manager
-
-    meta_path = tmp_path / "example.meta.json"
-    passages_path = tmp_path / "example.passages.jsonl"
-    index_path = tmp_path / "example.passages.idx"
-
-    passages_path.write_text("first\n", encoding="utf-8")
-    index_path.write_bytes(b"index")
-    _write_meta(meta_path, passages_path.name, index_path.name, total=1)
-
-    # Initial start populates signature
-    ok, port = manager.start_server(
-        port=6000,
-        model_name="test-model",
-        passages_file=str(meta_path),
-    )
-    assert ok
-    assert port == 6000
-    assert len(start_calls) == 1
-
-    initial_signature = start_calls[0]["passages_signature"]
-
-    # No metadata change => reuse existing server
-    ok, port_again = manager.start_server(
-        port=6000,
-        model_name="test-model",
-        passages_file=str(meta_path),
-    )
-    assert ok
-    assert port_again == 6000
-    assert len(start_calls) == 1
-
-    # Modify passage data and metadata to force signature change
-    time.sleep(0.01)  # Ensure filesystem timestamps move forward
-    passages_path.write_text("second\n", encoding="utf-8")
-    _write_meta(meta_path, passages_path.name, index_path.name, total=2)
-
-    ok, port_third = manager.start_server(
-        port=6000,
-        model_name="test-model",
-        passages_file=str(meta_path),
-    )
-    assert ok
-    assert port_third == 6000
-    assert len(start_calls) == 2
-
-    updated_signature = start_calls[1]["passages_signature"]
-    assert updated_signature != initial_signature

tests/test_mcp_integration.py

@@ -1,208 +0,0 @@
-#!/usr/bin/env python3
-"""
-Test script for MCP integration implementations.
-
-This script tests the basic functionality of the MCP readers and RAG applications
-without requiring actual MCP servers to be running.
-"""
-
-import sys
-from pathlib import Path
-
-# Add the parent directory to the path so we can import from apps
-sys.path.append(str(Path(__file__).parent.parent))
-
-from apps.slack_data.slack_mcp_reader import SlackMCPReader
-from apps.slack_rag import SlackMCPRAG
-from apps.twitter_data.twitter_mcp_reader import TwitterMCPReader
-from apps.twitter_rag import TwitterMCPRAG
-
-
-def test_slack_reader_initialization():
-    """Test that SlackMCPReader can be initialized with various parameters."""
-    print("Testing SlackMCPReader initialization...")
-
-    # Test basic initialization
-    reader = SlackMCPReader("slack-mcp-server")
-    assert reader.mcp_server_command == "slack-mcp-server"
-    assert reader.concatenate_conversations
-    assert reader.max_messages_per_conversation == 100
-
-    # Test with custom parameters
-    reader = SlackMCPReader(
-        "custom-slack-server",
-        workspace_name="test-workspace",
-        concatenate_conversations=False,
-        max_messages_per_conversation=50,
-    )
-    assert reader.workspace_name == "test-workspace"
-    assert not reader.concatenate_conversations
-    assert reader.max_messages_per_conversation == 50
-
-    print("✅ SlackMCPReader initialization tests passed")
-
-
-def test_twitter_reader_initialization():
-    """Test that TwitterMCPReader can be initialized with various parameters."""
-    print("Testing TwitterMCPReader initialization...")
-
-    # Test basic initialization
-    reader = TwitterMCPReader("twitter-mcp-server")
-    assert reader.mcp_server_command == "twitter-mcp-server"
-    assert reader.include_tweet_content
-    assert reader.include_metadata
-    assert reader.max_bookmarks == 1000
-
-    # Test with custom parameters
-    reader = TwitterMCPReader(
-        "custom-twitter-server",
-        username="testuser",
-        include_tweet_content=False,
-        include_metadata=False,
-        max_bookmarks=500,
-    )
-    assert reader.username == "testuser"
-    assert not reader.include_tweet_content
-    assert not reader.include_metadata
-    assert reader.max_bookmarks == 500
-
-    print("✅ TwitterMCPReader initialization tests passed")
-
-
-def test_slack_message_formatting():
-    """Test Slack message formatting functionality."""
-    print("Testing Slack message formatting...")
-
-    reader = SlackMCPReader("slack-mcp-server")
-
-    # Test basic message formatting
-    message = {
-        "text": "Hello, world!",
-        "user": "john_doe",
-        "channel": "general",
-        "ts": "1234567890.123456",
-    }
-
-    formatted = reader._format_message(message)
-    assert "Channel: #general" in formatted
-    assert "User: john_doe" in formatted
-    assert "Message: Hello, world!" in formatted
-    assert "Time:" in formatted
-
-    # Test with missing fields
-    message = {"text": "Simple message"}
-    formatted = reader._format_message(message)
-    assert "Message: Simple message" in formatted
-
-    print("✅ Slack message formatting tests passed")
-
-
-def test_twitter_bookmark_formatting():
-    """Test Twitter bookmark formatting functionality."""
-    print("Testing Twitter bookmark formatting...")
-
-    reader = TwitterMCPReader("twitter-mcp-server")
-
-    # Test basic bookmark formatting
-    bookmark = {
-        "text": "This is a great article about AI!",
-        "author": "ai_researcher",
-        "created_at": "2024-01-01T12:00:00Z",
-        "url": "https://twitter.com/ai_researcher/status/123456789",
-        "likes": 42,
-        "retweets": 15,
-    }
-
-    formatted = reader._format_bookmark(bookmark)
-    assert "=== Twitter Bookmark ===" in formatted
-    assert "Author: @ai_researcher" in formatted
-    assert "Content:" in formatted
-    assert "This is a great article about AI!" in formatted
-    assert "URL: https://twitter.com" in formatted
-    assert "Likes: 42" in formatted
-    assert "Retweets: 15" in formatted
-
-    # Test with minimal data
-    bookmark = {"text": "Simple tweet"}
-    formatted = reader._format_bookmark(bookmark)
-    assert "=== Twitter Bookmark ===" in formatted
-    assert "Simple tweet" in formatted
-
-    print("✅ Twitter bookmark formatting tests passed")
-
-
-def test_slack_rag_initialization():
-    """Test that SlackMCPRAG can be initialized."""
-    print("Testing SlackMCPRAG initialization...")
-
-    app = SlackMCPRAG()
-    assert app.default_index_name == "slack_messages"
-    assert hasattr(app, "parser")
-
-    print("✅ SlackMCPRAG initialization tests passed")
-
-
-def test_twitter_rag_initialization():
-    """Test that TwitterMCPRAG can be initialized."""
-    print("Testing TwitterMCPRAG initialization...")
-
-    app = TwitterMCPRAG()
-    assert app.default_index_name == "twitter_bookmarks"
-    assert hasattr(app, "parser")
-
-    print("✅ TwitterMCPRAG initialization tests passed")
-
-
-def test_concatenated_content_creation():
-    """Test creation of concatenated content from multiple messages."""
-    print("Testing concatenated content creation...")
-
-    reader = SlackMCPReader("slack-mcp-server", workspace_name="test-workspace")
-
-    messages = [
-        {"text": "First message", "user": "alice", "ts": "1000"},
-        {"text": "Second message", "user": "bob", "ts": "2000"},
-        {"text": "Third message", "user": "charlie", "ts": "3000"},
-    ]
-
-    content = reader._create_concatenated_content(messages, "general")
-
-    assert "Slack Channel: #general" in content
-    assert "Message Count: 3" in content
-    assert "Workspace: test-workspace" in content
-    assert "First message" in content
-    assert "Second message" in content
-    assert "Third message" in content
-
-    print("✅ Concatenated content creation tests passed")
-
-
-def main():
-    """Run all tests."""
-    print("🧪 Running MCP Integration Tests")
-    print("=" * 50)
-
-    try:
-        test_slack_reader_initialization()
-        test_twitter_reader_initialization()
-        test_slack_message_formatting()
-        test_twitter_bookmark_formatting()
-        test_slack_rag_initialization()
-        test_twitter_rag_initialization()
-        test_concatenated_content_creation()
-
-        print("\n" + "=" * 50)
-        print("🎉 All tests passed! MCP integration is working correctly.")
-        print("\nNext steps:")
-        print("1. Install actual MCP servers for Slack and Twitter")
-        print("2. Configure API credentials")
-        print("3. Test with --test-connection flag")
-        print("4. Start indexing your live data!")
-
-    except Exception as e:
-        print(f"\n❌ Test failed: {e}")
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    main()

tests/test_mcp_standalone.py

@@ -1,221 +0,0 @@
-#!/usr/bin/env python3
-"""
-Standalone test script for MCP integration implementations.
-
-This script tests the basic functionality of the MCP readers
-without requiring LEANN core dependencies.
-"""
-
-import json
-import sys
-from pathlib import Path
-
-# Add the parent directory to the path so we can import from apps
-sys.path.append(str(Path(__file__).parent.parent))
-
-
-def test_slack_reader_basic():
-    """Test basic SlackMCPReader functionality without async operations."""
-    print("Testing SlackMCPReader basic functionality...")
-
-    # Import and test initialization
-    from apps.slack_data.slack_mcp_reader import SlackMCPReader
-
-    reader = SlackMCPReader("slack-mcp-server")
-    assert reader.mcp_server_command == "slack-mcp-server"
-    assert reader.concatenate_conversations
-
-    # Test message formatting
-    message = {
-        "text": "Hello team! How's the project going?",
-        "user": "john_doe",
-        "channel": "general",
-        "ts": "1234567890.123456",
-    }
-
-    formatted = reader._format_message(message)
-    assert "Channel: #general" in formatted
-    assert "User: john_doe" in formatted
-    assert "Message: Hello team!" in formatted
-
-    # Test concatenated content creation
-    messages = [
-        {"text": "First message", "user": "alice", "ts": "1000"},
-        {"text": "Second message", "user": "bob", "ts": "2000"},
-    ]
-
-    content = reader._create_concatenated_content(messages, "dev-team")
-    assert "Slack Channel: #dev-team" in content
-    assert "Message Count: 2" in content
-    assert "First message" in content
-    assert "Second message" in content
-
-    print("✅ SlackMCPReader basic tests passed")
-
-
-def test_twitter_reader_basic():
-    """Test basic TwitterMCPReader functionality."""
-    print("Testing TwitterMCPReader basic functionality...")
-
-    from apps.twitter_data.twitter_mcp_reader import TwitterMCPReader
-
-    reader = TwitterMCPReader("twitter-mcp-server")
-    assert reader.mcp_server_command == "twitter-mcp-server"
-    assert reader.include_tweet_content
-    assert reader.max_bookmarks == 1000
-
-    # Test bookmark formatting
-    bookmark = {
-        "text": "Amazing article about the future of AI! Must read for everyone interested in tech.",
-        "author": "tech_guru",
-        "created_at": "2024-01-15T14:30:00Z",
-        "url": "https://twitter.com/tech_guru/status/123456789",
-        "likes": 156,
-        "retweets": 42,
-        "replies": 23,
-        "hashtags": ["AI", "tech", "future"],
-        "mentions": ["@openai", "@anthropic"],
-    }
-
-    formatted = reader._format_bookmark(bookmark)
-    assert "=== Twitter Bookmark ===" in formatted
-    assert "Author: @tech_guru" in formatted
-    assert "Amazing article about the future of AI!" in formatted
-    assert "Likes: 156" in formatted
-    assert "Retweets: 42" in formatted
-    assert "Hashtags: AI, tech, future" in formatted
-    assert "Mentions: @openai, @anthropic" in formatted
-
-    # Test with minimal data
-    simple_bookmark = {"text": "Short tweet", "author": "user123"}
-    formatted_simple = reader._format_bookmark(simple_bookmark)
-    assert "=== Twitter Bookmark ===" in formatted_simple
-    assert "Short tweet" in formatted_simple
-    assert "Author: @user123" in formatted_simple
-
-    print("✅ TwitterMCPReader basic tests passed")
-
-
-def test_mcp_request_format():
-    """Test MCP request formatting."""
-    print("Testing MCP request formatting...")
-
-    # Test initialization request format
-    init_request = {
-        "jsonrpc": "2.0",
-        "id": 1,
-        "method": "initialize",
-        "params": {
-            "protocolVersion": "2024-11-05",
-            "capabilities": {},
-            "clientInfo": {"name": "leann-slack-reader", "version": "1.0.0"},
-        },
-    }
-
-    # Verify it's valid JSON
-    json_str = json.dumps(init_request)
-    parsed = json.loads(json_str)
-    assert parsed["jsonrpc"] == "2.0"
-    assert parsed["method"] == "initialize"
-    assert parsed["params"]["protocolVersion"] == "2024-11-05"
-
-    # Test tools/list request
-    list_request = {"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}}
-
-    json_str = json.dumps(list_request)
-    parsed = json.loads(json_str)
-    assert parsed["method"] == "tools/list"
-
-    print("✅ MCP request formatting tests passed")
-
-
-def test_data_processing():
-    """Test data processing capabilities."""
-    print("Testing data processing capabilities...")
-
-    from apps.slack_data.slack_mcp_reader import SlackMCPReader
-    from apps.twitter_data.twitter_mcp_reader import TwitterMCPReader
-
-    # Test Slack message processing with various formats
-    slack_reader = SlackMCPReader("test-server")
-
-    messages_with_timestamps = [
-        {"text": "Meeting in 5 minutes", "user": "alice", "ts": "1000.123"},
-        {"text": "On my way!", "user": "bob", "ts": "1001.456"},
-        {"text": "Starting now", "user": "charlie", "ts": "1002.789"},
-    ]
-
-    content = slack_reader._create_concatenated_content(messages_with_timestamps, "meetings")
-    assert "Meeting in 5 minutes" in content
-    assert "On my way!" in content
-    assert "Starting now" in content
-
-    # Test Twitter bookmark processing with engagement data
-    twitter_reader = TwitterMCPReader("test-server", include_metadata=True)
-
-    high_engagement_bookmark = {
-        "text": "Thread about startup lessons learned 🧵",
-        "author": "startup_founder",
-        "likes": 1250,
-        "retweets": 340,
-        "replies": 89,
-    }
-
-    formatted = twitter_reader._format_bookmark(high_engagement_bookmark)
-    assert "Thread about startup lessons learned" in formatted
-    assert "Likes: 1250" in formatted
-    assert "Retweets: 340" in formatted
-    assert "Replies: 89" in formatted
-
-    # Test with metadata disabled
-    twitter_reader_no_meta = TwitterMCPReader("test-server", include_metadata=False)
-    formatted_no_meta = twitter_reader_no_meta._format_bookmark(high_engagement_bookmark)
-    assert "Thread about startup lessons learned" in formatted_no_meta
-    assert "Likes:" not in formatted_no_meta
-    assert "Retweets:" not in formatted_no_meta
-
-    print("✅ Data processing tests passed")
-
-
-def main():
-    """Run all standalone tests."""
-    print("🧪 Running MCP Integration Standalone Tests")
-    print("=" * 60)
-    print("Testing core functionality without LEANN dependencies...")
-    print()
-
-    try:
-        test_slack_reader_basic()
-        test_twitter_reader_basic()
-        test_mcp_request_format()
-        test_data_processing()
-
-        print("\n" + "=" * 60)
-        print("🎉 All standalone tests passed!")
-        print("\n✨ MCP Integration Summary:")
-        print("- SlackMCPReader: Ready for Slack message processing")
-        print("- TwitterMCPReader: Ready for Twitter bookmark processing")
-        print("- MCP Protocol: Properly formatted JSON-RPC requests")
-        print("- Data Processing: Handles various message/bookmark formats")
-
-        print("\n🚀 Next Steps:")
-        print("1. Install MCP servers: npm install -g slack-mcp-server twitter-mcp-server")
-        print("2. Configure API credentials for Slack and Twitter")
-        print("3. Test connections: python -m apps.slack_rag --test-connection")
-        print("4. Start indexing live data from your platforms!")
-
-        print("\n📖 Documentation:")
-        print("- Check README.md for detailed setup instructions")
-        print("- Run examples/mcp_integration_demo.py for usage examples")
-        print("- Explore apps/slack_rag.py and apps/twitter_rag.py for implementation details")
-
-    except Exception as e:
-        print(f"\n❌ Test failed: {e}")
-        import traceback
-
-        traceback.print_exc()
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    main()

uv.lock

@@ -14,7 +14,9 @@ version = "1.10.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "huggingface-hub" },
-    { name = "numpy" },
+    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
+    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
     { name = "packaging" },
     { name = "psutil" },
     { name = "pyyaml" },
@@ -201,7 +203,9 @@ name = "astchunk"
 version = "0.1.0"
 source = { editable = "packages/astchunk-leann" }
 dependencies = [
-    { name = "numpy" },
+    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
+    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
     { name = "pyrsistent" },
     { name = "tree-sitter", version = "0.23.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
     { name = "tree-sitter", version = "0.25.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
@@ -585,7 +589,7 @@ resolution-markers = [
     "python_full_version < '3.10'",
 ]
 dependencies = [
-    { name = "numpy", marker = "python_full_version < '3.10'" },
+    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/f5/f6/31a8f28b4a2a4fa0e01085e542f3081ab0588eff8e589d39d775172c9792/contourpy-1.3.0.tar.gz", hash = "sha256:7ffa0db17717a8ffb127efd0c95a4362d996b892c2904db72428d5b52e1938a4", size = 13464370 }
 wheels = [
@@ -663,7 +667,7 @@ resolution-markers = [
     "python_full_version == '3.10.*'",
 ]
 dependencies = [
-    { name = "numpy", marker = "python_full_version == '3.10.*'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/66/54/eb9bfc647b19f2009dd5c7f5ec51c4e6ca831725f1aea7a993034f483147/contourpy-1.3.2.tar.gz", hash = "sha256:b6945942715a034c671b7fc54f9588126b0b8bf23db2696e3ca8328f3ff0ab54", size = 13466130 }
 wheels = [
@@ -734,7 +738,7 @@ resolution-markers = [
     "python_full_version == '3.11.*'",
 ]
 dependencies = [
-    { name = "numpy", marker = "python_full_version >= '3.11'" },
+    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/58/01/1253e6698a07380cd31a736d248a3f2a50a7c88779a1813da27503cadc2a/contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880", size = 13466174 }
 wheels = [
@@ -1024,7 +1028,9 @@ dependencies = [
     { name = "fsspec", extra = ["http"] },
     { name = "huggingface-hub" },
     { name = "multiprocess" },
-    { name = "numpy" },
+    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
+    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
     { name = "packaging" },
     { name = "pandas" },
     { name = "pyarrow" },
@@ -1165,7 +1171,9 @@ dependencies = [
     { name = "fsspec", extra = ["http"] },
     { name = "huggingface-hub" },
     { name = "multiprocess" },
-    { name = "numpy" },
+    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
+    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
     { name = "packaging" },
     { name = "pandas" },
     { name = "requests" },
@@ -1602,7 +1610,7 @@ name = "importlib-metadata"
 version = "8.7.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "zipp", marker = "python_full_version < '3.10'" },
+    { name = "zipp" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/76/66/650a33bd90f786193e4de4b3ad86ea60b53c89b669a5c7be931fac31cdb0/importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000", size = 56641 }
 wheels = [
@@ -2159,7 +2167,9 @@ version = "0.3.4"
 source = { editable = "packages/leann-backend-diskann" }
 dependencies = [
     { name = "leann-core" },
-    { name = "numpy" },
+    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
+    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
     { name = "protobuf" },
 ]
 
@@ -2177,7 +2187,9 @@ source = { editable = "packages/leann-backend-hnsw" }
 dependencies = [
     { name = "leann-core" },
     { name = "msgpack" },
-    { name = "numpy" },
+    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
+    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
     { name = "pyzmq" },
 ]
 
@@ -2204,7 +2216,9 @@ dependencies = [
     { name = "mlx-lm", marker = "platform_machine == 'arm64' and sys_platform == 'darwin'" },
     { name = "msgpack" },
     { name = "nbconvert" },
-    { name = "numpy" },
+    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
+    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
     { name = "openai" },
     { name = "pdfplumber" },
     { name = "psutil" },
@@ -2241,12 +2255,12 @@ requires-dist = [
     { name = "python-dotenv", specifier = ">=1.0.0" },
     { name = "pyzmq", specifier = ">=23.0.0" },
     { name = "requests", specifier = ">=2.25.0" },
-    { name = "sentence-transformers", specifier = ">=3.0.0" },
+    { name = "sentence-transformers", specifier = ">=2.2.0" },
     { name = "torch", specifier = ">=2.0.0" },
     { name = "torch", marker = "extra == 'colab'", specifier = ">=2.0.0,<3.0.0" },
     { name = "tqdm", specifier = ">=4.60.0" },
-    { name = "transformers", specifier = ">=4.30.0,<4.46" },
-    { name = "transformers", marker = "extra == 'colab'", specifier = ">=4.30.0,<4.46" },
+    { name = "transformers", specifier = ">=4.30.0" },
+    { name = "transformers", marker = "extra == 'colab'", specifier = ">=4.30.0,<5.0.0" },
 ]
 provides-extras = ["colab"]
 
@@ -2272,7 +2286,9 @@ dependencies = [
     { name = "mlx-lm", marker = "platform_machine == 'arm64' and sys_platform == 'darwin'" },
     { name = "msgpack" },
     { name = "nbconvert" },
-    { name = "numpy" },
+    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
+    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
     { name = "ollama" },
     { name = "openai" },
     { name = "pathspec" },
@@ -2290,7 +2306,6 @@ dependencies = [
     { name = "torch" },
     { name = "torchvision" },
     { name = "tqdm" },
-    { name = "transformers" },
     { name = "tree-sitter", version = "0.23.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
     { name = "tree-sitter", version = "0.25.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
     { name = "tree-sitter-c-sharp" },
@@ -2366,12 +2381,11 @@ requires-dist = [
     { name = "pypdfium2", specifier = ">=4.30.0" },
     { name = "python-docx", marker = "extra == 'documents'", specifier = ">=0.8.11" },
     { name = "requests", specifier = ">=2.25.0" },
-    { name = "sentence-transformers", specifier = ">=3.0.0" },
+    { name = "sentence-transformers", specifier = ">=2.2.0" },
     { name = "sglang" },
     { name = "torch" },
     { name = "torchvision", specifier = ">=0.23.0" },
     { name = "tqdm" },
-    { name = "transformers", specifier = "<4.46" },
     { name = "tree-sitter", specifier = ">=0.20.0" },
     { name = "tree-sitter-c-sharp", specifier = ">=0.20.0" },
     { name = "tree-sitter-java", specifier = ">=0.20.0" },
@@ -2486,7 +2500,9 @@ dependencies = [
     { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
     { name = "networkx", version = "3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
     { name = "nltk" },
-    { name = "numpy" },
+    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
+    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
     { name = "pillow" },
     { name = "platformdirs" },
     { name = "pydantic" },
@@ -2907,7 +2923,7 @@ dependencies = [
     { name = "fonttools", marker = "python_full_version < '3.10'" },
     { name = "importlib-resources", marker = "python_full_version < '3.10'" },
     { name = "kiwisolver", version = "1.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
-    { name = "numpy", marker = "python_full_version < '3.10'" },
+    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
     { name = "packaging", marker = "python_full_version < '3.10'" },
     { name = "pillow", marker = "python_full_version < '3.10'" },
     { name = "pyparsing", marker = "python_full_version < '3.10'" },
@@ -2972,7 +2988,8 @@ dependencies = [
     { name = "cycler", marker = "python_full_version >= '3.10'" },
     { name = "fonttools", marker = "python_full_version >= '3.10'" },
     { name = "kiwisolver", version = "1.4.9", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
-    { name = "numpy", marker = "python_full_version >= '3.10'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
+    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
     { name = "packaging", marker = "python_full_version >= '3.10'" },
     { name = "pillow", marker = "python_full_version >= '3.10'" },
     { name = "pyparsing", marker = "python_full_version >= '3.10'" },
@@ -3101,7 +3118,9 @@ source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "jinja2" },
     { name = "mlx" },
-    { name = "numpy" },
+    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
+    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
     { name = "protobuf" },
     { name = "pyyaml" },
     { name = "transformers" },
@@ -3466,45 +3485,207 @@ wheels = [
 
 [[package]]
 name = "numpy"
-version = "1.26.4"
+version = "2.0.2"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/65/6e/09db70a523a96d25e115e71cc56a6f9031e7b8cd166c1ac8438307c14058/numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010", size = 15786129 }
+resolution-markers = [
+    "python_full_version < '3.10'",
+]
+sdist = { url = "https://files.pythonhosted.org/packages/a9/75/10dd1f8116a8b796cb2c737b674e02d02e80454bda953fa7e65d8c12b016/numpy-2.0.2.tar.gz", hash = "sha256:883c987dee1880e2a864ab0dc9892292582510604156762362d9326444636e78", size = 18902015 }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/a7/94/ace0fdea5241a27d13543ee117cbc65868e82213fb31a8eb7fe9ff23f313/numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0", size = 20631468 },
-    { url = "https://files.pythonhosted.org/packages/20/f7/b24208eba89f9d1b58c1668bc6c8c4fd472b20c45573cb767f59d49fb0f6/numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a", size = 13966411 },
-    { url = "https://files.pythonhosted.org/packages/fc/a5/4beee6488160798683eed5bdb7eead455892c3b4e1f78d79d8d3f3b084ac/numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4", size = 14219016 },
-    { url = "https://files.pythonhosted.org/packages/4b/d7/ecf66c1cd12dc28b4040b15ab4d17b773b87fa9d29ca16125de01adb36cd/numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f", size = 18240889 },
-    { url = "https://files.pythonhosted.org/packages/24/03/6f229fe3187546435c4f6f89f6d26c129d4f5bed40552899fcf1f0bf9e50/numpy-1.26.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a", size = 13876746 },
-    { url = "https://files.pythonhosted.org/packages/39/fe/39ada9b094f01f5a35486577c848fe274e374bbf8d8f472e1423a0bbd26d/numpy-1.26.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2", size = 18078620 },
-    { url = "https://files.pythonhosted.org/packages/d5/ef/6ad11d51197aad206a9ad2286dc1aac6a378059e06e8cf22cd08ed4f20dc/numpy-1.26.4-cp310-cp310-win32.whl", hash = "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07", size = 5972659 },
-    { url = "https://files.pythonhosted.org/packages/19/77/538f202862b9183f54108557bfda67e17603fc560c384559e769321c9d92/numpy-1.26.4-cp310-cp310-win_amd64.whl", hash = "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5", size = 15808905 },
-    { url = "https://files.pythonhosted.org/packages/11/57/baae43d14fe163fa0e4c47f307b6b2511ab8d7d30177c491960504252053/numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71", size = 20630554 },
-    { url = "https://files.pythonhosted.org/packages/1a/2e/151484f49fd03944c4a3ad9c418ed193cfd02724e138ac8a9505d056c582/numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef", size = 13997127 },
-    { url = "https://files.pythonhosted.org/packages/79/ae/7e5b85136806f9dadf4878bf73cf223fe5c2636818ba3ab1c585d0403164/numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e", size = 14222994 },
-    { url = "https://files.pythonhosted.org/packages/3a/d0/edc009c27b406c4f9cbc79274d6e46d634d139075492ad055e3d68445925/numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5", size = 18252005 },
-    { url = "https://files.pythonhosted.org/packages/09/bf/2b1aaf8f525f2923ff6cfcf134ae5e750e279ac65ebf386c75a0cf6da06a/numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a", size = 13885297 },
-    { url = "https://files.pythonhosted.org/packages/df/a0/4e0f14d847cfc2a633a1c8621d00724f3206cfeddeb66d35698c4e2cf3d2/numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a", size = 18093567 },
-    { url = "https://files.pythonhosted.org/packages/d2/b7/a734c733286e10a7f1a8ad1ae8c90f2d33bf604a96548e0a4a3a6739b468/numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20", size = 5968812 },
-    { url = "https://files.pythonhosted.org/packages/3f/6b/5610004206cf7f8e7ad91c5a85a8c71b2f2f8051a0c0c4d5916b76d6cbb2/numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2", size = 15811913 },
-    { url = "https://files.pythonhosted.org/packages/95/12/8f2020a8e8b8383ac0177dc9570aad031a3beb12e38847f7129bacd96228/numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218", size = 20335901 },
-    { url = "https://files.pythonhosted.org/packages/75/5b/ca6c8bd14007e5ca171c7c03102d17b4f4e0ceb53957e8c44343a9546dcc/numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b", size = 13685868 },
-    { url = "https://files.pythonhosted.org/packages/79/f8/97f10e6755e2a7d027ca783f63044d5b1bc1ae7acb12afe6a9b4286eac17/numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b", size = 13925109 },
-    { url = "https://files.pythonhosted.org/packages/0f/50/de23fde84e45f5c4fda2488c759b69990fd4512387a8632860f3ac9cd225/numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed", size = 17950613 },
-    { url = "https://files.pythonhosted.org/packages/4c/0c/9c603826b6465e82591e05ca230dfc13376da512b25ccd0894709b054ed0/numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a", size = 13572172 },
-    { url = "https://files.pythonhosted.org/packages/76/8c/2ba3902e1a0fc1c74962ea9bb33a534bb05984ad7ff9515bf8d07527cadd/numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0", size = 17786643 },
-    { url = "https://files.pythonhosted.org/packages/28/4a/46d9e65106879492374999e76eb85f87b15328e06bd1550668f79f7b18c6/numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110", size = 5677803 },
-    { url = "https://files.pythonhosted.org/packages/16/2e/86f24451c2d530c88daf997cb8d6ac622c1d40d19f5a031ed68a4b73a374/numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818", size = 15517754 },
-    { url = "https://files.pythonhosted.org/packages/7d/24/ce71dc08f06534269f66e73c04f5709ee024a1afe92a7b6e1d73f158e1f8/numpy-1.26.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c", size = 20636301 },
-    { url = "https://files.pythonhosted.org/packages/ae/8c/ab03a7c25741f9ebc92684a20125fbc9fc1b8e1e700beb9197d750fdff88/numpy-1.26.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be", size = 13971216 },
-    { url = "https://files.pythonhosted.org/packages/6d/64/c3bcdf822269421d85fe0d64ba972003f9bb4aa9a419da64b86856c9961f/numpy-1.26.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764", size = 14226281 },
-    { url = "https://files.pythonhosted.org/packages/54/30/c2a907b9443cf42b90c17ad10c1e8fa801975f01cb9764f3f8eb8aea638b/numpy-1.26.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3", size = 18249516 },
-    { url = "https://files.pythonhosted.org/packages/43/12/01a563fc44c07095996d0129b8899daf89e4742146f7044cdbdb3101c57f/numpy-1.26.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd", size = 13882132 },
-    { url = "https://files.pythonhosted.org/packages/16/ee/9df80b06680aaa23fc6c31211387e0db349e0e36d6a63ba3bd78c5acdf11/numpy-1.26.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c", size = 18084181 },
-    { url = "https://files.pythonhosted.org/packages/28/7d/4b92e2fe20b214ffca36107f1a3e75ef4c488430e64de2d9af5db3a4637d/numpy-1.26.4-cp39-cp39-win32.whl", hash = "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6", size = 5976360 },
-    { url = "https://files.pythonhosted.org/packages/b5/42/054082bd8220bbf6f297f982f0a8f5479fcbc55c8b511d928df07b965869/numpy-1.26.4-cp39-cp39-win_amd64.whl", hash = "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea", size = 15814633 },
-    { url = "https://files.pythonhosted.org/packages/3f/72/3df6c1c06fc83d9cfe381cccb4be2532bbd38bf93fbc9fad087b6687f1c0/numpy-1.26.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30", size = 20455961 },
-    { url = "https://files.pythonhosted.org/packages/8e/02/570545bac308b58ffb21adda0f4e220ba716fb658a63c151daecc3293350/numpy-1.26.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c", size = 18061071 },
-    { url = "https://files.pythonhosted.org/packages/f4/5f/fafd8c51235f60d49f7a88e2275e13971e90555b67da52dd6416caec32fe/numpy-1.26.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0", size = 15709730 },
+    { url = "https://files.pythonhosted.org/packages/21/91/3495b3237510f79f5d81f2508f9f13fea78ebfdf07538fc7444badda173d/numpy-2.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:51129a29dbe56f9ca83438b706e2e69a39892b5eda6cedcb6b0c9fdc9b0d3ece", size = 21165245 },
+    { url = "https://files.pythonhosted.org/packages/05/33/26178c7d437a87082d11019292dce6d3fe6f0e9026b7b2309cbf3e489b1d/numpy-2.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f15975dfec0cf2239224d80e32c3170b1d168335eaedee69da84fbe9f1f9cd04", size = 13738540 },
+    { url = "https://files.pythonhosted.org/packages/ec/31/cc46e13bf07644efc7a4bf68df2df5fb2a1a88d0cd0da9ddc84dc0033e51/numpy-2.0.2-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:8c5713284ce4e282544c68d1c3b2c7161d38c256d2eefc93c1d683cf47683e66", size = 5300623 },
+    { url = "https://files.pythonhosted.org/packages/6e/16/7bfcebf27bb4f9d7ec67332ffebee4d1bf085c84246552d52dbb548600e7/numpy-2.0.2-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:becfae3ddd30736fe1889a37f1f580e245ba79a5855bff5f2a29cb3ccc22dd7b", size = 6901774 },
+    { url = "https://files.pythonhosted.org/packages/f9/a3/561c531c0e8bf082c5bef509d00d56f82e0ea7e1e3e3a7fc8fa78742a6e5/numpy-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2da5960c3cf0df7eafefd806d4e612c5e19358de82cb3c343631188991566ccd", size = 13907081 },
+    { url = "https://files.pythonhosted.org/packages/fa/66/f7177ab331876200ac7563a580140643d1179c8b4b6a6b0fc9838de2a9b8/numpy-2.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:496f71341824ed9f3d2fd36cf3ac57ae2e0165c143b55c3a035ee219413f3318", size = 19523451 },
+    { url = "https://files.pythonhosted.org/packages/25/7f/0b209498009ad6453e4efc2c65bcdf0ae08a182b2b7877d7ab38a92dc542/numpy-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a61ec659f68ae254e4d237816e33171497e978140353c0c2038d46e63282d0c8", size = 19927572 },
+    { url = "https://files.pythonhosted.org/packages/3e/df/2619393b1e1b565cd2d4c4403bdd979621e2c4dea1f8532754b2598ed63b/numpy-2.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d731a1c6116ba289c1e9ee714b08a8ff882944d4ad631fd411106a30f083c326", size = 14400722 },
+    { url = "https://files.pythonhosted.org/packages/22/ad/77e921b9f256d5da36424ffb711ae79ca3f451ff8489eeca544d0701d74a/numpy-2.0.2-cp310-cp310-win32.whl", hash = "sha256:984d96121c9f9616cd33fbd0618b7f08e0cfc9600a7ee1d6fd9b239186d19d97", size = 6472170 },
+    { url = "https://files.pythonhosted.org/packages/10/05/3442317535028bc29cf0c0dd4c191a4481e8376e9f0db6bcf29703cadae6/numpy-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:c7b0be4ef08607dd04da4092faee0b86607f111d5ae68036f16cc787e250a131", size = 15905558 },
+    { url = "https://files.pythonhosted.org/packages/8b/cf/034500fb83041aa0286e0fb16e7c76e5c8b67c0711bb6e9e9737a717d5fe/numpy-2.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:49ca4decb342d66018b01932139c0961a8f9ddc7589611158cb3c27cbcf76448", size = 21169137 },
+    { url = "https://files.pythonhosted.org/packages/4a/d9/32de45561811a4b87fbdee23b5797394e3d1504b4a7cf40c10199848893e/numpy-2.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:11a76c372d1d37437857280aa142086476136a8c0f373b2e648ab2c8f18fb195", size = 13703552 },
+    { url = "https://files.pythonhosted.org/packages/c1/ca/2f384720020c7b244d22508cb7ab23d95f179fcfff33c31a6eeba8d6c512/numpy-2.0.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:807ec44583fd708a21d4a11d94aedf2f4f3c3719035c76a2bbe1fe8e217bdc57", size = 5298957 },
+    { url = "https://files.pythonhosted.org/packages/0e/78/a3e4f9fb6aa4e6fdca0c5428e8ba039408514388cf62d89651aade838269/numpy-2.0.2-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8cafab480740e22f8d833acefed5cc87ce276f4ece12fdaa2e8903db2f82897a", size = 6905573 },
+    { url = "https://files.pythonhosted.org/packages/a0/72/cfc3a1beb2caf4efc9d0b38a15fe34025230da27e1c08cc2eb9bfb1c7231/numpy-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a15f476a45e6e5a3a79d8a14e62161d27ad897381fecfa4a09ed5322f2085669", size = 13914330 },
+    { url = "https://files.pythonhosted.org/packages/ba/a8/c17acf65a931ce551fee11b72e8de63bf7e8a6f0e21add4c937c83563538/numpy-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13e689d772146140a252c3a28501da66dfecd77490b498b168b501835041f951", size = 19534895 },
+    { url = "https://files.pythonhosted.org/packages/ba/86/8767f3d54f6ae0165749f84648da9dcc8cd78ab65d415494962c86fac80f/numpy-2.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9ea91dfb7c3d1c56a0e55657c0afb38cf1eeae4544c208dc465c3c9f3a7c09f9", size = 19937253 },
+    { url = "https://files.pythonhosted.org/packages/df/87/f76450e6e1c14e5bb1eae6836478b1028e096fd02e85c1c37674606ab752/numpy-2.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c1c9307701fec8f3f7a1e6711f9089c06e6284b3afbbcd259f7791282d660a15", size = 14414074 },
+    { url = "https://files.pythonhosted.org/packages/5c/ca/0f0f328e1e59f73754f06e1adfb909de43726d4f24c6a3f8805f34f2b0fa/numpy-2.0.2-cp311-cp311-win32.whl", hash = "sha256:a392a68bd329eafac5817e5aefeb39038c48b671afd242710b451e76090e81f4", size = 6470640 },
+    { url = "https://files.pythonhosted.org/packages/eb/57/3a3f14d3a759dcf9bf6e9eda905794726b758819df4663f217d658a58695/numpy-2.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:286cd40ce2b7d652a6f22efdfc6d1edf879440e53e76a75955bc0c826c7e64dc", size = 15910230 },
+    { url = "https://files.pythonhosted.org/packages/45/40/2e117be60ec50d98fa08c2f8c48e09b3edea93cfcabd5a9ff6925d54b1c2/numpy-2.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:df55d490dea7934f330006d0f81e8551ba6010a5bf035a249ef61a94f21c500b", size = 20895803 },
+    { url = "https://files.pythonhosted.org/packages/46/92/1b8b8dee833f53cef3e0a3f69b2374467789e0bb7399689582314df02651/numpy-2.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8df823f570d9adf0978347d1f926b2a867d5608f434a7cff7f7908c6570dcf5e", size = 13471835 },
+    { url = "https://files.pythonhosted.org/packages/7f/19/e2793bde475f1edaea6945be141aef6c8b4c669b90c90a300a8954d08f0a/numpy-2.0.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9a92ae5c14811e390f3767053ff54eaee3bf84576d99a2456391401323f4ec2c", size = 5038499 },
+    { url = "https://files.pythonhosted.org/packages/e3/ff/ddf6dac2ff0dd50a7327bcdba45cb0264d0e96bb44d33324853f781a8f3c/numpy-2.0.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:a842d573724391493a97a62ebbb8e731f8a5dcc5d285dfc99141ca15a3302d0c", size = 6633497 },
+    { url = "https://files.pythonhosted.org/packages/72/21/67f36eac8e2d2cd652a2e69595a54128297cdcb1ff3931cfc87838874bd4/numpy-2.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c05e238064fc0610c840d1cf6a13bf63d7e391717d247f1bf0318172e759e692", size = 13621158 },
+    { url = "https://files.pythonhosted.org/packages/39/68/e9f1126d757653496dbc096cb429014347a36b228f5a991dae2c6b6cfd40/numpy-2.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0123ffdaa88fa4ab64835dcbde75dcdf89c453c922f18dced6e27c90d1d0ec5a", size = 19236173 },
+    { url = "https://files.pythonhosted.org/packages/d1/e9/1f5333281e4ebf483ba1c888b1d61ba7e78d7e910fdd8e6499667041cc35/numpy-2.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:96a55f64139912d61de9137f11bf39a55ec8faec288c75a54f93dfd39f7eb40c", size = 19634174 },
+    { url = "https://files.pythonhosted.org/packages/71/af/a469674070c8d8408384e3012e064299f7a2de540738a8e414dcfd639996/numpy-2.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ec9852fb39354b5a45a80bdab5ac02dd02b15f44b3804e9f00c556bf24b4bded", size = 14099701 },
+    { url = "https://files.pythonhosted.org/packages/d0/3d/08ea9f239d0e0e939b6ca52ad403c84a2bce1bde301a8eb4888c1c1543f1/numpy-2.0.2-cp312-cp312-win32.whl", hash = "sha256:671bec6496f83202ed2d3c8fdc486a8fc86942f2e69ff0e986140339a63bcbe5", size = 6174313 },
+    { url = "https://files.pythonhosted.org/packages/b2/b5/4ac39baebf1fdb2e72585c8352c56d063b6126be9fc95bd2bb5ef5770c20/numpy-2.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:cfd41e13fdc257aa5778496b8caa5e856dc4896d4ccf01841daee1d96465467a", size = 15606179 },
+    { url = "https://files.pythonhosted.org/packages/43/c1/41c8f6df3162b0c6ffd4437d729115704bd43363de0090c7f913cfbc2d89/numpy-2.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9059e10581ce4093f735ed23f3b9d283b9d517ff46009ddd485f1747eb22653c", size = 21169942 },
+    { url = "https://files.pythonhosted.org/packages/39/bc/fd298f308dcd232b56a4031fd6ddf11c43f9917fbc937e53762f7b5a3bb1/numpy-2.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:423e89b23490805d2a5a96fe40ec507407b8ee786d66f7328be214f9679df6dd", size = 13711512 },
+    { url = "https://files.pythonhosted.org/packages/96/ff/06d1aa3eeb1c614eda245c1ba4fb88c483bee6520d361641331872ac4b82/numpy-2.0.2-cp39-cp39-macosx_14_0_arm64.whl", hash = "sha256:2b2955fa6f11907cf7a70dab0d0755159bca87755e831e47932367fc8f2f2d0b", size = 5306976 },
+    { url = "https://files.pythonhosted.org/packages/2d/98/121996dcfb10a6087a05e54453e28e58694a7db62c5a5a29cee14c6e047b/numpy-2.0.2-cp39-cp39-macosx_14_0_x86_64.whl", hash = "sha256:97032a27bd9d8988b9a97a8c4d2c9f2c15a81f61e2f21404d7e8ef00cb5be729", size = 6906494 },
+    { url = "https://files.pythonhosted.org/packages/15/31/9dffc70da6b9bbf7968f6551967fc21156207366272c2a40b4ed6008dc9b/numpy-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e795a8be3ddbac43274f18588329c72939870a16cae810c2b73461c40718ab1", size = 13912596 },
+    { url = "https://files.pythonhosted.org/packages/b9/14/78635daab4b07c0930c919d451b8bf8c164774e6a3413aed04a6d95758ce/numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f26b258c385842546006213344c50655ff1555a9338e2e5e02a0756dc3e803dd", size = 19526099 },
+    { url = "https://files.pythonhosted.org/packages/26/4c/0eeca4614003077f68bfe7aac8b7496f04221865b3a5e7cb230c9d055afd/numpy-2.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fec9451a7789926bcf7c2b8d187292c9f93ea30284802a0ab3f5be8ab36865d", size = 19932823 },
+    { url = "https://files.pythonhosted.org/packages/f1/46/ea25b98b13dccaebddf1a803f8c748680d972e00507cd9bc6dcdb5aa2ac1/numpy-2.0.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9189427407d88ff25ecf8f12469d4d39d35bee1db5d39fc5c168c6f088a6956d", size = 14404424 },
+    { url = "https://files.pythonhosted.org/packages/c8/a6/177dd88d95ecf07e722d21008b1b40e681a929eb9e329684d449c36586b2/numpy-2.0.2-cp39-cp39-win32.whl", hash = "sha256:905d16e0c60200656500c95b6b8dca5d109e23cb24abc701d41c02d74c6b3afa", size = 6476809 },
+    { url = "https://files.pythonhosted.org/packages/ea/2b/7fc9f4e7ae5b507c1a3a21f0f15ed03e794c1242ea8a242ac158beb56034/numpy-2.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:a3f4ab0caa7f053f6797fcd4e1e25caee367db3112ef2b6ef82d749530768c73", size = 15911314 },
+    { url = "https://files.pythonhosted.org/packages/8f/3b/df5a870ac6a3be3a86856ce195ef42eec7ae50d2a202be1f5a4b3b340e14/numpy-2.0.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:7f0a0c6f12e07fa94133c8a67404322845220c06a9e80e85999afe727f7438b8", size = 21025288 },
+    { url = "https://files.pythonhosted.org/packages/2c/97/51af92f18d6f6f2d9ad8b482a99fb74e142d71372da5d834b3a2747a446e/numpy-2.0.2-pp39-pypy39_pp73-macosx_14_0_x86_64.whl", hash = "sha256:312950fdd060354350ed123c0e25a71327d3711584beaef30cdaa93320c392d4", size = 6762793 },
+    { url = "https://files.pythonhosted.org/packages/12/46/de1fbd0c1b5ccaa7f9a005b66761533e2f6a3e560096682683a223631fe9/numpy-2.0.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26df23238872200f63518dd2aa984cfca675d82469535dc7162dc2ee52d9dd5c", size = 19334885 },
+    { url = "https://files.pythonhosted.org/packages/cc/dc/d330a6faefd92b446ec0f0dfea4c3207bb1fef3c4771d19cf4543efd2c78/numpy-2.0.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a46288ec55ebbd58947d31d72be2c63cbf839f0a63b49cb755022310792a3385", size = 15828784 },
+]
+
+[[package]]
+name = "numpy"
+version = "2.2.6"
+source = { registry = "https://pypi.org/simple" }
+resolution-markers = [
+    "python_full_version == '3.10.*'",
+]
+sdist = { url = "https://files.pythonhosted.org/packages/76/21/7d2a95e4bba9dc13d043ee156a356c0a8f0c6309dff6b21b4d71a073b8a8/numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd", size = 20276440 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/9a/3e/ed6db5be21ce87955c0cbd3009f2803f59fa08df21b5df06862e2d8e2bdd/numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb", size = 21165245 },
+    { url = "https://files.pythonhosted.org/packages/22/c2/4b9221495b2a132cc9d2eb862e21d42a009f5a60e45fc44b00118c174bff/numpy-2.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e41fd67c52b86603a91c1a505ebaef50b3314de0213461c7a6e99c9a3beff90", size = 14360048 },
+    { url = "https://files.pythonhosted.org/packages/fd/77/dc2fcfc66943c6410e2bf598062f5959372735ffda175b39906d54f02349/numpy-2.2.6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:37e990a01ae6ec7fe7fa1c26c55ecb672dd98b19c3d0e1d1f326fa13cb38d163", size = 5340542 },
+    { url = "https://files.pythonhosted.org/packages/7a/4f/1cb5fdc353a5f5cc7feb692db9b8ec2c3d6405453f982435efc52561df58/numpy-2.2.6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:5a6429d4be8ca66d889b7cf70f536a397dc45ba6faeb5f8c5427935d9592e9cf", size = 6878301 },
+    { url = "https://files.pythonhosted.org/packages/eb/17/96a3acd228cec142fcb8723bd3cc39c2a474f7dcf0a5d16731980bcafa95/numpy-2.2.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efd28d4e9cd7d7a8d39074a4d44c63eda73401580c5c76acda2ce969e0a38e83", size = 14297320 },
+    { url = "https://files.pythonhosted.org/packages/b4/63/3de6a34ad7ad6646ac7d2f55ebc6ad439dbbf9c4370017c50cf403fb19b5/numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc7b73d02efb0e18c000e9ad8b83480dfcd5dfd11065997ed4c6747470ae8915", size = 16801050 },
+    { url = "https://files.pythonhosted.org/packages/07/b6/89d837eddef52b3d0cec5c6ba0456c1bf1b9ef6a6672fc2b7873c3ec4e2e/numpy-2.2.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:74d4531beb257d2c3f4b261bfb0fc09e0f9ebb8842d82a7b4209415896adc680", size = 15807034 },
+    { url = "https://files.pythonhosted.org/packages/01/c8/dc6ae86e3c61cfec1f178e5c9f7858584049b6093f843bca541f94120920/numpy-2.2.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8fc377d995680230e83241d8a96def29f204b5782f371c532579b4f20607a289", size = 18614185 },
+    { url = "https://files.pythonhosted.org/packages/5b/c5/0064b1b7e7c89137b471ccec1fd2282fceaae0ab3a9550f2568782d80357/numpy-2.2.6-cp310-cp310-win32.whl", hash = "sha256:b093dd74e50a8cba3e873868d9e93a85b78e0daf2e98c6797566ad8044e8363d", size = 6527149 },
+    { url = "https://files.pythonhosted.org/packages/a3/dd/4b822569d6b96c39d1215dbae0582fd99954dcbcf0c1a13c61783feaca3f/numpy-2.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:f0fd6321b839904e15c46e0d257fdd101dd7f530fe03fd6359c1ea63738703f3", size = 12904620 },
+    { url = "https://files.pythonhosted.org/packages/da/a8/4f83e2aa666a9fbf56d6118faaaf5f1974d456b1823fda0a176eff722839/numpy-2.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f9f1adb22318e121c5c69a09142811a201ef17ab257a1e66ca3025065b7f53ae", size = 21176963 },
+    { url = "https://files.pythonhosted.org/packages/b3/2b/64e1affc7972decb74c9e29e5649fac940514910960ba25cd9af4488b66c/numpy-2.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c820a93b0255bc360f53eca31a0e676fd1101f673dda8da93454a12e23fc5f7a", size = 14406743 },
+    { url = "https://files.pythonhosted.org/packages/4a/9f/0121e375000b5e50ffdd8b25bf78d8e1a5aa4cca3f185d41265198c7b834/numpy-2.2.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3d70692235e759f260c3d837193090014aebdf026dfd167834bcba43e30c2a42", size = 5352616 },
+    { url = "https://files.pythonhosted.org/packages/31/0d/b48c405c91693635fbe2dcd7bc84a33a602add5f63286e024d3b6741411c/numpy-2.2.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:481b49095335f8eed42e39e8041327c05b0f6f4780488f61286ed3c01368d491", size = 6889579 },
+    { url = "https://files.pythonhosted.org/packages/52/b8/7f0554d49b565d0171eab6e99001846882000883998e7b7d9f0d98b1f934/numpy-2.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b64d8d4d17135e00c8e346e0a738deb17e754230d7e0810ac5012750bbd85a5a", size = 14312005 },
+    { url = "https://files.pythonhosted.org/packages/b3/dd/2238b898e51bd6d389b7389ffb20d7f4c10066d80351187ec8e303a5a475/numpy-2.2.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba10f8411898fc418a521833e014a77d3ca01c15b0c6cdcce6a0d2897e6dbbdf", size = 16821570 },
+    { url = "https://files.pythonhosted.org/packages/83/6c/44d0325722cf644f191042bf47eedad61c1e6df2432ed65cbe28509d404e/numpy-2.2.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd48227a919f1bafbdda0583705e547892342c26fb127219d60a5c36882609d1", size = 15818548 },
+    { url = "https://files.pythonhosted.org/packages/ae/9d/81e8216030ce66be25279098789b665d49ff19eef08bfa8cb96d4957f422/numpy-2.2.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9551a499bf125c1d4f9e250377c1ee2eddd02e01eac6644c080162c0c51778ab", size = 18620521 },
+    { url = "https://files.pythonhosted.org/packages/6a/fd/e19617b9530b031db51b0926eed5345ce8ddc669bb3bc0044b23e275ebe8/numpy-2.2.6-cp311-cp311-win32.whl", hash = "sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47", size = 6525866 },
+    { url = "https://files.pythonhosted.org/packages/31/0a/f354fb7176b81747d870f7991dc763e157a934c717b67b58456bc63da3df/numpy-2.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:e8213002e427c69c45a52bbd94163084025f533a55a59d6f9c5b820774ef3303", size = 12907455 },
+    { url = "https://files.pythonhosted.org/packages/82/5d/c00588b6cf18e1da539b45d3598d3557084990dcc4331960c15ee776ee41/numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff", size = 20875348 },
+    { url = "https://files.pythonhosted.org/packages/66/ee/560deadcdde6c2f90200450d5938f63a34b37e27ebff162810f716f6a230/numpy-2.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c", size = 14119362 },
+    { url = "https://files.pythonhosted.org/packages/3c/65/4baa99f1c53b30adf0acd9a5519078871ddde8d2339dc5a7fde80d9d87da/numpy-2.2.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3", size = 5084103 },
+    { url = "https://files.pythonhosted.org/packages/cc/89/e5a34c071a0570cc40c9a54eb472d113eea6d002e9ae12bb3a8407fb912e/numpy-2.2.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282", size = 6625382 },
+    { url = "https://files.pythonhosted.org/packages/f8/35/8c80729f1ff76b3921d5c9487c7ac3de9b2a103b1cd05e905b3090513510/numpy-2.2.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87", size = 14018462 },
+    { url = "https://files.pythonhosted.org/packages/8c/3d/1e1db36cfd41f895d266b103df00ca5b3cbe965184df824dec5c08c6b803/numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249", size = 16527618 },
+    { url = "https://files.pythonhosted.org/packages/61/c6/03ed30992602c85aa3cd95b9070a514f8b3c33e31124694438d88809ae36/numpy-2.2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49", size = 15505511 },
+    { url = "https://files.pythonhosted.org/packages/b7/25/5761d832a81df431e260719ec45de696414266613c9ee268394dd5ad8236/numpy-2.2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de", size = 18313783 },
+    { url = "https://files.pythonhosted.org/packages/57/0a/72d5a3527c5ebffcd47bde9162c39fae1f90138c961e5296491ce778e682/numpy-2.2.6-cp312-cp312-win32.whl", hash = "sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4", size = 6246506 },
+    { url = "https://files.pythonhosted.org/packages/36/fa/8c9210162ca1b88529ab76b41ba02d433fd54fecaf6feb70ef9f124683f1/numpy-2.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2", size = 12614190 },
+    { url = "https://files.pythonhosted.org/packages/f9/5c/6657823f4f594f72b5471f1db1ab12e26e890bb2e41897522d134d2a3e81/numpy-2.2.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0811bb762109d9708cca4d0b13c4f67146e3c3b7cf8d34018c722adb2d957c84", size = 20867828 },
+    { url = "https://files.pythonhosted.org/packages/dc/9e/14520dc3dadf3c803473bd07e9b2bd1b69bc583cb2497b47000fed2fa92f/numpy-2.2.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:287cc3162b6f01463ccd86be154f284d0893d2b3ed7292439ea97eafa8170e0b", size = 14143006 },
+    { url = "https://files.pythonhosted.org/packages/4f/06/7e96c57d90bebdce9918412087fc22ca9851cceaf5567a45c1f404480e9e/numpy-2.2.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f1372f041402e37e5e633e586f62aa53de2eac8d98cbfb822806ce4bbefcb74d", size = 5076765 },
+    { url = "https://files.pythonhosted.org/packages/73/ed/63d920c23b4289fdac96ddbdd6132e9427790977d5457cd132f18e76eae0/numpy-2.2.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:55a4d33fa519660d69614a9fad433be87e5252f4b03850642f88993f7b2ca566", size = 6617736 },
+    { url = "https://files.pythonhosted.org/packages/85/c5/e19c8f99d83fd377ec8c7e0cf627a8049746da54afc24ef0a0cb73d5dfb5/numpy-2.2.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f92729c95468a2f4f15e9bb94c432a9229d0d50de67304399627a943201baa2f", size = 14010719 },
+    { url = "https://files.pythonhosted.org/packages/19/49/4df9123aafa7b539317bf6d342cb6d227e49f7a35b99c287a6109b13dd93/numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bc23a79bfabc5d056d106f9befb8d50c31ced2fbc70eedb8155aec74a45798f", size = 16526072 },
+    { url = "https://files.pythonhosted.org/packages/b2/6c/04b5f47f4f32f7c2b0e7260442a8cbcf8168b0e1a41ff1495da42f42a14f/numpy-2.2.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3143e4451880bed956e706a3220b4e5cf6172ef05fcc397f6f36a550b1dd868", size = 15503213 },
+    { url = "https://files.pythonhosted.org/packages/17/0a/5cd92e352c1307640d5b6fec1b2ffb06cd0dabe7d7b8227f97933d378422/numpy-2.2.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4f13750ce79751586ae2eb824ba7e1e8dba64784086c98cdbbcc6a42112ce0d", size = 18316632 },
+    { url = "https://files.pythonhosted.org/packages/f0/3b/5cba2b1d88760ef86596ad0f3d484b1cbff7c115ae2429678465057c5155/numpy-2.2.6-cp313-cp313-win32.whl", hash = "sha256:5beb72339d9d4fa36522fc63802f469b13cdbe4fdab4a288f0c441b74272ebfd", size = 6244532 },
+    { url = "https://files.pythonhosted.org/packages/cb/3b/d58c12eafcb298d4e6d0d40216866ab15f59e55d148a5658bb3132311fcf/numpy-2.2.6-cp313-cp313-win_amd64.whl", hash = "sha256:b0544343a702fa80c95ad5d3d608ea3599dd54d4632df855e4c8d24eb6ecfa1c", size = 12610885 },
+    { url = "https://files.pythonhosted.org/packages/6b/9e/4bf918b818e516322db999ac25d00c75788ddfd2d2ade4fa66f1f38097e1/numpy-2.2.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0bca768cd85ae743b2affdc762d617eddf3bcf8724435498a1e80132d04879e6", size = 20963467 },
+    { url = "https://files.pythonhosted.org/packages/61/66/d2de6b291507517ff2e438e13ff7b1e2cdbdb7cb40b3ed475377aece69f9/numpy-2.2.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fc0c5673685c508a142ca65209b4e79ed6740a4ed6b2267dbba90f34b0b3cfda", size = 14225144 },
+    { url = "https://files.pythonhosted.org/packages/e4/25/480387655407ead912e28ba3a820bc69af9adf13bcbe40b299d454ec011f/numpy-2.2.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:5bd4fc3ac8926b3819797a7c0e2631eb889b4118a9898c84f585a54d475b7e40", size = 5200217 },
+    { url = "https://files.pythonhosted.org/packages/aa/4a/6e313b5108f53dcbf3aca0c0f3e9c92f4c10ce57a0a721851f9785872895/numpy-2.2.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:fee4236c876c4e8369388054d02d0e9bb84821feb1a64dd59e137e6511a551f8", size = 6712014 },
+    { url = "https://files.pythonhosted.org/packages/b7/30/172c2d5c4be71fdf476e9de553443cf8e25feddbe185e0bd88b096915bcc/numpy-2.2.6-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1dda9c7e08dc141e0247a5b8f49cf05984955246a327d4c48bda16821947b2f", size = 14077935 },
+    { url = "https://files.pythonhosted.org/packages/12/fb/9e743f8d4e4d3c710902cf87af3512082ae3d43b945d5d16563f26ec251d/numpy-2.2.6-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f447e6acb680fd307f40d3da4852208af94afdfab89cf850986c3ca00562f4fa", size = 16600122 },
+    { url = "https://files.pythonhosted.org/packages/12/75/ee20da0e58d3a66f204f38916757e01e33a9737d0b22373b3eb5a27358f9/numpy-2.2.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:389d771b1623ec92636b0786bc4ae56abafad4a4c513d36a55dce14bd9ce8571", size = 15586143 },
+    { url = "https://files.pythonhosted.org/packages/76/95/bef5b37f29fc5e739947e9ce5179ad402875633308504a52d188302319c8/numpy-2.2.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8e9ace4a37db23421249ed236fdcdd457d671e25146786dfc96835cd951aa7c1", size = 18385260 },
+    { url = "https://files.pythonhosted.org/packages/09/04/f2f83279d287407cf36a7a8053a5abe7be3622a4363337338f2585e4afda/numpy-2.2.6-cp313-cp313t-win32.whl", hash = "sha256:038613e9fb8c72b0a41f025a7e4c3f0b7a1b5d768ece4796b674c8f3fe13efff", size = 6377225 },
+    { url = "https://files.pythonhosted.org/packages/67/0e/35082d13c09c02c011cf21570543d202ad929d961c02a147493cb0c2bdf5/numpy-2.2.6-cp313-cp313t-win_amd64.whl", hash = "sha256:6031dd6dfecc0cf9f668681a37648373bddd6421fff6c66ec1624eed0180ee06", size = 12771374 },
+    { url = "https://files.pythonhosted.org/packages/9e/3b/d94a75f4dbf1ef5d321523ecac21ef23a3cd2ac8b78ae2aac40873590229/numpy-2.2.6-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0b605b275d7bd0c640cad4e5d30fa701a8d59302e127e5f79138ad62762c3e3d", size = 21040391 },
+    { url = "https://files.pythonhosted.org/packages/17/f4/09b2fa1b58f0fb4f7c7963a1649c64c4d315752240377ed74d9cd878f7b5/numpy-2.2.6-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:7befc596a7dc9da8a337f79802ee8adb30a552a94f792b9c9d18c840055907db", size = 6786754 },
+    { url = "https://files.pythonhosted.org/packages/af/30/feba75f143bdc868a1cc3f44ccfa6c4b9ec522b36458e738cd00f67b573f/numpy-2.2.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce47521a4754c8f4593837384bd3424880629f718d87c5d44f8ed763edd63543", size = 16643476 },
+    { url = "https://files.pythonhosted.org/packages/37/48/ac2a9584402fb6c0cd5b5d1a91dcf176b15760130dd386bbafdbfe3640bf/numpy-2.2.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d042d24c90c41b54fd506da306759e06e568864df8ec17ccc17e9e884634fd00", size = 12812666 },
+]
+
+[[package]]
+name = "numpy"
+version = "2.3.3"
+source = { registry = "https://pypi.org/simple" }
+resolution-markers = [
+    "python_full_version >= '3.12'",
+    "python_full_version == '3.11.*'",
+]
+sdist = { url = "https://files.pythonhosted.org/packages/d0/19/95b3d357407220ed24c139018d2518fab0a61a948e68286a25f1a4d049ff/numpy-2.3.3.tar.gz", hash = "sha256:ddc7c39727ba62b80dfdbedf400d1c10ddfa8eefbd7ec8dcb118be8b56d31029", size = 20576648 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/7a/45/e80d203ef6b267aa29b22714fb558930b27960a0c5ce3c19c999232bb3eb/numpy-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0ffc4f5caba7dfcbe944ed674b7eef683c7e94874046454bb79ed7ee0236f59d", size = 21259253 },
+    { url = "https://files.pythonhosted.org/packages/52/18/cf2c648fccf339e59302e00e5f2bc87725a3ce1992f30f3f78c9044d7c43/numpy-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7e946c7170858a0295f79a60214424caac2ffdb0063d4d79cb681f9aa0aa569", size = 14450980 },
+    { url = "https://files.pythonhosted.org/packages/93/fb/9af1082bec870188c42a1c239839915b74a5099c392389ff04215dcee812/numpy-2.3.3-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:cd4260f64bc794c3390a63bf0728220dd1a68170c169088a1e0dfa2fde1be12f", size = 5379709 },
+    { url = "https://files.pythonhosted.org/packages/75/0f/bfd7abca52bcbf9a4a65abc83fe18ef01ccdeb37bfb28bbd6ad613447c79/numpy-2.3.3-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:f0ddb4b96a87b6728df9362135e764eac3cfa674499943ebc44ce96c478ab125", size = 6913923 },
+    { url = "https://files.pythonhosted.org/packages/79/55/d69adad255e87ab7afda1caf93ca997859092afeb697703e2f010f7c2e55/numpy-2.3.3-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:afd07d377f478344ec6ca2b8d4ca08ae8bd44706763d1efb56397de606393f48", size = 14589591 },
+    { url = "https://files.pythonhosted.org/packages/10/a2/010b0e27ddeacab7839957d7a8f00e91206e0c2c47abbb5f35a2630e5387/numpy-2.3.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc92a5dedcc53857249ca51ef29f5e5f2f8c513e22cfb90faeb20343b8c6f7a6", size = 16938714 },
+    { url = "https://files.pythonhosted.org/packages/1c/6b/12ce8ede632c7126eb2762b9e15e18e204b81725b81f35176eac14dc5b82/numpy-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7af05ed4dc19f308e1d9fc759f36f21921eb7bbfc82843eeec6b2a2863a0aefa", size = 16370592 },
+    { url = "https://files.pythonhosted.org/packages/b4/35/aba8568b2593067bb6a8fe4c52babb23b4c3b9c80e1b49dff03a09925e4a/numpy-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:433bf137e338677cebdd5beac0199ac84712ad9d630b74eceeb759eaa45ddf30", size = 18884474 },
+    { url = "https://files.pythonhosted.org/packages/45/fa/7f43ba10c77575e8be7b0138d107e4f44ca4a1ef322cd16980ea3e8b8222/numpy-2.3.3-cp311-cp311-win32.whl", hash = "sha256:eb63d443d7b4ffd1e873f8155260d7f58e7e4b095961b01c91062935c2491e57", size = 6599794 },
+    { url = "https://files.pythonhosted.org/packages/0a/a2/a4f78cb2241fe5664a22a10332f2be886dcdea8784c9f6a01c272da9b426/numpy-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:ec9d249840f6a565f58d8f913bccac2444235025bbb13e9a4681783572ee3caa", size = 13088104 },
+    { url = "https://files.pythonhosted.org/packages/79/64/e424e975adbd38282ebcd4891661965b78783de893b381cbc4832fb9beb2/numpy-2.3.3-cp311-cp311-win_arm64.whl", hash = "sha256:74c2a948d02f88c11a3c075d9733f1ae67d97c6bdb97f2bb542f980458b257e7", size = 10460772 },
+    { url = "https://files.pythonhosted.org/packages/51/5d/bb7fc075b762c96329147799e1bcc9176ab07ca6375ea976c475482ad5b3/numpy-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cfdd09f9c84a1a934cde1eec2267f0a43a7cd44b2cca4ff95b7c0d14d144b0bf", size = 20957014 },
+    { url = "https://files.pythonhosted.org/packages/6b/0e/c6211bb92af26517acd52125a237a92afe9c3124c6a68d3b9f81b62a0568/numpy-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cb32e3cf0f762aee47ad1ddc6672988f7f27045b0783c887190545baba73aa25", size = 14185220 },
+    { url = "https://files.pythonhosted.org/packages/22/f2/07bb754eb2ede9073f4054f7c0286b0d9d2e23982e090a80d478b26d35ca/numpy-2.3.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:396b254daeb0a57b1fe0ecb5e3cff6fa79a380fa97c8f7781a6d08cd429418fe", size = 5113918 },
+    { url = "https://files.pythonhosted.org/packages/81/0a/afa51697e9fb74642f231ea36aca80fa17c8fb89f7a82abd5174023c3960/numpy-2.3.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:067e3d7159a5d8f8a0b46ee11148fc35ca9b21f61e3c49fbd0a027450e65a33b", size = 6647922 },
+    { url = "https://files.pythonhosted.org/packages/5d/f5/122d9cdb3f51c520d150fef6e87df9279e33d19a9611a87c0d2cf78a89f4/numpy-2.3.3-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c02d0629d25d426585fb2e45a66154081b9fa677bc92a881ff1d216bc9919a8", size = 14281991 },
+    { url = "https://files.pythonhosted.org/packages/51/64/7de3c91e821a2debf77c92962ea3fe6ac2bc45d0778c1cbe15d4fce2fd94/numpy-2.3.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d9192da52b9745f7f0766531dcfa978b7763916f158bb63bdb8a1eca0068ab20", size = 16641643 },
+    { url = "https://files.pythonhosted.org/packages/30/e4/961a5fa681502cd0d68907818b69f67542695b74e3ceaa513918103b7e80/numpy-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:cd7de500a5b66319db419dc3c345244404a164beae0d0937283b907d8152e6ea", size = 16056787 },
+    { url = "https://files.pythonhosted.org/packages/99/26/92c912b966e47fbbdf2ad556cb17e3a3088e2e1292b9833be1dfa5361a1a/numpy-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:93d4962d8f82af58f0b2eb85daaf1b3ca23fe0a85d0be8f1f2b7bb46034e56d7", size = 18579598 },
+    { url = "https://files.pythonhosted.org/packages/17/b6/fc8f82cb3520768718834f310c37d96380d9dc61bfdaf05fe5c0b7653e01/numpy-2.3.3-cp312-cp312-win32.whl", hash = "sha256:5534ed6b92f9b7dca6c0a19d6df12d41c68b991cef051d108f6dbff3babc4ebf", size = 6320800 },
+    { url = "https://files.pythonhosted.org/packages/32/ee/de999f2625b80d043d6d2d628c07d0d5555a677a3cf78fdf868d409b8766/numpy-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:497d7cad08e7092dba36e3d296fe4c97708c93daf26643a1ae4b03f6294d30eb", size = 12786615 },
+    { url = "https://files.pythonhosted.org/packages/49/6e/b479032f8a43559c383acb20816644f5f91c88f633d9271ee84f3b3a996c/numpy-2.3.3-cp312-cp312-win_arm64.whl", hash = "sha256:ca0309a18d4dfea6fc6262a66d06c26cfe4640c3926ceec90e57791a82b6eee5", size = 10195936 },
+    { url = "https://files.pythonhosted.org/packages/7d/b9/984c2b1ee61a8b803bf63582b4ac4242cf76e2dbd663efeafcb620cc0ccb/numpy-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f5415fb78995644253370985342cd03572ef8620b934da27d77377a2285955bf", size = 20949588 },
+    { url = "https://files.pythonhosted.org/packages/a6/e4/07970e3bed0b1384d22af1e9912527ecbeb47d3b26e9b6a3bced068b3bea/numpy-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d00de139a3324e26ed5b95870ce63be7ec7352171bc69a4cf1f157a48e3eb6b7", size = 14177802 },
+    { url = "https://files.pythonhosted.org/packages/35/c7/477a83887f9de61f1203bad89cf208b7c19cc9fef0cebef65d5a1a0619f2/numpy-2.3.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:9dc13c6a5829610cc07422bc74d3ac083bd8323f14e2827d992f9e52e22cd6a6", size = 5106537 },
+    { url = "https://files.pythonhosted.org/packages/52/47/93b953bd5866a6f6986344d045a207d3f1cfbad99db29f534ea9cee5108c/numpy-2.3.3-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:d79715d95f1894771eb4e60fb23f065663b2298f7d22945d66877aadf33d00c7", size = 6640743 },
+    { url = "https://files.pythonhosted.org/packages/23/83/377f84aaeb800b64c0ef4de58b08769e782edcefa4fea712910b6f0afd3c/numpy-2.3.3-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:952cfd0748514ea7c3afc729a0fc639e61655ce4c55ab9acfab14bda4f402b4c", size = 14278881 },
+    { url = "https://files.pythonhosted.org/packages/9a/a5/bf3db6e66c4b160d6ea10b534c381a1955dfab34cb1017ea93aa33c70ed3/numpy-2.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5b83648633d46f77039c29078751f80da65aa64d5622a3cd62aaef9d835b6c93", size = 16636301 },
+    { url = "https://files.pythonhosted.org/packages/a2/59/1287924242eb4fa3f9b3a2c30400f2e17eb2707020d1c5e3086fe7330717/numpy-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b001bae8cea1c7dfdb2ae2b017ed0a6f2102d7a70059df1e338e307a4c78a8ae", size = 16053645 },
+    { url = "https://files.pythonhosted.org/packages/e6/93/b3d47ed882027c35e94ac2320c37e452a549f582a5e801f2d34b56973c97/numpy-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8e9aced64054739037d42fb84c54dd38b81ee238816c948c8f3ed134665dcd86", size = 18578179 },
+    { url = "https://files.pythonhosted.org/packages/20/d9/487a2bccbf7cc9d4bfc5f0f197761a5ef27ba870f1e3bbb9afc4bbe3fcc2/numpy-2.3.3-cp313-cp313-win32.whl", hash = "sha256:9591e1221db3f37751e6442850429b3aabf7026d3b05542d102944ca7f00c8a8", size = 6312250 },
+    { url = "https://files.pythonhosted.org/packages/1b/b5/263ebbbbcede85028f30047eab3d58028d7ebe389d6493fc95ae66c636ab/numpy-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f0dadeb302887f07431910f67a14d57209ed91130be0adea2f9793f1a4f817cf", size = 12783269 },
+    { url = "https://files.pythonhosted.org/packages/fa/75/67b8ca554bbeaaeb3fac2e8bce46967a5a06544c9108ec0cf5cece559b6c/numpy-2.3.3-cp313-cp313-win_arm64.whl", hash = "sha256:3c7cf302ac6e0b76a64c4aecf1a09e51abd9b01fc7feee80f6c43e3ab1b1dbc5", size = 10195314 },
+    { url = "https://files.pythonhosted.org/packages/11/d0/0d1ddec56b162042ddfafeeb293bac672de9b0cfd688383590090963720a/numpy-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:eda59e44957d272846bb407aad19f89dc6f58fecf3504bd144f4c5cf81a7eacc", size = 21048025 },
+    { url = "https://files.pythonhosted.org/packages/36/9e/1996ca6b6d00415b6acbdd3c42f7f03ea256e2c3f158f80bd7436a8a19f3/numpy-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:823d04112bc85ef5c4fda73ba24e6096c8f869931405a80aa8b0e604510a26bc", size = 14301053 },
+    { url = "https://files.pythonhosted.org/packages/05/24/43da09aa764c68694b76e84b3d3f0c44cb7c18cdc1ba80e48b0ac1d2cd39/numpy-2.3.3-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:40051003e03db4041aa325da2a0971ba41cf65714e65d296397cc0e32de6018b", size = 5229444 },
+    { url = "https://files.pythonhosted.org/packages/bc/14/50ffb0f22f7218ef8af28dd089f79f68289a7a05a208db9a2c5dcbe123c1/numpy-2.3.3-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:6ee9086235dd6ab7ae75aba5662f582a81ced49f0f1c6de4260a78d8f2d91a19", size = 6738039 },
+    { url = "https://files.pythonhosted.org/packages/55/52/af46ac0795e09657d45a7f4db961917314377edecf66db0e39fa7ab5c3d3/numpy-2.3.3-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94fcaa68757c3e2e668ddadeaa86ab05499a70725811e582b6a9858dd472fb30", size = 14352314 },
+    { url = "https://files.pythonhosted.org/packages/a7/b1/dc226b4c90eb9f07a3fff95c2f0db3268e2e54e5cce97c4ac91518aee71b/numpy-2.3.3-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da1a74b90e7483d6ce5244053399a614b1d6b7bc30a60d2f570e5071f8959d3e", size = 16701722 },
+    { url = "https://files.pythonhosted.org/packages/9d/9d/9d8d358f2eb5eced14dba99f110d83b5cd9a4460895230f3b396ad19a323/numpy-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2990adf06d1ecee3b3dcbb4977dfab6e9f09807598d647f04d385d29e7a3c3d3", size = 16132755 },
+    { url = "https://files.pythonhosted.org/packages/b6/27/b3922660c45513f9377b3fb42240bec63f203c71416093476ec9aa0719dc/numpy-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ed635ff692483b8e3f0fcaa8e7eb8a75ee71aa6d975388224f70821421800cea", size = 18651560 },
+    { url = "https://files.pythonhosted.org/packages/5b/8e/3ab61a730bdbbc201bb245a71102aa609f0008b9ed15255500a99cd7f780/numpy-2.3.3-cp313-cp313t-win32.whl", hash = "sha256:a333b4ed33d8dc2b373cc955ca57babc00cd6f9009991d9edc5ddbc1bac36bcd", size = 6442776 },
+    { url = "https://files.pythonhosted.org/packages/1c/3a/e22b766b11f6030dc2decdeff5c2fb1610768055603f9f3be88b6d192fb2/numpy-2.3.3-cp313-cp313t-win_amd64.whl", hash = "sha256:4384a169c4d8f97195980815d6fcad04933a7e1ab3b530921c3fef7a1c63426d", size = 12927281 },
+    { url = "https://files.pythonhosted.org/packages/7b/42/c2e2bc48c5e9b2a83423f99733950fbefd86f165b468a3d85d52b30bf782/numpy-2.3.3-cp313-cp313t-win_arm64.whl", hash = "sha256:75370986cc0bc66f4ce5110ad35aae6d182cc4ce6433c40ad151f53690130bf1", size = 10265275 },
+    { url = "https://files.pythonhosted.org/packages/6b/01/342ad585ad82419b99bcf7cebe99e61da6bedb89e213c5fd71acc467faee/numpy-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:cd052f1fa6a78dee696b58a914b7229ecfa41f0a6d96dc663c1220a55e137593", size = 20951527 },
+    { url = "https://files.pythonhosted.org/packages/ef/d8/204e0d73fc1b7a9ee80ab1fe1983dd33a4d64a4e30a05364b0208e9a241a/numpy-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:414a97499480067d305fcac9716c29cf4d0d76db6ebf0bf3cbce666677f12652", size = 14186159 },
+    { url = "https://files.pythonhosted.org/packages/22/af/f11c916d08f3a18fb8ba81ab72b5b74a6e42ead4c2846d270eb19845bf74/numpy-2.3.3-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:50a5fe69f135f88a2be9b6ca0481a68a136f6febe1916e4920e12f1a34e708a7", size = 5114624 },
+    { url = "https://files.pythonhosted.org/packages/fb/11/0ed919c8381ac9d2ffacd63fd1f0c34d27e99cab650f0eb6f110e6ae4858/numpy-2.3.3-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:b912f2ed2b67a129e6a601e9d93d4fa37bef67e54cac442a2f588a54afe5c67a", size = 6642627 },
+    { url = "https://files.pythonhosted.org/packages/ee/83/deb5f77cb0f7ba6cb52b91ed388b47f8f3c2e9930d4665c600408d9b90b9/numpy-2.3.3-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9e318ee0596d76d4cb3d78535dc005fa60e5ea348cd131a51e99d0bdbe0b54fe", size = 14296926 },
+    { url = "https://files.pythonhosted.org/packages/77/cc/70e59dcb84f2b005d4f306310ff0a892518cc0c8000a33d0e6faf7ca8d80/numpy-2.3.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ce020080e4a52426202bdb6f7691c65bb55e49f261f31a8f506c9f6bc7450421", size = 16638958 },
+    { url = "https://files.pythonhosted.org/packages/b6/5a/b2ab6c18b4257e099587d5b7f903317bd7115333ad8d4ec4874278eafa61/numpy-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e6687dc183aa55dae4a705b35f9c0f8cb178bcaa2f029b241ac5356221d5c021", size = 16071920 },
+    { url = "https://files.pythonhosted.org/packages/b8/f1/8b3fdc44324a259298520dd82147ff648979bed085feeacc1250ef1656c0/numpy-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d8f3b1080782469fdc1718c4ed1d22549b5fb12af0d57d35e992158a772a37cf", size = 18577076 },
+    { url = "https://files.pythonhosted.org/packages/f0/a1/b87a284fb15a42e9274e7fcea0dad259d12ddbf07c1595b26883151ca3b4/numpy-2.3.3-cp314-cp314-win32.whl", hash = "sha256:cb248499b0bc3be66ebd6578b83e5acacf1d6cb2a77f2248ce0e40fbec5a76d0", size = 6366952 },
+    { url = "https://files.pythonhosted.org/packages/70/5f/1816f4d08f3b8f66576d8433a66f8fa35a5acfb3bbd0bf6c31183b003f3d/numpy-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:691808c2b26b0f002a032c73255d0bd89751425f379f7bcd22d140db593a96e8", size = 12919322 },
+    { url = "https://files.pythonhosted.org/packages/8c/de/072420342e46a8ea41c324a555fa90fcc11637583fb8df722936aed1736d/numpy-2.3.3-cp314-cp314-win_arm64.whl", hash = "sha256:9ad12e976ca7b10f1774b03615a2a4bab8addce37ecc77394d8e986927dc0dfe", size = 10478630 },
+    { url = "https://files.pythonhosted.org/packages/d5/df/ee2f1c0a9de7347f14da5dd3cd3c3b034d1b8607ccb6883d7dd5c035d631/numpy-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9cc48e09feb11e1db00b320e9d30a4151f7369afb96bd0e48d942d09da3a0d00", size = 21047987 },
+    { url = "https://files.pythonhosted.org/packages/d6/92/9453bdc5a4e9e69cf4358463f25e8260e2ffc126d52e10038b9077815989/numpy-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:901bf6123879b7f251d3631967fd574690734236075082078e0571977c6a8e6a", size = 14301076 },
+    { url = "https://files.pythonhosted.org/packages/13/77/1447b9eb500f028bb44253105bd67534af60499588a5149a94f18f2ca917/numpy-2.3.3-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:7f025652034199c301049296b59fa7d52c7e625017cae4c75d8662e377bf487d", size = 5229491 },
+    { url = "https://files.pythonhosted.org/packages/3d/f9/d72221b6ca205f9736cb4b2ce3b002f6e45cd67cd6a6d1c8af11a2f0b649/numpy-2.3.3-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:533ca5f6d325c80b6007d4d7fb1984c303553534191024ec6a524a4c92a5935a", size = 6737913 },
+    { url = "https://files.pythonhosted.org/packages/3c/5f/d12834711962ad9c46af72f79bb31e73e416ee49d17f4c797f72c96b6ca5/numpy-2.3.3-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0edd58682a399824633b66885d699d7de982800053acf20be1eaa46d92009c54", size = 14352811 },
+    { url = "https://files.pythonhosted.org/packages/a1/0d/fdbec6629d97fd1bebed56cd742884e4eead593611bbe1abc3eb40d304b2/numpy-2.3.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:367ad5d8fbec5d9296d18478804a530f1191e24ab4d75ab408346ae88045d25e", size = 16702689 },
+    { url = "https://files.pythonhosted.org/packages/9b/09/0a35196dc5575adde1eb97ddfbc3e1687a814f905377621d18ca9bc2b7dd/numpy-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8f6ac61a217437946a1fa48d24c47c91a0c4f725237871117dea264982128097", size = 16133855 },
+    { url = "https://files.pythonhosted.org/packages/7a/ca/c9de3ea397d576f1b6753eaa906d4cdef1bf97589a6d9825a349b4729cc2/numpy-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:179a42101b845a816d464b6fe9a845dfaf308fdfc7925387195570789bb2c970", size = 18652520 },
+    { url = "https://files.pythonhosted.org/packages/fd/c2/e5ed830e08cd0196351db55db82f65bc0ab05da6ef2b72a836dcf1936d2f/numpy-2.3.3-cp314-cp314t-win32.whl", hash = "sha256:1250c5d3d2562ec4174bce2e3a1523041595f9b651065e4a4473f5f48a6bc8a5", size = 6515371 },
+    { url = "https://files.pythonhosted.org/packages/47/c7/b0f6b5b67f6788a0725f744496badbb604d226bf233ba716683ebb47b570/numpy-2.3.3-cp314-cp314t-win_amd64.whl", hash = "sha256:b37a0b2e5935409daebe82c1e42274d30d9dd355852529eab91dab8dcca7419f", size = 13112576 },
+    { url = "https://files.pythonhosted.org/packages/06/b9/33bba5ff6fb679aa0b1f8a07e853f002a6b04b9394db3069a1270a7784ca/numpy-2.3.3-cp314-cp314t-win_arm64.whl", hash = "sha256:78c9f6560dc7e6b3990e32df7ea1a50bbd0e2a111e05209963f5ddcab7073b0b", size = 10545953 },
+    { url = "https://files.pythonhosted.org/packages/b8/f2/7e0a37cfced2644c9563c529f29fa28acbd0960dde32ece683aafa6f4949/numpy-2.3.3-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1e02c7159791cd481e1e6d5ddd766b62a4d5acf8df4d4d1afe35ee9c5c33a41e", size = 21131019 },
+    { url = "https://files.pythonhosted.org/packages/1a/7e/3291f505297ed63831135a6cc0f474da0c868a1f31b0dd9a9f03a7a0d2ed/numpy-2.3.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:dca2d0fc80b3893ae72197b39f69d55a3cd8b17ea1b50aa4c62de82419936150", size = 14376288 },
+    { url = "https://files.pythonhosted.org/packages/bf/4b/ae02e985bdeee73d7b5abdefeb98aef1207e96d4c0621ee0cf228ddfac3c/numpy-2.3.3-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:99683cbe0658f8271b333a1b1b4bb3173750ad59c0c61f5bbdc5b318918fffe3", size = 5305425 },
+    { url = "https://files.pythonhosted.org/packages/8b/eb/9df215d6d7250db32007941500dc51c48190be25f2401d5b2b564e467247/numpy-2.3.3-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:d9d537a39cc9de668e5cd0e25affb17aec17b577c6b3ae8a3d866b479fbe88d0", size = 6819053 },
+    { url = "https://files.pythonhosted.org/packages/57/62/208293d7d6b2a8998a4a1f23ac758648c3c32182d4ce4346062018362e29/numpy-2.3.3-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8596ba2f8af5f93b01d97563832686d20206d303024777f6dfc2e7c7c3f1850e", size = 14420354 },
+    { url = "https://files.pythonhosted.org/packages/ed/0c/8e86e0ff7072e14a71b4c6af63175e40d1e7e933ce9b9e9f765a95b4e0c3/numpy-2.3.3-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1ec5615b05369925bd1125f27df33f3b6c8bc10d788d5999ecd8769a1fa04db", size = 16760413 },
+    { url = "https://files.pythonhosted.org/packages/af/11/0cc63f9f321ccf63886ac203336777140011fb669e739da36d8db3c53b98/numpy-2.3.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2e267c7da5bf7309670523896df97f93f6e469fb931161f483cd6882b3b1a5dc", size = 12971844 },
 ]
 
 [[package]]
@@ -3691,7 +3872,9 @@ name = "pandas"
 version = "2.2.3"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "numpy" },
+    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
+    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
     { name = "python-dateutil" },
     { name = "pytz" },
     { name = "tzdata" },
@@ -5085,7 +5268,7 @@ resolution-markers = [
 ]
 dependencies = [
     { name = "joblib", marker = "python_full_version < '3.10'" },
-    { name = "numpy", marker = "python_full_version < '3.10'" },
+    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
     { name = "scipy", version = "1.13.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
     { name = "threadpoolctl", marker = "python_full_version < '3.10'" },
 ]
@@ -5133,7 +5316,8 @@ resolution-markers = [
 ]
 dependencies = [
     { name = "joblib", marker = "python_full_version >= '3.10'" },
-    { name = "numpy", marker = "python_full_version >= '3.10'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
+    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
     { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
     { name = "scipy", version = "1.16.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
     { name = "threadpoolctl", marker = "python_full_version >= '3.10'" },
@@ -5180,7 +5364,7 @@ resolution-markers = [
     "python_full_version < '3.10'",
 ]
 dependencies = [
-    { name = "numpy", marker = "python_full_version < '3.10'" },
+    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/ae/00/48c2f661e2816ccf2ecd77982f6605b2950afe60f60a52b4cbbc2504aa8f/scipy-1.13.1.tar.gz", hash = "sha256:095a87a0312b08dfd6a6155cbbd310a8c51800fc931b8c0b84003014b874ed3c", size = 57210720 }
 wheels = [
@@ -5218,7 +5402,7 @@ resolution-markers = [
     "python_full_version == '3.10.*'",
 ]
 dependencies = [
-    { name = "numpy", marker = "python_full_version == '3.10.*'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/0f/37/6964b830433e654ec7485e45a00fc9a27cf868d622838f6b6d9c5ec0d532/scipy-1.15.3.tar.gz", hash = "sha256:eae3cf522bc7df64b42cad3925c876e1b0b6c35c1337c93e12c0f366f55b0eaf", size = 59419214 }
 wheels = [
@@ -5278,7 +5462,7 @@ resolution-markers = [
     "python_full_version == '3.11.*'",
 ]
 dependencies = [
-    { name = "numpy", marker = "python_full_version >= '3.11'" },
+    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/4c/3b/546a6f0bfe791bbb7f8d591613454d15097e53f906308ec6f7c1ce588e8e/scipy-1.16.2.tar.gz", hash = "sha256:af029b153d243a80afb6eabe40b0a07f8e35c9adc269c019f364ad747f826a6b", size = 30580599 }
 wheels = [
@@ -5479,7 +5663,7 @@ resolution-markers = [
 dependencies = [
     { name = "aiohttp", marker = "python_full_version < '3.10'" },
     { name = "ipython", version = "8.18.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
-    { name = "numpy", marker = "python_full_version < '3.10'" },
+    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
     { name = "requests", marker = "python_full_version < '3.10'" },
     { name = "setproctitle", marker = "python_full_version < '3.10'" },
     { name = "tqdm", marker = "python_full_version < '3.10'" },
@@ -5502,7 +5686,8 @@ dependencies = [
     { name = "aiohttp", marker = "python_full_version >= '3.10'" },
     { name = "ipython", version = "8.37.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
     { name = "ipython", version = "9.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
-    { name = "numpy", marker = "python_full_version >= '3.10'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
+    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
     { name = "requests", marker = "python_full_version >= '3.10'" },
     { name = "setproctitle", marker = "python_full_version >= '3.10'" },
     { name = "tqdm", marker = "python_full_version >= '3.10'" },
@@ -5715,75 +5900,27 @@ wheels = [
 
 [[package]]
 name = "tokenizers"
-version = "0.19.1"
+version = "0.22.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "huggingface-hub" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/48/04/2071c150f374aab6d5e92aaec38d0f3c368d227dd9e0469a1f0966ac68d1/tokenizers-0.19.1.tar.gz", hash = "sha256:ee59e6680ed0fdbe6b724cf38bd70400a0c1dd623b07ac729087270caeac88e3", size = 321039 }
+sdist = { url = "https://files.pythonhosted.org/packages/1c/46/fb6854cec3278fbfa4a75b50232c77622bc517ac886156e6afbfa4d8fc6e/tokenizers-0.22.1.tar.gz", hash = "sha256:61de6522785310a309b3407bac22d99c4db5dba349935e99e4d15ea2226af2d9", size = 363123 }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/c1/60/91cac8d496b304ec5a22f07606893cad35ea8e1a8406dc8909e365f97a80/tokenizers-0.19.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:952078130b3d101e05ecfc7fc3640282d74ed26bcf691400f872563fca15ac97", size = 2533301 },
-    { url = "https://files.pythonhosted.org/packages/4c/12/9cb68762ff5fee1efd51aefe2f62cb225f26f060a68a3779e1060bbc7a59/tokenizers-0.19.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:82c8b8063de6c0468f08e82c4e198763e7b97aabfe573fd4cf7b33930ca4df77", size = 2440223 },
-    { url = "https://files.pythonhosted.org/packages/e4/03/b2020e6a78fb994cff1ec962adc157c23109172a46b4fe451d6d0dd33fdb/tokenizers-0.19.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f03727225feaf340ceeb7e00604825addef622d551cbd46b7b775ac834c1e1c4", size = 3683779 },
-    { url = "https://files.pythonhosted.org/packages/50/4e/2e5549a26dc6f9e434f83bebf16c2d7dc9dc3477cc0ec8b23ede4d465b90/tokenizers-0.19.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:453e4422efdfc9c6b6bf2eae00d5e323f263fff62b29a8c9cd526c5003f3f642", size = 3569431 },
-    { url = "https://files.pythonhosted.org/packages/75/79/158626bd794e75551e0c6bb93f1cd3c9ba08ba14b181b98f09e95994f609/tokenizers-0.19.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:02e81bf089ebf0e7f4df34fa0207519f07e66d8491d963618252f2e0729e0b46", size = 3424739 },
-    { url = "https://files.pythonhosted.org/packages/65/8e/5f4316976c26009f1ae0b6543f3d97af29afa5ba5dc145251e6a07314618/tokenizers-0.19.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b07c538ba956843833fee1190cf769c60dc62e1cf934ed50d77d5502194d63b1", size = 3965791 },
-    { url = "https://files.pythonhosted.org/packages/6a/e1/5dbac9618709972434eea072670cd69fba1aa988e6200f16057722b4bf96/tokenizers-0.19.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e28cab1582e0eec38b1f38c1c1fb2e56bce5dc180acb1724574fc5f47da2a4fe", size = 4049879 },
-    { url = "https://files.pythonhosted.org/packages/40/4f/eb78de4af3b17b589f43a369cbf0c3a7173f25c3d2cd93068852c07689aa/tokenizers-0.19.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b01afb7193d47439f091cd8f070a1ced347ad0f9144952a30a41836902fe09e", size = 3607049 },
-    { url = "https://files.pythonhosted.org/packages/f5/f8/141dcb0f88e9452af8d20d14dd53aab5937222a2bb4f2c04bfed6829263c/tokenizers-0.19.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7fb297edec6c6841ab2e4e8f357209519188e4a59b557ea4fafcf4691d1b4c98", size = 9634084 },
-    { url = "https://files.pythonhosted.org/packages/2e/be/debb7caa3f88ed54015170db16e07aa3a5fea2d3983d0dde92f98d888dc8/tokenizers-0.19.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2e8a3dd055e515df7054378dc9d6fa8c8c34e1f32777fb9a01fea81496b3f9d3", size = 9949480 },
-    { url = "https://files.pythonhosted.org/packages/7a/e7/26bedf5d270d293d572a90bd66b0b030012aedb95d8ee87e8bcd446b76fb/tokenizers-0.19.1-cp310-none-win32.whl", hash = "sha256:7ff898780a155ea053f5d934925f3902be2ed1f4d916461e1a93019cc7250837", size = 2041462 },
-    { url = "https://files.pythonhosted.org/packages/f4/85/d999b9a05fd101d48f1a365d68be0b109277bb25c89fb37a389d669f9185/tokenizers-0.19.1-cp310-none-win_amd64.whl", hash = "sha256:bea6f9947e9419c2fda21ae6c32871e3d398cba549b93f4a65a2d369662d9403", size = 2220036 },
-    { url = "https://files.pythonhosted.org/packages/c8/d6/6e1d728d765eb4102767f071bf7f6439ab10d7f4a975c9217db65715207a/tokenizers-0.19.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:5c88d1481f1882c2e53e6bb06491e474e420d9ac7bdff172610c4f9ad3898059", size = 2533448 },
-    { url = "https://files.pythonhosted.org/packages/90/79/d17a0f491d10817cd30f1121a07aa09c8e97a81114b116e473baf1577f09/tokenizers-0.19.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ddf672ed719b4ed82b51499100f5417d7d9f6fb05a65e232249268f35de5ed14", size = 2440254 },
-    { url = "https://files.pythonhosted.org/packages/c7/28/2d11c3ff94f9d42eceb2ea549a06e3f166fe391c5a025e5d96fac898a3ac/tokenizers-0.19.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:dadc509cc8a9fe460bd274c0e16ac4184d0958117cf026e0ea8b32b438171594", size = 3684971 },
-    { url = "https://files.pythonhosted.org/packages/36/c6/537f22b57e6003904d35d07962dbde2f2e9bdd791d0241da976a4c7f8194/tokenizers-0.19.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfedf31824ca4915b511b03441784ff640378191918264268e6923da48104acc", size = 3568894 },
-    { url = "https://files.pythonhosted.org/packages/af/ef/3c1deed14ec59b2c8e7e2fa27b2a53f7d101181277a43b89ab17d891ef2e/tokenizers-0.19.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac11016d0a04aa6487b1513a3a36e7bee7eec0e5d30057c9c0408067345c48d2", size = 3426873 },
-    { url = "https://files.pythonhosted.org/packages/06/db/c0320c4798ac6bd12d2ef895bec9d10d216a3b4d6fff10e9d68883ea7edc/tokenizers-0.19.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:76951121890fea8330d3a0df9a954b3f2a37e3ec20e5b0530e9a0044ca2e11fe", size = 3965050 },
-    { url = "https://files.pythonhosted.org/packages/4c/8a/a166888d6cb14db55f5eb7ce0b1d4777d145aa27cbf4f945712cf6c29935/tokenizers-0.19.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b342d2ce8fc8d00f376af068e3274e2e8649562e3bc6ae4a67784ded6b99428d", size = 4047855 },
-    { url = "https://files.pythonhosted.org/packages/a7/03/fb50fc03f86016b227a967c8d474f90230c885c0d18f78acdfda7a96ce56/tokenizers-0.19.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d16ff18907f4909dca9b076b9c2d899114dd6abceeb074eca0c93e2353f943aa", size = 3608228 },
-    { url = "https://files.pythonhosted.org/packages/5b/cd/0385e1026e1e03732fd398e964792a3a8433918b166748c82507e014d748/tokenizers-0.19.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:706a37cc5332f85f26efbe2bdc9ef8a9b372b77e4645331a405073e4b3a8c1c6", size = 9633115 },
-    { url = "https://files.pythonhosted.org/packages/25/50/8f8ad0bbdaf09d04b15e6502d1fa1c653754ed7e016e4ae009726aa1a4e4/tokenizers-0.19.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:16baac68651701364b0289979ecec728546133e8e8fe38f66fe48ad07996b88b", size = 9949062 },
-    { url = "https://files.pythonhosted.org/packages/db/11/31be66710f1d14526f3588a441efadeb184e1e68458067007b20ead03c59/tokenizers-0.19.1-cp311-none-win32.whl", hash = "sha256:9ed240c56b4403e22b9584ee37d87b8bfa14865134e3e1c3fb4b2c42fafd3256", size = 2041039 },
-    { url = "https://files.pythonhosted.org/packages/65/8e/6d7d72b28f22c422cff8beae10ac3c2e4376b9be721ef8167b7eecd1da62/tokenizers-0.19.1-cp311-none-win_amd64.whl", hash = "sha256:ad57d59341710b94a7d9dbea13f5c1e7d76fd8d9bcd944a7a6ab0b0da6e0cc66", size = 2220386 },
-    { url = "https://files.pythonhosted.org/packages/63/90/2890cd096898dcdb596ee172cde40c0f54a9cf43b0736aa260a5501252af/tokenizers-0.19.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:621d670e1b1c281a1c9698ed89451395d318802ff88d1fc1accff0867a06f153", size = 2530580 },
-    { url = "https://files.pythonhosted.org/packages/74/d1/f4e1e950adb36675dfd8f9d0f4be644f3f3aaf22a5677a4f5c81282b662e/tokenizers-0.19.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d924204a3dbe50b75630bd16f821ebda6a5f729928df30f582fb5aade90c818a", size = 2436682 },
-    { url = "https://files.pythonhosted.org/packages/ed/30/89b321a16c58d233e301ec15072c0d3ed5014825e72da98604cd3ab2fba1/tokenizers-0.19.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:4f3fefdc0446b1a1e6d81cd4c07088ac015665d2e812f6dbba4a06267d1a2c95", size = 3693494 },
-    { url = "https://files.pythonhosted.org/packages/05/40/fa899f32de483500fbc78befd378fd7afba4270f17db707d1a78c0a4ddc3/tokenizers-0.19.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9620b78e0b2d52ef07b0d428323fb34e8ea1219c5eac98c2596311f20f1f9266", size = 3566541 },
-    { url = "https://files.pythonhosted.org/packages/67/14/e7da32ae5fb4971830f1ef335932fae3fa57e76b537e852f146c850aefdf/tokenizers-0.19.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:04ce49e82d100594715ac1b2ce87d1a36e61891a91de774755f743babcd0dd52", size = 3430792 },
-    { url = "https://files.pythonhosted.org/packages/f2/4b/aae61bdb6ab584d2612170801703982ee0e35f8b6adacbeefe5a3b277621/tokenizers-0.19.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c5c2ff13d157afe413bf7e25789879dd463e5a4abfb529a2d8f8473d8042e28f", size = 3962812 },
-    { url = "https://files.pythonhosted.org/packages/0a/b6/f7b7ef89c4da7b20256e6eab23d3835f05d1ca8f451d31c16cbfe3cd9eb6/tokenizers-0.19.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3174c76efd9d08f836bfccaca7cfec3f4d1c0a4cf3acbc7236ad577cc423c840", size = 4024688 },
-    { url = "https://files.pythonhosted.org/packages/80/54/12047a69f5b382d7ee72044dc89151a2dd0d13b2c9bdcc22654883704d31/tokenizers-0.19.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7c9d5b6c0e7a1e979bec10ff960fae925e947aab95619a6fdb4c1d8ff3708ce3", size = 3610961 },
-    { url = "https://files.pythonhosted.org/packages/52/b7/1e8a913d18ac28feeda42d4d2d51781874398fb59cd1c1e2653a4b5742ed/tokenizers-0.19.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a179856d1caee06577220ebcfa332af046d576fb73454b8f4d4b0ba8324423ea", size = 9631367 },
-    { url = "https://files.pythonhosted.org/packages/ac/3d/2284f6d99f8f21d09352b88b8cfefa24ab88468d962aeb0aa15c20d76b32/tokenizers-0.19.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:952b80dac1a6492170f8c2429bd11fcaa14377e097d12a1dbe0ef2fb2241e16c", size = 9950121 },
-    { url = "https://files.pythonhosted.org/packages/2a/94/ec3369dbc9b7200c14c8c7a1a04c78b7a7398d0c001e1b7d1ffe30eb93a0/tokenizers-0.19.1-cp312-none-win32.whl", hash = "sha256:01d62812454c188306755c94755465505836fd616f75067abcae529c35edeb57", size = 2044069 },
-    { url = "https://files.pythonhosted.org/packages/0c/97/80bff6937e0c67d30c0facacd4f0bcf4254e581aa4995c73cef8c8640e56/tokenizers-0.19.1-cp312-none-win_amd64.whl", hash = "sha256:b70bfbe3a82d3e3fb2a5e9b22a39f8d1740c96c68b6ace0086b39074f08ab89a", size = 2214527 },
-    { url = "https://files.pythonhosted.org/packages/1a/ed/42801618bab16c79d6bd222977c212dba5770e6c935ba53728b731653a3d/tokenizers-0.19.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:0b9394bd204842a2a1fd37fe29935353742be4a3460b6ccbaefa93f58a8df43d", size = 2533937 },
-    { url = "https://files.pythonhosted.org/packages/0a/2b/4e5718e806ff23e5e758e02bd4b34967b5218f085b0c189335fd27c14dc1/tokenizers-0.19.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4692ab92f91b87769d950ca14dbb61f8a9ef36a62f94bad6c82cc84a51f76f6a", size = 2440312 },
-    { url = "https://files.pythonhosted.org/packages/c5/28/ac2a277bd23b631e1ff986182c4fcb9028ccc7ff7c07743ef906fa5389e7/tokenizers-0.19.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6258c2ef6f06259f70a682491c78561d492e885adeaf9f64f5389f78aa49a051", size = 3686532 },
-    { url = "https://files.pythonhosted.org/packages/ba/26/139bd2371228a0e203da7b3e3eddcb02f45b2b7edd91df00e342e4b55e13/tokenizers-0.19.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c85cf76561fbd01e0d9ea2d1cbe711a65400092bc52b5242b16cfd22e51f0c58", size = 3570575 },
-    { url = "https://files.pythonhosted.org/packages/3b/6b/98383dff29416127c73dc196844ed23e29d790f1ad4b4ecf69d45e03841d/tokenizers-0.19.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:670b802d4d82bbbb832ddb0d41df7015b3e549714c0e77f9bed3e74d42400fbe", size = 3425806 },
-    { url = "https://files.pythonhosted.org/packages/33/74/fa1f86d161db482e10c92d83e924600b691210c5d676fa323738c91d8dba/tokenizers-0.19.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:85aa3ab4b03d5e99fdd31660872249df5e855334b6c333e0bc13032ff4469c4a", size = 3965120 },
-    { url = "https://files.pythonhosted.org/packages/e0/4a/59a0aa37b8bfe1888a72f75bbf24b94c888a1aa333aab2910ae22c369e23/tokenizers-0.19.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cbf001afbbed111a79ca47d75941e9e5361297a87d186cbfc11ed45e30b5daba", size = 4048157 },
-    { url = "https://files.pythonhosted.org/packages/0f/cb/8fc733c8f251bac1e5c4ae52458c353b3faa98f41d734c226cad3783da03/tokenizers-0.19.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4c89aa46c269e4e70c4d4f9d6bc644fcc39bb409cb2a81227923404dd6f5227", size = 3608229 },
-    { url = "https://files.pythonhosted.org/packages/76/05/badd3a66571ad257270b38c33b9a7470afd2ae12e409c7c74baedf16f2ef/tokenizers-0.19.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:39c1ec76ea1027438fafe16ecb0fb84795e62e9d643444c1090179e63808c69d", size = 9634933 },
-    { url = "https://files.pythonhosted.org/packages/d9/46/97f8e84ba6a9133e34b148631d2933fda2a6ad8e0767b6e07ad0af9d83c2/tokenizers-0.19.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c2a0d47a89b48d7daa241e004e71fb5a50533718897a4cd6235cb846d511a478", size = 9950957 },
-    { url = "https://files.pythonhosted.org/packages/81/b2/bf9a0f9136964df5e94dd9854ba071480c5425ff0db6d1ad9a6a8e683d55/tokenizers-0.19.1-cp39-none-win32.whl", hash = "sha256:61b7fe8886f2e104d4caf9218b157b106207e0f2a4905c9c7ac98890688aabeb", size = 2040628 },
-    { url = "https://files.pythonhosted.org/packages/25/aa/c6992cdc0a74bcbb666e7c00ada6826f5b49fc4cbdafc50db0d1369503fe/tokenizers-0.19.1-cp39-none-win_amd64.whl", hash = "sha256:f97660f6c43efd3e0bfd3f2e3e5615bf215680bad6ee3d469df6454b8c6e8256", size = 2220919 },
-    { url = "https://files.pythonhosted.org/packages/cf/7b/38fb7207cde3d1dc5272411cd18178e6437cdc1ef08cac5d0e8cfd57f38c/tokenizers-0.19.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3b11853f17b54c2fe47742c56d8a33bf49ce31caf531e87ac0d7d13d327c9334", size = 2532668 },
-    { url = "https://files.pythonhosted.org/packages/1d/0d/2c452fe17fc17f0cdb713acb811eebb1f714b8c21d497c4672af4f491229/tokenizers-0.19.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d26194ef6c13302f446d39972aaa36a1dda6450bc8949f5eb4c27f51191375bd", size = 2438321 },
-    { url = "https://files.pythonhosted.org/packages/19/e0/f9e915d028b45798723eab59c253da28040aa66b9f31dcb7cfc3be88fa37/tokenizers-0.19.1-pp310-pypy310_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e8d1ed93beda54bbd6131a2cb363a576eac746d5c26ba5b7556bc6f964425594", size = 3682304 },
-    { url = "https://files.pythonhosted.org/packages/ce/2b/db8a94608c392752681c2ca312487b7cd5bcc4f77e24a90daa4916138271/tokenizers-0.19.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca407133536f19bdec44b3da117ef0d12e43f6d4b56ac4c765f37eca501c7bda", size = 3566208 },
-    { url = "https://files.pythonhosted.org/packages/d8/58/2e998462677c4c0eb5123ce386bcb488a155664d273d0283122866515f09/tokenizers-0.19.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce05fde79d2bc2e46ac08aacbc142bead21614d937aac950be88dc79f9db9022", size = 3605791 },
-    { url = "https://files.pythonhosted.org/packages/83/ac/26bc2e2bb2a054dc2e51699628936f5474e093b68da6ccdde04b2fc39ab8/tokenizers-0.19.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:35583cd46d16f07c054efd18b5d46af4a2f070a2dd0a47914e66f3ff5efb2b1e", size = 9632867 },
-    { url = "https://files.pythonhosted.org/packages/45/b6/36c1bb106bbe96012c9367df89ed01599cada036c0b96d38fbbdbeb75c9f/tokenizers-0.19.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:43350270bfc16b06ad3f6f07eab21f089adb835544417afda0f83256a8bf8b75", size = 9945103 },
-    { url = "https://files.pythonhosted.org/packages/aa/9c/deed1e549b767832cc4ee5b386d1660bde3408bbd6d1ab48352fb61c54e2/tokenizers-0.19.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:56ae39d4036b753994476a1b935584071093b55c7a72e3b8288e68c313ca26e7", size = 2533737 },
-    { url = "https://files.pythonhosted.org/packages/c8/59/4dbebca9ef6b61d10a94cbf404d3abf509dfedb52cdcf2fe7ed1fb52460d/tokenizers-0.19.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:f9939ca7e58c2758c01b40324a59c034ce0cebad18e0d4563a9b1beab3018243", size = 2439981 },
-    { url = "https://files.pythonhosted.org/packages/72/42/e18b67ab9fd31e433171cf447d85bf5dede8009db04a46f3905bff5ca715/tokenizers-0.19.1-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6c330c0eb815d212893c67a032e9dc1b38a803eccb32f3e8172c19cc69fbb439", size = 3683158 },
-    { url = "https://files.pythonhosted.org/packages/08/5c/54419545d61c085d7adcbd54f5711815ffbb1164d6132209172c984320be/tokenizers-0.19.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec11802450a2487cdf0e634b750a04cbdc1c4d066b97d94ce7dd2cb51ebb325b", size = 3568486 },
-    { url = "https://files.pythonhosted.org/packages/6d/61/f8b59cc2580297ca78a7b5b2cefc8996b8417dc6cb9abb6a1d303973156b/tokenizers-0.19.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a2b718f316b596f36e1dae097a7d5b91fc5b85e90bf08b01ff139bd8953b25af", size = 3608836 },
-    { url = "https://files.pythonhosted.org/packages/a5/02/4944f51c7248ae78c9758266f4e92d72fe98cf58f3c973949bcdede0313a/tokenizers-0.19.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:ed69af290c2b65169f0ba9034d1dc39a5db9459b32f1dd8b5f3f32a3fcf06eab", size = 9634426 },
-    { url = "https://files.pythonhosted.org/packages/f1/2a/5ac32ef70d6f9464155c4c4239139dc5aa9297052180b171f5ae22fd7b7a/tokenizers-0.19.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f8a9c828277133af13f3859d1b6bf1c3cb6e9e1637df0e45312e6b7c2e622b1f", size = 9947379 },
+    { url = "https://files.pythonhosted.org/packages/bf/33/f4b2d94ada7ab297328fc671fed209368ddb82f965ec2224eb1892674c3a/tokenizers-0.22.1-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:59fdb013df17455e5f950b4b834a7b3ee2e0271e6378ccb33aa74d178b513c73", size = 3069318 },
+    { url = "https://files.pythonhosted.org/packages/1c/58/2aa8c874d02b974990e89ff95826a4852a8b2a273c7d1b4411cdd45a4565/tokenizers-0.22.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:8d4e484f7b0827021ac5f9f71d4794aaef62b979ab7608593da22b1d2e3c4edc", size = 2926478 },
+    { url = "https://files.pythonhosted.org/packages/1e/3b/55e64befa1e7bfea963cf4b787b2cea1011362c4193f5477047532ce127e/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19d2962dd28bc67c1f205ab180578a78eef89ac60ca7ef7cbe9635a46a56422a", size = 3256994 },
+    { url = "https://files.pythonhosted.org/packages/71/0b/fbfecf42f67d9b7b80fde4aabb2b3110a97fac6585c9470b5bff103a80cb/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:38201f15cdb1f8a6843e6563e6e79f4abd053394992b9bbdf5213ea3469b4ae7", size = 3153141 },
+    { url = "https://files.pythonhosted.org/packages/17/a9/b38f4e74e0817af8f8ef925507c63c6ae8171e3c4cb2d5d4624bf58fca69/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d1cbe5454c9a15df1b3443c726063d930c16f047a3cc724b9e6e1a91140e5a21", size = 3508049 },
+    { url = "https://files.pythonhosted.org/packages/d2/48/dd2b3dac46bb9134a88e35d72e1aa4869579eacc1a27238f1577270773ff/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e7d094ae6312d69cc2a872b54b91b309f4f6fbce871ef28eb27b52a98e4d0214", size = 3710730 },
+    { url = "https://files.pythonhosted.org/packages/93/0e/ccabc8d16ae4ba84a55d41345207c1e2ea88784651a5a487547d80851398/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:afd7594a56656ace95cdd6df4cca2e4059d294c5cfb1679c57824b605556cb2f", size = 3412560 },
+    { url = "https://files.pythonhosted.org/packages/d0/c6/dc3a0db5a6766416c32c034286d7c2d406da1f498e4de04ab1b8959edd00/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2ef6063d7a84994129732b47e7915e8710f27f99f3a3260b8a38fc7ccd083f4", size = 3250221 },
+    { url = "https://files.pythonhosted.org/packages/d7/a6/2c8486eef79671601ff57b093889a345dd3d576713ef047776015dc66de7/tokenizers-0.22.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ba0a64f450b9ef412c98f6bcd2a50c6df6e2443b560024a09fa6a03189726879", size = 9345569 },
+    { url = "https://files.pythonhosted.org/packages/6b/16/32ce667f14c35537f5f605fe9bea3e415ea1b0a646389d2295ec348d5657/tokenizers-0.22.1-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:331d6d149fa9c7d632cde4490fb8bbb12337fa3a0232e77892be656464f4b446", size = 9271599 },
+    { url = "https://files.pythonhosted.org/packages/51/7c/a5f7898a3f6baa3fc2685c705e04c98c1094c523051c805cdd9306b8f87e/tokenizers-0.22.1-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:607989f2ea68a46cb1dfbaf3e3aabdf3f21d8748312dbeb6263d1b3b66c5010a", size = 9533862 },
+    { url = "https://files.pythonhosted.org/packages/36/65/7e75caea90bc73c1dd8d40438adf1a7bc26af3b8d0a6705ea190462506e1/tokenizers-0.22.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a0f307d490295717726598ef6fa4f24af9d484809223bbc253b201c740a06390", size = 9681250 },
+    { url = "https://files.pythonhosted.org/packages/30/2c/959dddef581b46e6209da82df3b78471e96260e2bc463f89d23b1bf0e52a/tokenizers-0.22.1-cp39-abi3-win32.whl", hash = "sha256:b5120eed1442765cd90b903bb6cfef781fd8fe64e34ccaecbae4c619b7b12a82", size = 2472003 },
+    { url = "https://files.pythonhosted.org/packages/b3/46/e33a8c93907b631a99377ef4c5f817ab453d0b34f93529421f42ff559671/tokenizers-0.22.1-cp39-abi3-win_amd64.whl", hash = "sha256:65fd6e3fb11ca1e78a6a93602490f134d1fdeb13bcef99389d5102ea318ed138", size = 2674684 },
 ]
 
 [[package]]
@@ -5887,7 +6024,9 @@ name = "torchvision"
 version = "0.23.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "numpy" },
+    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
+    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
     { name = "pillow" },
     { name = "torch" },
 ]
@@ -5960,12 +6099,14 @@ wheels = [
 
 [[package]]
 name = "transformers"
-version = "4.42.4"
+version = "4.56.2"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "filelock" },
     { name = "huggingface-hub" },
-    { name = "numpy" },
+    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
+    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
     { name = "packaging" },
     { name = "pyyaml" },
     { name = "regex" },
@@ -5974,9 +6115,9 @@ dependencies = [
     { name = "tokenizers" },
     { name = "tqdm" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/84/eb/259afff0df9ece338dc224007bbe7dd6c9aae8e26957dc4033a3ec857588/transformers-4.42.4.tar.gz", hash = "sha256:f956e25e24df851f650cb2c158b6f4352dfae9d702f04c113ed24fc36ce7ae2d", size = 8054872 }
+sdist = { url = "https://files.pythonhosted.org/packages/e5/82/0bcfddd134cdf53440becb5e738257cc3cf34cf229d63b57bfd288e6579f/transformers-4.56.2.tar.gz", hash = "sha256:5e7c623e2d7494105c726dd10f6f90c2c99a55ebe86eef7233765abd0cb1c529", size = 9844296 }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/6a/dc/23c26b7b0bce5aaccf2b767db3e9c4f5ae4331bd47688c1f2ef091b23696/transformers-4.42.4-py3-none-any.whl", hash = "sha256:6d59061392d0f1da312af29c962df9017ff3c0108c681a56d1bc981004d16d24", size = 9337817 },
+    { url = "https://files.pythonhosted.org/packages/70/26/2591b48412bde75e33bfd292034103ffe41743cacd03120e3242516cd143/transformers-4.56.2-py3-none-any.whl", hash = "sha256:79c03d0e85b26cb573c109ff9eafa96f3c8d4febfd8a0774e8bba32702dd6dde", size = 11608055 },
 ]
 
 [[package]]