update exp for vldb

* Metadata filtering initial version

* Metadata filtering initial version

* Fixes linter issues

* Cleanup code

* Clean up and readme

* Fix after review

* Use UV in example

* Merge main into feature/metadata-filtering

.gitattributes

@@ -1 +0,0 @@
-paper_plot/data/big_graph_degree_data.npz filter=lfs diff=lfs merge=lfs -text

.github/workflows/build-and-publish.yml

@@ -0,0 +1,12 @@
+name: CI
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  workflow_dispatch:
+
+jobs:
+  build:
+    uses: ./.github/workflows/build-reusable.yml

.github/workflows/build-reusable.yml

@@ -0,0 +1,358 @@
+name: Reusable Build
+
+on:
+  workflow_call:
+    inputs:
+      ref:
+        description: 'Git ref to build'
+        required: false
+        type: string
+        default: ''
+
+jobs:
+  lint:
+    name: Lint and Format Check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Install ruff
+        run: |
+          uv tool install ruff
+
+      - name: Run ruff check
+        run: |
+          ruff check .
+
+      - name: Run ruff format check
+        run: |
+          ruff format --check .
+
+  build:
+    needs: lint
+    name: Build ${{ matrix.os }} Python ${{ matrix.python }}
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-22.04
+            python: '3.9'
+          - os: ubuntu-22.04
+            python: '3.10'
+          - os: ubuntu-22.04
+            python: '3.11'
+          - os: ubuntu-22.04
+            python: '3.12'
+          - os: ubuntu-22.04
+            python: '3.13'
+          - os: macos-14
+            python: '3.9'
+          - os: macos-14
+            python: '3.10'
+          - os: macos-14
+            python: '3.11'
+          - os: macos-14
+            python: '3.12'
+          - os: macos-14
+            python: '3.13'
+          - os: macos-15
+            python: '3.9'
+          - os: macos-15
+            python: '3.10'
+          - os: macos-15
+            python: '3.11'
+          - os: macos-15
+            python: '3.12'
+          - os: macos-15
+            python: '3.13'
+          - os: macos-13
+            python: '3.9'
+          - os: macos-13
+            python: '3.10'
+          - os: macos-13
+            python: '3.11'
+          - os: macos-13
+            python: '3.12'
+          # Note: macos-13 + Python 3.13 excluded due to PyTorch compatibility
+          # (PyTorch 2.5+ supports Python 3.13 but not Intel Mac x86_64)
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          ref: ${{ inputs.ref }}
+          submodules: recursive
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+
+      - name: Install system dependencies (Ubuntu)
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \
+            pkg-config libabsl-dev libaio-dev libprotobuf-dev \
+            patchelf
+
+          # Install Intel MKL for DiskANN
+          wget -q https://registrationcenter-download.intel.com/akdlm/IRC_NAS/79153e0f-74d7-45af-b8c2-258941adf58a/intel-onemkl-2025.0.0.940.sh
+          sudo sh intel-onemkl-2025.0.0.940.sh -a --components intel.oneapi.lin.mkl.devel --action install --eula accept -s
+          source /opt/intel/oneapi/setvars.sh
+          echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV
+          echo "LD_LIBRARY_PATH=/opt/intel/oneapi/compiler/latest/linux/compiler/lib/intel64_lin" >> $GITHUB_ENV
+          echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/intel/oneapi/mkl/latest/lib/intel64" >> $GITHUB_ENV
+
+      - name: Install system dependencies (macOS)
+        if: runner.os == 'macOS'
+        run: |
+          # Don't install LLVM, use system clang for better compatibility
+          brew install libomp boost protobuf zeromq
+
+      - name: Install build dependencies
+        run: |
+          uv pip install --system scikit-build-core numpy swig Cython pybind11
+          if [[ "$RUNNER_OS" == "Linux" ]]; then
+            uv pip install --system auditwheel
+          else
+            uv pip install --system delocate
+          fi
+
+      - name: Set macOS environment variables
+        if: runner.os == 'macOS'
+        run: |
+          # Use brew --prefix to automatically detect Homebrew installation path
+          HOMEBREW_PREFIX=$(brew --prefix)
+          echo "HOMEBREW_PREFIX=${HOMEBREW_PREFIX}" >> $GITHUB_ENV
+          echo "OpenMP_ROOT=${HOMEBREW_PREFIX}/opt/libomp" >> $GITHUB_ENV
+
+          # Set CMAKE_PREFIX_PATH to let CMake find all packages automatically
+          echo "CMAKE_PREFIX_PATH=${HOMEBREW_PREFIX}" >> $GITHUB_ENV
+
+          # Set compiler flags for OpenMP (required for both backends)
+          echo "LDFLAGS=-L${HOMEBREW_PREFIX}/opt/libomp/lib" >> $GITHUB_ENV
+          echo "CPPFLAGS=-I${HOMEBREW_PREFIX}/opt/libomp/include" >> $GITHUB_ENV
+
+      - name: Build packages
+        run: |
+          # Build core (platform independent)
+          cd packages/leann-core
+          uv build
+          cd ../..
+
+          # Build HNSW backend
+          cd packages/leann-backend-hnsw
+          if [[ "${{ matrix.os }}" == macos-* ]]; then
+            # Use system clang for better compatibility
+            export CC=clang
+            export CXX=clang++
+            # Homebrew libraries on each macOS version require matching minimum version
+            if [[ "${{ matrix.os }}" == "macos-13" ]]; then
+              export MACOSX_DEPLOYMENT_TARGET=13.0
+            elif [[ "${{ matrix.os }}" == "macos-14" ]]; then
+              export MACOSX_DEPLOYMENT_TARGET=14.0
+            elif [[ "${{ matrix.os }}" == "macos-15" ]]; then
+              export MACOSX_DEPLOYMENT_TARGET=15.0
+            fi
+            uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
+          else
+            uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
+          fi
+          cd ../..
+
+          # Build DiskANN backend
+          cd packages/leann-backend-diskann
+          if [[ "${{ matrix.os }}" == macos-* ]]; then
+            # Use system clang for better compatibility
+            export CC=clang
+            export CXX=clang++
+            # DiskANN requires macOS 13.3+ for sgesdd_ LAPACK function
+            # But Homebrew libraries on each macOS version require matching minimum version
+            if [[ "${{ matrix.os }}" == "macos-13" ]]; then
+              export MACOSX_DEPLOYMENT_TARGET=13.3
+            elif [[ "${{ matrix.os }}" == "macos-14" ]]; then
+              export MACOSX_DEPLOYMENT_TARGET=14.0
+            elif [[ "${{ matrix.os }}" == "macos-15" ]]; then
+              export MACOSX_DEPLOYMENT_TARGET=15.0
+            fi
+            uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
+          else
+            uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
+          fi
+          cd ../..
+
+          # Build meta package (platform independent)
+          cd packages/leann
+          uv build
+          cd ../..
+
+      - name: Repair wheels (Linux)
+        if: runner.os == 'Linux'
+        run: |
+          # Repair HNSW wheel
+          cd packages/leann-backend-hnsw
+          if [ -d dist ]; then
+            auditwheel repair dist/*.whl -w dist_repaired
+            rm -rf dist
+            mv dist_repaired dist
+          fi
+          cd ../..
+
+          # Repair DiskANN wheel
+          cd packages/leann-backend-diskann
+          if [ -d dist ]; then
+            auditwheel repair dist/*.whl -w dist_repaired
+            rm -rf dist
+            mv dist_repaired dist
+          fi
+          cd ../..
+
+      - name: Repair wheels (macOS)
+        if: runner.os == 'macOS'
+        run: |
+          # Determine deployment target based on runner OS
+          # Must match the Homebrew libraries for each macOS version
+          if [[ "${{ matrix.os }}" == "macos-13" ]]; then
+            HNSW_TARGET="13.0"
+            DISKANN_TARGET="13.3"
+          elif [[ "${{ matrix.os }}" == "macos-14" ]]; then
+            HNSW_TARGET="14.0"
+            DISKANN_TARGET="14.0"
+          elif [[ "${{ matrix.os }}" == "macos-15" ]]; then
+            HNSW_TARGET="15.0"
+            DISKANN_TARGET="15.0"
+          fi
+
+          # Repair HNSW wheel
+          cd packages/leann-backend-hnsw
+          if [ -d dist ]; then
+            export MACOSX_DEPLOYMENT_TARGET=$HNSW_TARGET
+            delocate-wheel -w dist_repaired -v --require-target-macos-version $HNSW_TARGET dist/*.whl
+            rm -rf dist
+            mv dist_repaired dist
+          fi
+          cd ../..
+
+          # Repair DiskANN wheel
+          cd packages/leann-backend-diskann
+          if [ -d dist ]; then
+            export MACOSX_DEPLOYMENT_TARGET=$DISKANN_TARGET
+            delocate-wheel -w dist_repaired -v --require-target-macos-version $DISKANN_TARGET dist/*.whl
+            rm -rf dist
+            mv dist_repaired dist
+          fi
+          cd ../..
+
+      - name: List built packages
+        run: |
+          echo "📦 Built packages:"
+          find packages/*/dist -name "*.whl" -o -name "*.tar.gz" | sort
+
+
+      - name: Install built packages for testing
+        run: |
+          # Create a virtual environment with the correct Python version
+          uv venv --python ${{ matrix.python }}
+          source .venv/bin/activate || source .venv/Scripts/activate
+
+          # Install packages using --find-links to prioritize local builds
+          uv pip install --find-links packages/leann-core/dist --find-links packages/leann-backend-hnsw/dist --find-links packages/leann-backend-diskann/dist packages/leann-core/dist/*.whl || uv pip install --find-links packages/leann-core/dist packages/leann-core/dist/*.tar.gz
+          uv pip install --find-links packages/leann-core/dist packages/leann-backend-hnsw/dist/*.whl
+          uv pip install --find-links packages/leann-core/dist packages/leann-backend-diskann/dist/*.whl
+          uv pip install packages/leann/dist/*.whl || uv pip install packages/leann/dist/*.tar.gz
+
+          # Install test dependencies using extras
+          uv pip install -e ".[test]"
+
+      - name: Run tests with pytest
+        env:
+          CI: true
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          HF_HUB_DISABLE_SYMLINKS: 1
+          TOKENIZERS_PARALLELISM: false
+          PYTORCH_ENABLE_MPS_FALLBACK: 0
+          OMP_NUM_THREADS: 1
+          MKL_NUM_THREADS: 1
+        run: |
+          source .venv/bin/activate || source .venv/Scripts/activate
+          pytest tests/ -v --tb=short
+
+      - name: Run sanity checks (optional)
+        run: |
+          # Activate virtual environment
+          source .venv/bin/activate || source .venv/Scripts/activate
+
+          # Run distance function tests if available
+          if [ -f test/sanity_checks/test_distance_functions.py ]; then
+            echo "Running distance function sanity checks..."
+            python test/sanity_checks/test_distance_functions.py || echo "⚠️ Distance function test failed, continuing..."
+          fi
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: packages-${{ matrix.os }}-py${{ matrix.python }}
+          path: packages/*/dist/
+
+
+  arch-smoke:
+    name: Arch Linux smoke test (install & import)
+    needs: build
+    runs-on: ubuntu-latest
+    container:
+      image: archlinux:latest
+
+    steps:
+      - name: Prepare system
+        run: |
+          pacman -Syu --noconfirm
+          pacman -S --noconfirm python python-pip gcc git zlib openssl
+
+      - name: Download ALL wheel artifacts from this run
+        uses: actions/download-artifact@v5
+        with:
+          # Don't specify name, download all artifacts
+          path: ./wheels
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+
+      - name: Create virtual environment and install wheels
+        run: |
+          uv venv
+          source .venv/bin/activate || source .venv/Scripts/activate
+          uv pip install --find-links wheels leann-core
+          uv pip install --find-links wheels leann-backend-hnsw
+          uv pip install --find-links wheels leann-backend-diskann
+          uv pip install --find-links wheels leann
+
+      - name: Import & tiny runtime check
+        env:
+          OMP_NUM_THREADS: 1
+          MKL_NUM_THREADS: 1
+        run: |
+          source .venv/bin/activate || source .venv/Scripts/activate
+          python - <<'PY'
+          import leann
+          import leann_backend_hnsw as h
+          import leann_backend_diskann as d
+          from leann import LeannBuilder, LeannSearcher
+          b = LeannBuilder(backend_name="hnsw")
+          b.add_text("hello arch")
+          b.build_index("arch_demo.leann")
+          s = LeannSearcher("arch_demo.leann")
+          print("search:", s.search("hello", top_k=1))
+          PY

.github/workflows/link-check.yml

@@ -0,0 +1,19 @@
+name: Link Check
+
+on:
+  push:
+    branches: [ main, master ]
+  pull_request:
+  schedule:
+    - cron: "0 3 * * 1"
+
+jobs:
+  link-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: lycheeverse/lychee-action@v2
+        with:
+          args: --no-progress --insecure --user-agent 'curl/7.68.0' README.md docs/ apps/ examples/ benchmarks/
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release-manual.yml

@@ -0,0 +1,129 @@
+name: Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version to release (e.g., 0.1.2)'
+        required: true
+        type: string
+
+jobs:
+  update-version:
+    name: Update Version
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    outputs:
+      commit-sha: ${{ steps.push.outputs.commit-sha }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Validate version
+        run: |
+          # Remove 'v' prefix if present for validation
+          VERSION_CLEAN="${{ inputs.version }}"
+          VERSION_CLEAN="${VERSION_CLEAN#v}"
+          if ! [[ "$VERSION_CLEAN" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "❌ Invalid version format. Expected format: X.Y.Z or vX.Y.Z"
+            exit 1
+          fi
+          echo "✅ Version format valid: ${{ inputs.version }}"
+
+      - name: Update versions and push
+        id: push
+        run: |
+          # Check current version
+          CURRENT_VERSION=$(grep "^version" packages/leann-core/pyproject.toml | cut -d'"' -f2)
+          echo "Current version: $CURRENT_VERSION"
+          echo "Target version: ${{ inputs.version }}"
+
+          if [ "$CURRENT_VERSION" = "${{ inputs.version }}" ]; then
+            echo "⚠️  Version is already ${{ inputs.version }}, skipping update"
+            COMMIT_SHA=$(git rev-parse HEAD)
+          else
+            ./scripts/bump_version.sh ${{ inputs.version }}
+            git config user.name "GitHub Actions"
+            git config user.email "actions@github.com"
+            git add packages/*/pyproject.toml
+            git commit -m "chore: release v${{ inputs.version }}"
+            git push origin main
+            COMMIT_SHA=$(git rev-parse HEAD)
+            echo "✅ Pushed version update: $COMMIT_SHA"
+          fi
+
+          echo "commit-sha=$COMMIT_SHA" >> $GITHUB_OUTPUT
+
+  build-packages:
+    name: Build packages
+    needs: update-version
+    uses: ./.github/workflows/build-reusable.yml
+    with:
+      ref: 'main'
+
+  publish:
+    name: Publish and Release
+    needs: [update-version, build-packages]
+    if: always() && needs.update-version.result == 'success' && needs.build-packages.result == 'success'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: 'main'
+
+      - name: Download all artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: dist-artifacts
+
+      - name: Collect packages
+        run: |
+          mkdir -p dist
+          find dist-artifacts -name "*.whl" -exec cp {} dist/ \;
+          find dist-artifacts -name "*.tar.gz" -exec cp {} dist/ \;
+
+          echo "📦 Packages to publish:"
+          ls -la dist/
+
+      - name: Publish to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+        run: |
+          if [ -z "$TWINE_PASSWORD" ]; then
+            echo "❌ PYPI_API_TOKEN not configured!"
+            exit 1
+          fi
+
+          pip install twine
+          twine upload dist/* --skip-existing --verbose
+
+          echo "✅ Published to PyPI!"
+
+      - name: Create release
+        run: |
+          # Check if tag already exists
+          if git rev-parse "v${{ inputs.version }}" >/dev/null 2>&1; then
+            echo "⚠️  Tag v${{ inputs.version }} already exists, skipping tag creation"
+          else
+            git tag "v${{ inputs.version }}"
+            git push origin "v${{ inputs.version }}"
+            echo "✅ Created and pushed tag v${{ inputs.version }}"
+          fi
+
+          # Check if release already exists
+          if gh release view "v${{ inputs.version }}" >/dev/null 2>&1; then
+            echo "⚠️  Release v${{ inputs.version }} already exists, skipping release creation"
+          else
+            gh release create "v${{ inputs.version }}" \
+              --title "Release v${{ inputs.version }}" \
+              --notes "🚀 Released to PyPI: https://pypi.org/project/leann/${{ inputs.version }}/" \
+              --latest
+            echo "✅ Created GitHub release v${{ inputs.version }}"
+          fi
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -8,11 +8,17 @@ demo/indices/
 *pycache*
 outputs/
 *.pkl
+*.pdf
+*.idx
+*.map
 .history/
-scripts/
 lm_eval.egg-info/
 demo/experiment_results/**/*.json
 *.jsonl
+*.eml
+*.emlx
+*.json
+!.vscode/*.json
 *.sh
 *.txt
 !CMakeLists.txt
@@ -29,7 +35,15 @@ build/
 nprobe_logs/
 micro/results
 micro/contriever-INT8
-examples/data/
+data/*
+!data/2501.14312v1 (1).pdf
+!data/2506.08276v1.pdf
+!data/PrideandPrejudice.txt
+!data/huawei_pangu.md
+!data/ground_truth/
+!data/indices/
+!data/queries/
+!data/.gitattributes
 *.qdstrm
 benchmark_results/
 results/
@@ -42,6 +56,7 @@ embedding_comparison_results/
 *.ivecs
 *.index
 *.bin
+*.old
 
 read_graph
 analyze_diskann_graph
@@ -71,3 +86,25 @@ test_indices*/
 test_*.py
 !tests/**
 packages/leann-backend-diskann/third_party/DiskANN/_deps/
+
+*.meta.json
+*.passages.json
+
+batchtest.py
+tests/__pytest_cache__/
+tests/__pycache__/
+paru-bin/
+
+CLAUDE.md
+CLAUDE.local.md
+.claude/*.local.*
+.claude/local/*
+
+benchmarks/data/
+!benchmarks/data/prompts_g5/*.txt
+!benchmarks/run_all.sh
+!benchmarks/run_speed_bench_all.sh
+!benchmarks/simple_mac_tpt_test.py
+!benchmarks/run_speed_bench_all.sh
+!benchmarks/run_speed_bench_all.sh
+!benchmarks/run_speed_bench_all.sh

.gitmodules

@@ -1,6 +1,16 @@
 [submodule "packages/leann-backend-diskann/third_party/DiskANN"]
 	path = packages/leann-backend-diskann/third_party/DiskANN
-	url = https://github.com/yichuan520030910320/DiskANN.git
+	url = https://github.com/yichuan-w/DiskANN.git
 [submodule "packages/leann-backend-hnsw/third_party/faiss"]
 	path = packages/leann-backend-hnsw/third_party/faiss
-	url = https://github.com/yichuan520030910320/faiss.git
+	url = https://github.com/yichuan-w/faiss.git
+[submodule "packages/leann-backend-hnsw/third_party/msgpack-c"]
+	path = packages/leann-backend-hnsw/third_party/msgpack-c
+	url = https://github.com/msgpack/msgpack-c.git
+	branch = cpp_master
+[submodule "packages/leann-backend-hnsw/third_party/cppzmq"]
+	path = packages/leann-backend-hnsw/third_party/cppzmq
+	url = https://github.com/zeromq/cppzmq.git
+[submodule "packages/leann-backend-hnsw/third_party/libzmq"]
+	path = packages/leann-backend-hnsw/third_party/libzmq
+	url = https://github.com/zeromq/libzmq.git

.pre-commit-config.yaml

@@ -0,0 +1,17 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files
+      - id: check-merge-conflict
+      - id: debug-statements
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.12.7  # Fixed version to match pyproject.toml
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
+      - id: ruff-format

.vscode/extensions.json

@@ -1,9 +1,5 @@
 {
     "recommendations": [
-        "llvm-vs-code-extensions.vscode-clangd",
-        "ms-python.python",
-        "ms-vscode.cmake-tools",
-        "vadimcn.vscode-lldb",
-        "eamodio.gitlens",
+        "charliermarsh.ruff",
     ]
 }

.vscode/launch.json

@@ -1,283 +0,0 @@
-{
-    // Use IntelliSense to learn about possible attributes.
-    // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-    "version": "0.2.0",
-    "configurations": [
-        // new emdedder
-        {
-            "name": "New Embedder",
-            "type": "debugpy",
-            "request": "launch",
-            "program": "demo/main.py",
-            "console": "integratedTerminal",
-            "args": [
-                "--search",
-                "--use-original",
-                "--domain",
-                "dpr",
-                "--nprobe",
-                "5000",
-                "--load",
-                "flat",
-                "--embedder",
-                "intfloat/multilingual-e5-small"
-            ]
-        }
-        //python /home/ubuntu/Power-RAG/faiss/demo/simple_build.py
-        {
-            "name": "main.py",
-            "type": "debugpy",
-            "request": "launch",
-            "program": "demo/main.py",
-            "console": "integratedTerminal",
-            "cwd": "${workspaceFolder}",
-            "args": [
-                "--query",
-                "1000",
-                "--load",
-                "bm25"
-            ]
-        },
-        {
-            "name": "Simple Build",
-            "type": "lldb",
-            "request": "launch",
-            "program": "${workspaceFolder}/.venv/bin/python",
-            "console": "integratedTerminal",
-            "cwd": "${workspaceFolder}",
-            "args": [
-                "faiss/demo/simple_build.py"
-            ],
-            "env": {
-                "LD_PRELOAD": "/lib/x86_64-linux-gnu/libmkl_core.so:/lib/x86_64-linux-gnu/libmkl_intel_thread.so:/lib/x86_64-linux-gnu/libmkl_intel_lp64.so:/lib/x86_64-linux-gnu/libiomp5.so"
-            }
-        },
-        //# Fix for Intel MKL error
-        //export LD_PRELOAD=/lib/x86_64-linux-gnu/libmkl_core.so:/lib/x86_64-linux-gnu/libmkl_intel_thread.so:/lib/x86_64-linux-gnu/libmkl_intel_lp64.so:/lib/x86_64-linux-gnu/libiomp5.so
-        //python faiss/demo/build_demo.py
-        {
-            "name": "Build Demo",
-            "type": "lldb",
-            "request": "launch",
-            "program": "${workspaceFolder}/.venv/bin/python",
-            "console": "integratedTerminal",
-            "cwd": "${workspaceFolder}",
-            "args": [
-                "faiss/demo/build_demo.py"
-            ],
-            "env": {
-                "LD_PRELOAD": "/lib/x86_64-linux-gnu/libmkl_core.so:/lib/x86_64-linux-gnu/libmkl_intel_thread.so:/lib/x86_64-linux-gnu/libmkl_intel_lp64.so:/lib/x86_64-linux-gnu/libiomp5.so"
-            }
-        },
-        {
-            "name": "DiskANN Serve",
-            "type": "lldb",
-            "request": "launch",
-            "program": "${workspaceFolder}/.venv/bin/python",
-            "console": "integratedTerminal",
-            "cwd": "${workspaceFolder}",
-            "args": [
-                "demo/main.py",
-                "--mode",
-                "serve",
-                "--engine",
-                "sglang",
-                "--load-indices",
-                "diskann",
-                "--domain",
-                "rpj_wiki",
-                "--lazy-load",
-                "--recompute-beighbor-embeddings",
-                "--port",
-                "8082",
-                "--diskann-search-memory-maximum",
-                "2",
-                "--diskann-graph",
-                "240",
-                "--search-only"
-            ],
-            "env": {
-                "PYTHONPATH": "${workspaceFolder}/faiss_repo/build/faiss/python:$PYTHONPATH"
-            },
-            "preLaunchTask": "CMake: build",
-        },
-        {
-            "name": "DiskANN Serve MAC",
-            "type": "lldb",
-            "request": "launch",
-            "program": "${workspaceFolder}/.venv/bin/python",
-            "console": "integratedTerminal",
-            "cwd": "${workspaceFolder}",
-            "args": [
-                "demo/main.py",
-                "--mode",
-                "serve",
-                "--engine",
-                "ollama",
-                "--load-indices",
-                "diskann",
-                "--domain",
-                "rpj_wiki",
-                "--lazy-load",
-                "--recompute-beighbor-embeddings"
-            ],
-            "preLaunchTask": "CMake: build",
-            "env": {
-                "KMP_DUPLICATE_LIB_OK": "TRUE",
-                "OMP_NUM_THREADS": "1",
-                "MKL_NUM_THREADS": "1",
-                "DYLD_INSERT_LIBRARIES": "/Users/ec2-user/Power-RAG/.venv/lib/python3.10/site-packages/torch/lib/libomp.dylib",
-                "KMP_BLOCKTIME": "0"
-            }
-        },
-        {
-            "name": "Python Debugger: Current File with Arguments",
-            "type": "debugpy",
-            "request": "launch",
-            "program": "ric/main_ric.py",
-            "console": "integratedTerminal",
-            "cwd": "${workspaceFolder}",
-            "args": [
-                "--config-name",
-                "${input:configSelection}"
-            ],
-            "justMyCode": false
-        },
-        //python ./demo/validate_equivalence.py sglang
-        {
-            "name": "Validate Equivalence",
-            "type": "debugpy",
-            "request": "launch",
-            "program": "demo/validate_equivalence.py",
-            "console": "integratedTerminal",
-            "args": [
-                "sglang"
-            ],
-        },
-        //python demo/retrieval_demo.py --engine sglang  --skip-embeddings --domain dpr --load-indices flat ivf_flat
-        {
-            "name": "Retrieval Demo",
-            "type": "debugpy",
-            "request": "launch",
-            "program": "demo/retrieval_demo.py",
-            "console": "integratedTerminal",
-            "args": [
-                "--engine",
-                "vllm",
-                "--skip-embeddings",
-                "--domain",
-                "dpr",
-                "--load-indices",
-                // "flat",
-                "ivf_flat"
-            ],
-        },
-        //python demo/retrieval_demo.py --engine sglang  --skip-embeddings --domain dpr  --load-indices  diskann --hnsw-M 64 --hnsw-efConstruction 150 --hnsw-efSearch 128  --hnsw-sq-bits 8 
-        {
-            "name": "Retrieval Demo DiskANN",
-            "type": "debugpy",
-            "request": "launch",
-            "program": "demo/retrieval_demo.py",
-            "console": "integratedTerminal",
-            "args": [
-                "--engine",
-                "sglang",
-                "--skip-embeddings",
-                "--domain",
-                "dpr",
-                "--load-indices",
-                "diskann",
-                "--hnsw-M",
-                "64",
-                "--hnsw-efConstruction",
-                "150",
-                "--hnsw-efSearch",
-                "128",
-                "--hnsw-sq-bits",
-                "8"
-            ],
-        },
-        {
-            "name": "Find Probe",
-            "type": "debugpy",
-            "request": "launch",
-            "program": "find_probe.py",
-            "console": "integratedTerminal",
-            "cwd": "${workspaceFolder}",
-        },
-        {
-            "name": "Python: Attach",
-            "type": "debugpy",
-            "request": "attach",
-            "processId": "${command:pickProcess}",
-            "justMyCode": true
-        },
-        {
-            "name": "Edge RAG",
-            "type": "lldb",
-            "request": "launch",
-            "program": "${workspaceFolder}/.venv/bin/python",
-            "console": "integratedTerminal",
-            "cwd": "${workspaceFolder}",
-            "args": [
-                "edgerag_demo.py"
-            ],
-            "env": {
-                "LD_PRELOAD": "/lib/x86_64-linux-gnu/libiomp5.so /lib/x86_64-linux-gnu/libmkl_core.so /lib/x86_64-linux-gnu/libmkl_intel_lp64.so /lib/x86_64-linux-gnu/libmkl_intel_thread.so",
-                "MKL_NUM_THREADS": "1",
-                "OMP_NUM_THREADS": "1",
-            }
-        },
-        {
-            "name": "Launch Embedding Server",
-            "type": "debugpy",
-            "request": "launch",
-            "program": "demo/embedding_server.py",
-            "console": "integratedTerminal",
-            "cwd": "${workspaceFolder}",
-            "args": [
-                "--domain",
-                "rpj_wiki",
-                "--zmq-port",
-                "5556",
-            ]
-        },
-        {
-            "name": "HNSW Serve",
-            "type": "lldb",
-            "request": "launch",
-            "program": "${workspaceFolder}/.venv/bin/python",
-            "console": "integratedTerminal",
-            "cwd": "${workspaceFolder}",
-            "args": [
-                "demo/main.py",
-                "--domain",
-                "rpj_wiki",
-                "--load",
-                "hnsw",
-                "--mode",
-                "serve",
-                "--search",
-                "--skip-pa",
-                "--recompute",
-                "--hnsw-old"
-            ],
-            "env": {
-                "LD_PRELOAD": "/lib/x86_64-linux-gnu/libmkl_core.so:/lib/x86_64-linux-gnu/libmkl_intel_thread.so:/lib/x86_64-linux-gnu/libmkl_intel_lp64.so:/lib/x86_64-linux-gnu/libiomp5.so"
-            }
-        },
-    ],
-    "inputs": [
-        {
-            "id": "configSelection",
-            "type": "pickString",
-            "description": "Select a configuration",
-            "options": [
-                "example_config",
-                "vllm_gritlm"
-            ],
-            "default": "example_config"
-        }
-    ],
-}

.vscode/settings.json

@@ -1,43 +1,22 @@
 {
-    "python.analysis.extraPaths": [
-        "./sglang_repo/python"
-    ],
-    "cmake.sourceDirectory": "${workspaceFolder}/DiskANN",
-    "cmake.configureArgs": [
-        "-DPYBIND=True",
-        "-DUPDATE_EDITABLE_INSTALL=ON",
-    ],
-    "cmake.environment": {
-        "PATH": "/Users/ec2-user/Power-RAG/.venv/bin:${env:PATH}"
+  "python.defaultInterpreterPath": ".venv/bin/python",
+  "python.terminal.activateEnvironment": true,
+  "[python]": {
+    "editor.defaultFormatter": "charliermarsh.ruff",
+    "editor.formatOnSave": true,
+    "editor.codeActionsOnSave": {
+      "source.organizeImports": "explicit",
+      "source.fixAll": "explicit"
     },
-    "cmake.buildDirectory": "${workspaceFolder}/build",
-    "files.associations": {
-        "*.tcc": "cpp",
-        "deque": "cpp",
-        "string": "cpp",
-        "unordered_map": "cpp",
-        "vector": "cpp",
-        "map": "cpp",
-        "unordered_set": "cpp",
-        "atomic": "cpp",
-        "inplace_vector": "cpp",
-        "*.ipp": "cpp",
-        "forward_list": "cpp",
-        "list": "cpp",
-        "any": "cpp",
-        "system_error": "cpp",
-        "__hash_table": "cpp",
-        "__split_buffer": "cpp",
-        "__tree": "cpp",
-        "ios": "cpp",
-        "set": "cpp",
-        "__string": "cpp",
-        "string_view": "cpp",
-        "ranges": "cpp",
-        "iosfwd": "cpp"
+    "editor.insertSpaces": true,
+    "editor.tabSize": 4
   },
-    "lldb.displayFormat": "auto",
-    "lldb.showDisassembly": "auto",
-    "lldb.dereferencePointers": true,
-    "lldb.consoleMode": "commands",
+  "ruff.enable": true,
+  "files.watcherExclude": {
+    "**/.venv/**": true,
+    "**/__pycache__/**": true,
+    "**/*.egg-info/**": true,
+    "**/build/**": true,
+    "**/dist/**": true
+  }
 }

.vscode/tasks.json

@@ -1,16 +0,0 @@
-{
-	"version": "2.0.0",
-	"tasks": [
-		{
-			"type": "cmake",
-			"label": "CMake: build",
-			"command": "build",
-			"targets": [
-				"all"
-			],
-			"group": "build",
-			"problemMatcher": [],
-			"detail": "CMake template build task"
-		}
-	]
-}

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2024 Rulin Shao
+Copyright (c) 2025 LEANN Contributors
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 

README.md

@@ -1,172 +1,702 @@
-# 🚀 LEANN: A Low-Storage Vector Index
+<p align="center">
+  <img src="assets/logo-text.png" alt="LEANN Logo" width="400">
+</p>
 
 <p align="center">
-  <img src="https://img.shields.io/badge/Python-3.9%2B-blue.svg" alt="Python 3.9+">
+  <img src="https://img.shields.io/badge/Python-3.9%20%7C%203.10%20%7C%203.11%20%7C%203.12%20%7C%203.13-blue.svg" alt="Python Versions">
+  <img src="https://github.com/yichuan-w/LEANN/actions/workflows/build-and-publish.yml/badge.svg" alt="CI Status">
+  <img src="https://img.shields.io/badge/Platform-Ubuntu%20%26%20Arch%20%26%20WSL%20%7C%20macOS%20(ARM64%2FIntel)-lightgrey" alt="Platform">
   <img src="https://img.shields.io/badge/License-MIT-green.svg" alt="MIT License">
-  <img src="https://img.shields.io/badge/PRs-welcome-brightgreen.svg" alt="PRs Welcome">
-  <img src="https://img.shields.io/badge/Platform-Linux%20%7C%20macOS%20%7C%20Windows-lightgrey" alt="Platform">
+  <img src="https://img.shields.io/badge/MCP-Native%20Integration-blue" alt="MCP Integration">
 </p>
 
+<h2 align="center" tabindex="-1" class="heading-element" dir="auto">
+    The smallest vector index in the world. RAG Everything with LEANN!
+</h2>
+
+LEANN is an innovative vector database that democratizes personal AI. Transform your laptop into a powerful RAG system that can index and search through millions of documents while using **97% less storage** than traditional solutions **without accuracy loss**.
+
+LEANN achieves this through *graph-based selective recomputation* with *high-degree preserving pruning*, computing embeddings on-demand instead of storing them all. [Illustration Fig →](#️-architecture--how-it-works) | [Paper →](https://arxiv.org/abs/2506.08276)
+
+**Ready to RAG Everything?** Transform your laptop into a personal AI assistant that can semantic search your **[file system](#-personal-data-manager-process-any-documents-pdf-txt-md)**, **[emails](#-your-personal-email-secretary-rag-on-apple-mail)**, **[browser history](#-time-machine-for-the-web-rag-your-entire-browser-history)**, **[chat history](#-wechat-detective-unlock-your-golden-memories)**, **[codebase](#-claude-code-integration-transform-your-development-workflow)**\* , or external knowledge bases (i.e., 60M documents) - all on your laptop, with zero cloud costs and complete privacy.
+
+
+\* Claude Code only supports basic `grep`-style keyword search. **LEANN** is a drop-in **semantic search MCP service fully compatible with Claude Code**, unlocking intelligent retrieval without changing your workflow. 🔥 Check out [the easy setup →](packages/leann-mcp/README.md)
+
+
+
+## Why LEANN?
+
 <p align="center">
-  <strong>⚡ Real-time embedding computation for large-scale RAG on consumer hardware</strong>
+  <img src="assets/effects.png" alt="LEANN vs Traditional Vector DB Storage Comparison" width="70%">
 </p>
 
+> **The numbers speak for themselves:** Index 60 million text chunks in just 6GB instead of 201GB. From emails to browser history, everything fits on your laptop. [See detailed benchmarks for different applications below ↓](#-storage-comparison)
+
+
+🔒 **Privacy:** Your data never leaves your laptop. No OpenAI, no cloud, no "terms of service".
+
+🪶 **Lightweight:** Graph-based recomputation eliminates heavy embedding storage, while smart graph pruning and CSR format minimize graph storage overhead. Always less storage, less memory usage!
+
+📦 **Portable:** Transfer your entire knowledge base between devices (even with others) with minimal cost - your personal AI memory travels with you.
+
+📈 **Scalability:** Handle messy personal data that would crash traditional vector DBs, easily managing your growing personalized data and agent generated memory!
+
+✨ **No Accuracy Loss:** Maintain the same search quality as heavyweight solutions while using 97% less storage.
+
+## Installation
+
+### 📦 Prerequisites: Install uv
+
+[Install uv](https://docs.astral.sh/uv/getting-started/installation/#installation-methods) first if you don't have it. Typically, you can install it with:
+
+```bash
+curl -LsSf https://astral.sh/uv/install.sh | sh
+```
+
+### 🚀 Quick Install
+
+Clone the repository to access all examples and try amazing applications,
+
+```bash
+git clone https://github.com/yichuan-w/LEANN.git leann
+cd leann
+```
+
+and install LEANN from [PyPI](https://pypi.org/project/leann/) to run them immediately:
+
+```bash
+uv venv
+source .venv/bin/activate
+uv pip install leann
+```
+<!--
+> Low-resource? See “Low-resource setups” in the [Configuration Guide](docs/configuration-guide.md#low-resource-setups). -->
+
+<details>
+<summary>
+<strong>🔧 Build from Source (Recommended for development)</strong>
+</summary>
+
+
+
+```bash
+git clone https://github.com/yichuan-w/LEANN.git leann
+cd leann
+git submodule update --init --recursive
+```
+
+**macOS:**
+
+Note: DiskANN requires MacOS 13.3 or later.
+
+```bash
+brew install libomp boost protobuf zeromq pkgconf
+uv sync --extra diskann
+```
+
+**Linux (Ubuntu/Debian):**
+
+Note: On Ubuntu 20.04, you may need to build a newer Abseil and pin Protobuf (e.g., v3.20.x) for building DiskANN. See [Issue #30](https://github.com/yichuan-w/LEANN/issues/30) for a step-by-step note.
+
+You can manually install [Intel oneAPI MKL](https://www.intel.com/content/www/us/en/developer/tools/oneapi/onemkl.html) instead of `libmkl-full-dev` for DiskANN. You can also use `libopenblas-dev` for building HNSW only, by removing `--extra diskann` in the command below.
+
+```bash
+sudo apt-get update && sudo apt-get install -y \
+  libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \
+  pkg-config libabsl-dev libaio-dev libprotobuf-dev \
+  libmkl-full-dev
+
+uv sync --extra diskann
+```
+
+**Linux (Arch Linux):**
+
+```bash
+sudo pacman -Syu && sudo pacman -S --needed base-devel cmake pkgconf git gcc \
+  boost boost-libs protobuf abseil-cpp libaio zeromq
+
+# For MKL in DiskANN
+sudo pacman -S --needed base-devel git
+git clone https://aur.archlinux.org/paru-bin.git
+cd paru-bin && makepkg -si
+paru -S intel-oneapi-mkl intel-oneapi-compiler
+source /opt/intel/oneapi/setvars.sh
+
+uv sync --extra diskann
+```
+
+**Linux (RHEL / CentOS Stream / Oracle / Rocky / AlmaLinux):**
+
+See [Issue #50](https://github.com/yichuan-w/LEANN/issues/50) for more details.
+
+```bash
+sudo dnf groupinstall -y "Development Tools"
+sudo dnf install -y libomp-devel boost-devel protobuf-compiler protobuf-devel \
+  abseil-cpp-devel libaio-devel zeromq-devel pkgconf-pkg-config
+
+# For MKL in DiskANN
+sudo dnf install -y intel-oneapi-mkl intel-oneapi-mkl-devel \
+  intel-oneapi-openmp || sudo dnf install -y intel-oneapi-compiler
+source /opt/intel/oneapi/setvars.sh
+
+uv sync --extra diskann
+```
+
+</details>
+
+
+## Quick Start
+
+Our declarative API makes RAG as easy as writing a config file.
+
+Check out [demo.ipynb](demo.ipynb) or [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/yichuan-w/LEANN/blob/main/demo.ipynb)
+
+```python
+from leann import LeannBuilder, LeannSearcher, LeannChat
+from pathlib import Path
+INDEX_PATH = str(Path("./").resolve() / "demo.leann")
+
+# Build an index
+builder = LeannBuilder(backend_name="hnsw")
+builder.add_text("LEANN saves 97% storage compared to traditional vector databases.")
+builder.add_text("Tung Tung Tung Sahur called—they need their banana‑crocodile hybrid back")
+builder.build_index(INDEX_PATH)
+
+# Search
+searcher = LeannSearcher(INDEX_PATH)
+results = searcher.search("fantastical AI-generated creatures", top_k=1)
+
+# Chat with your data
+chat = LeannChat(INDEX_PATH, llm_config={"type": "hf", "model": "Qwen/Qwen3-0.6B"})
+response = chat.ask("How much storage does LEANN save?", top_k=1)
+```
+
+## RAG on Everything!
+
+LEANN supports RAG on various data sources including documents (`.pdf`, `.txt`, `.md`), Apple Mail, Google Search History, WeChat, and more.
+
+
+
+### Generation Model Setup
+
+LEANN supports multiple LLM providers for text generation (OpenAI API, HuggingFace, Ollama).
+
+<details>
+<summary><strong>🔑 OpenAI API Setup (Default)</strong></summary>
+
+Set your OpenAI API key as an environment variable:
+
+```bash
+export OPENAI_API_KEY="your-api-key-here"
+```
+
+</details>
+
+<details>
+<summary><strong>🔧 Ollama Setup (Recommended for full privacy)</strong></summary>
+
+**macOS:**
+
+First, [download Ollama for macOS](https://ollama.com/download/mac).
+
+```bash
+# Pull a lightweight model (recommended for consumer hardware)
+ollama pull llama3.2:1b
+```
+
+**Linux:**
+
+```bash
+# Install Ollama
+curl -fsSL https://ollama.ai/install.sh | sh
+
+# Start Ollama service manually
+ollama serve &
+
+# Pull a lightweight model (recommended for consumer hardware)
+ollama pull llama3.2:1b
+```
+
+</details>
+
+
+## ⭐ Flexible Configuration
+
+LEANN provides flexible parameters for embedding models, search strategies, and data processing to fit your specific needs.
+
+📚 **Need configuration best practices?** Check our [Configuration Guide](docs/configuration-guide.md) for detailed optimization tips, model selection advice, and solutions to common issues like slow embeddings or poor search quality.
+
+<details>
+<summary><strong>📋 Click to expand: Common Parameters (Available in All Examples)</strong></summary>
+
+All RAG examples share these common parameters. **Interactive mode** is available in all examples - simply run without `--query` to start a continuous Q&A session where you can ask multiple questions. Type 'quit' to exit.
+
+```bash
+# Core Parameters (General preprocessing for all examples)
+--index-dir DIR              # Directory to store the index (default: current directory)
+--query "YOUR QUESTION"      # Single query mode. Omit for interactive chat (type 'quit' to exit), and now you can play with your index interactively
+--max-items N                # Limit data preprocessing (default: -1, process all data)
+--force-rebuild              # Force rebuild index even if it exists
+
+# Embedding Parameters
+--embedding-model MODEL      # e.g., facebook/contriever, text-embedding-3-small, mlx-community/Qwen3-Embedding-0.6B-8bit or nomic-embed-text
+--embedding-mode MODE        # sentence-transformers, openai, mlx, or ollama
+
+# LLM Parameters (Text generation models)
+--llm TYPE                   # LLM backend: openai, ollama, or hf (default: openai)
+--llm-model MODEL            # Model name (default: gpt-4o) e.g., gpt-4o-mini, llama3.2:1b, Qwen/Qwen2.5-1.5B-Instruct
+--thinking-budget LEVEL      # Thinking budget for reasoning models: low/medium/high (supported by o3, o3-mini, GPT-Oss:20b, and other reasoning models)
+
+# Search Parameters
+--top-k N                    # Number of results to retrieve (default: 20)
+--search-complexity N        # Search complexity for graph traversal (default: 32)
+
+# Chunking Parameters
+--chunk-size N               # Size of text chunks (default varies by source: 256 for most, 192 for WeChat)
+--chunk-overlap N            # Overlap between chunks (default varies: 25-128 depending on source)
+
+# Index Building Parameters
+--backend-name NAME          # Backend to use: hnsw or diskann (default: hnsw)
+--graph-degree N             # Graph degree for index construction (default: 32)
+--build-complexity N         # Build complexity for index construction (default: 64)
+--compact / --no-compact     # Use compact storage (default: true). Must be `no-compact` for `no-recompute` build.
+--recompute / --no-recompute # Enable/disable embedding recomputation (default: enabled). Should not do a `no-recompute` search in a `recompute` build.
+```
+
+</details>
+
+### 📄 Personal Data Manager: Process Any Documents (`.pdf`, `.txt`, `.md`)!
+
+Ask questions directly about your personal PDFs, documents, and any directory containing your files!
+
 <p align="center">
-  <a href="#-quick-start">Quick Start</a> •
-  <a href="#-features">Features</a> •
-  <a href="#-benchmarks">Benchmarks</a> •
-  <a href="#-documentation">Documentation</a> •
-  <a href="#-paper">Paper</a>
+  <img src="videos/paper_clear.gif" alt="LEANN Document Search Demo" width="600">
 </p>
 
----
+The example below asks a question about summarizing our paper (uses default data in `data/`, which is a directory with diverse data sources: two papers, Pride and Prejudice, and a Technical report about LLM in Huawei in Chinese), and this is the **easiest example** to run here:
 
-## 🌟 What is Leann?
+```bash
+source .venv/bin/activate # Don't forget to activate the virtual environment
+python -m apps.document_rag --query "What are the main techniques LEANN explores?"
+```
 
-**Leann** revolutionizes Retrieval-Augmented Generation (RAG) by eliminating the storage bottleneck of traditional vector databases. Instead of pre-computing and storing billions of embeddings, Leann dynamically computes embeddings at query time using highly optimized graph-based search algorithms.
+<details>
+<summary><strong>📋 Click to expand: Document-Specific Arguments</strong></summary>
 
-### 🎯 Why Leann?
+#### Parameters
+```bash
+--data-dir DIR           # Directory containing documents to process (default: data)
+--file-types .ext .ext   # Filter by specific file types (optional - all LlamaIndex supported types if omitted)
+```
 
-Traditional RAG systems face a fundamental trade-off:
-- **💾 Storage**: Storing embeddings for millions of documents requires massive disk space
-- **🔄 Freshness**: Pre-computed embeddings become stale when documents change
-- **💰 Cost**: Vector databases are expensive to scale
+#### Example Commands
+```bash
+# Process all documents with larger chunks for academic papers
+python -m apps.document_rag --data-dir "~/Documents/Papers" --chunk-size 1024
 
-**Leann solves this by:**
-- ✅ **Zero embedding storage** - Only graph structure is persisted
-- ✅ **Real-time computation** - Embeddings computed on-demand with ms latency  
-- ✅ **Memory efficient** - Runs on consumer hardware (8GB RAM)
-- ✅ **Always fresh** - No stale embeddings, ever
+# Filter only markdown and Python files with smaller chunks
+python -m apps.document_rag --data-dir "./docs" --chunk-size 256 --file-types .md .py
 
-## 🚀 Quick Start
+# Enable AST-aware chunking for code files
+python -m apps.document_rag --enable-code-chunking --data-dir "./my_project"
+
+# Or use the specialized code RAG for better code understanding
+python -m apps.code_rag --repo-dir "./my_codebase" --query "How does authentication work?"
+```
+
+</details>
+
+### 📧 Your Personal Email Secretary: RAG on Apple Mail!
+
+> **Note:** The examples below currently support macOS only. Windows support coming soon.
+
+
+<p align="center">
+  <img src="videos/mail_clear.gif" alt="LEANN Email Search Demo" width="600">
+</p>
+
+Before running the example below, you need to grant full disk access to your terminal/VS Code in System Preferences → Privacy & Security → Full Disk Access.
+
+```bash
+python -m apps.email_rag --query "What's the food I ordered by DoorDash or Uber Eats mostly?"
+```
+**780K email chunks → 78MB storage.** Finally, search your email like you search Google.
+
+<details>
+<summary><strong>📋 Click to expand: Email-Specific Arguments</strong></summary>
+
+#### Parameters
+```bash
+--mail-path PATH         # Path to specific mail directory (auto-detects if omitted)
+--include-html          # Include HTML content in processing (useful for newsletters)
+```
+
+#### Example Commands
+```bash
+# Search work emails from a specific account
+python -m apps.email_rag --mail-path "~/Library/Mail/V10/WORK_ACCOUNT"
+
+# Find all receipts and order confirmations (includes HTML)
+python -m apps.email_rag --query "receipt order confirmation invoice" --include-html
+```
+
+</details>
+
+<details>
+<summary><strong>📋 Click to expand: Example queries you can try</strong></summary>
+
+Once the index is built, you can ask questions like:
+- "Find emails from my boss about deadlines"
+- "What did John say about the project timeline?"
+- "Show me emails about travel expenses"
+</details>
+
+### 🔍 Time Machine for the Web: RAG Your Entire Chrome Browser History!
+
+<p align="center">
+  <img src="videos/google_clear.gif" alt="LEANN Browser History Search Demo" width="600">
+</p>
+
+```bash
+python -m apps.browser_rag --query "Tell me my browser history about machine learning?"
+```
+**38K browser entries → 6MB storage.** Your browser history becomes your personal search engine.
+
+<details>
+<summary><strong>📋 Click to expand: Browser-Specific Arguments</strong></summary>
+
+#### Parameters
+```bash
+--chrome-profile PATH    # Path to Chrome profile directory (auto-detects if omitted)
+```
+
+#### Example Commands
+```bash
+# Search academic research from your browsing history
+python -m apps.browser_rag --query "arxiv papers machine learning transformer architecture"
+
+# Track competitor analysis across work profile
+python -m apps.browser_rag --chrome-profile "~/Library/Application Support/Google/Chrome/Work Profile" --max-items 5000
+```
+
+</details>
+
+<details>
+<summary><strong>📋 Click to expand: How to find your Chrome profile</strong></summary>
+
+The default Chrome profile path is configured for a typical macOS setup. If you need to find your specific Chrome profile:
+
+1. Open Terminal
+2. Run: `ls ~/Library/Application\ Support/Google/Chrome/`
+3. Look for folders like "Default", "Profile 1", "Profile 2", etc.
+4. Use the full path as your `--chrome-profile` argument
+
+**Common Chrome profile locations:**
+- macOS: `~/Library/Application Support/Google/Chrome/Default`
+- Linux: `~/.config/google-chrome/Default`
+
+</details>
+
+<details>
+<summary><strong>💬 Click to expand: Example queries you can try</strong></summary>
+
+Once the index is built, you can ask questions like:
+
+- "What websites did I visit about machine learning?"
+- "Find my search history about programming"
+- "What YouTube videos did I watch recently?"
+- "Show me websites I visited about travel planning"
+
+</details>
+
+### 💬 WeChat Detective: Unlock Your Golden Memories!
+
+<p align="center">
+  <img src="videos/wechat_clear.gif" alt="LEANN WeChat Search Demo" width="600">
+</p>
+
+```bash
+python -m apps.wechat_rag --query "Show me all group chats about weekend plans"
+```
+**400K messages → 64MB storage** Search years of chat history in any language.
+
+
+<details>
+<summary><strong>🔧 Click to expand: Installation Requirements</strong></summary>
+
+First, you need to install the [WeChat exporter](https://github.com/sunnyyoung/WeChatTweak-CLI),
+
+```bash
+brew install sunnyyoung/repo/wechattweak-cli
+```
+
+or install it manually (if you have issues with Homebrew):
+
+```bash
+sudo packages/wechat-exporter/wechattweak-cli install
+```
+
+**Troubleshooting:**
+- **Installation issues**: Check the [WeChatTweak-CLI issues page](https://github.com/sunnyyoung/WeChatTweak-CLI/issues/41)
+- **Export errors**: If you encounter the error below, try restarting WeChat
+  ```bash
+  Failed to export WeChat data. Please ensure WeChat is running and WeChatTweak is installed.
+  Failed to find or export WeChat data. Exiting.
+  ```
+</details>
+
+<details>
+<summary><strong>📋 Click to expand: WeChat-Specific Arguments</strong></summary>
+
+#### Parameters
+```bash
+--export-dir DIR         # Directory to store exported WeChat data (default: wechat_export_direct)
+--force-export          # Force re-export even if data exists
+```
+
+#### Example Commands
+```bash
+# Search for travel plans discussed in group chats
+python -m apps.wechat_rag --query "travel plans" --max-items 10000
+
+# Re-export and search recent chats (useful after new messages)
+python -m apps.wechat_rag --force-export --query "work schedule"
+```
+
+</details>
+
+<details>
+<summary><strong>💬 Click to expand: Example queries you can try</strong></summary>
+
+Once the index is built, you can ask questions like:
+
+- "我想买魔术师约翰逊的球衣，给我一些对应聊天记录?" (Chinese: Show me chat records about buying Magic Johnson's jersey)
+
+</details>
+
+### 🚀 Claude Code Integration: Transform Your Development Workflow!
+
+<details>
+<summary><strong>NEW!! AST‑Aware Code Chunking</strong></summary>
+
+LEANN features intelligent code chunking that preserves semantic boundaries (functions, classes, methods) for Python, Java, C#, and TypeScript, improving code understanding compared to text-based chunking.
+
+📖 Read the [AST Chunking Guide →](docs/ast_chunking_guide.md)
+
+</details>
+
+**The future of code assistance is here.** Transform your development workflow with LEANN's native MCP integration for Claude Code. Index your entire codebase and get intelligent code assistance directly in your IDE.
+
+**Key features:**
+- 🔍 **Semantic code search** across your entire project, fully local index and lightweight
+- 🧠 **AST-aware chunking** preserves code structure (functions, classes)
+- 📚 **Context-aware assistance** for debugging and development
+- 🚀 **Zero-config setup** with automatic language detection
+
+```bash
+# Install LEANN globally for MCP integration
+uv tool install leann-core --with leann
+claude mcp add --scope user leann-server -- leann_mcp
+# Setup is automatic - just start using Claude Code!
+```
+Try our fully agentic pipeline with auto query rewriting, semantic search planning, and more:
+
+![LEANN MCP Integration](assets/mcp_leann.png)
+
+**🔥 Ready to supercharge your coding?** [Complete Setup Guide →](packages/leann-mcp/README.md)
+
+## 🖥️ Command Line Interface
+
+LEANN includes a powerful CLI for document processing and search. Perfect for quick document indexing and interactive chat.
 
 ### Installation
 
+If you followed the Quick Start, `leann` is already installed in your virtual environment:
 ```bash
-git clone git@github.com:yichuan520030910320/LEANN-RAG.git leann
-cd leann
-git submodule update --init --recursive
-uv sync
+source .venv/bin/activate
+leann --help
 ```
 
-### 30-Second Example
+**To make it globally available:**
+```bash
+# Install the LEANN CLI globally using uv tool
+uv tool install leann-core --with leann
+
+
+# Now you can use leann from anywhere without activating venv
+leann --help
+```
+
+> **Note**: Global installation is required for Claude Code integration. The `leann_mcp` server depends on the globally available `leann` command.
+
+
+
+### Usage Examples
+
+```bash
+# build from a specific directory, and my_docs is the index name(Here you can also build from multiple dict or multiple files)
+leann build my-docs --docs ./your_documents
+
+# Search your documents
+leann search my-docs "machine learning concepts"
+
+# Interactive chat with your documents
+leann ask my-docs --interactive
+
+# List all your indexes
+leann list
+
+# Remove an index
+leann remove my-docs
+```
+
+**Key CLI features:**
+- Auto-detects document formats (PDF, TXT, MD, DOCX, PPTX + code files)
+- **🧠 AST-aware chunking** for Python, Java, C#, TypeScript files
+- Smart text chunking with overlap for all other content
+- Multiple LLM providers (Ollama, OpenAI, HuggingFace)
+- Organized index storage in `.leann/indexes/` (project-local)
+- Support for advanced search parameters
+
+<details>
+<summary><strong>📋 Click to expand: Complete CLI Reference</strong></summary>
+
+You can use `leann --help`, or `leann build --help`, `leann search --help`, `leann ask --help`, `leann list --help`, `leann remove --help` to get the complete CLI reference.
+
+**Build Command:**
+```bash
+leann build INDEX_NAME --docs DIRECTORY|FILE [DIRECTORY|FILE ...] [OPTIONS]
+
+Options:
+  --backend {hnsw,diskann}     Backend to use (default: hnsw)
+  --embedding-model MODEL      Embedding model (default: facebook/contriever)
+  --graph-degree N             Graph degree (default: 32)
+  --complexity N               Build complexity (default: 64)
+  --force                      Force rebuild existing index
+  --compact / --no-compact     Use compact storage (default: true). Must be `no-compact` for `no-recompute` build.
+  --recompute / --no-recompute Enable recomputation (default: true)
+```
+
+**Search Command:**
+```bash
+leann search INDEX_NAME QUERY [OPTIONS]
+
+Options:
+  --top-k N                     Number of results (default: 5)
+  --complexity N                Search complexity (default: 64)
+  --recompute / --no-recompute  Enable/disable embedding recomputation (default: enabled). Should not do a `no-recompute` search in a `recompute` build.
+  --pruning-strategy {global,local,proportional}
+```
+
+**Ask Command:**
+```bash
+leann ask INDEX_NAME [OPTIONS]
+
+Options:
+  --llm {ollama,openai,hf}    LLM provider (default: ollama)
+  --model MODEL               Model name (default: qwen3:8b)
+  --interactive              Interactive chat mode
+  --top-k N                  Retrieval count (default: 20)
+```
+
+**List Command:**
+```bash
+leann list
+
+# Lists all indexes across all projects with status indicators:
+# ✅ - Index is complete and ready to use
+# ❌ - Index is incomplete or corrupted
+# 📁 - CLI-created index (in .leann/indexes/)
+# 📄 - App-created index (*.leann.meta.json files)
+```
+
+**Remove Command:**
+```bash
+leann remove INDEX_NAME [OPTIONS]
+
+Options:
+  --force, -f    Force removal without confirmation
+
+# Smart removal: automatically finds and safely removes indexes
+# - Shows all matching indexes across projects
+# - Requires confirmation for cross-project removal
+# - Interactive selection when multiple matches found
+# - Supports both CLI and app-created indexes
+```
+
+</details>
+
+## 🚀 Advanced Features
+
+### 🎯 Metadata Filtering
+
+LEANN supports a simple metadata filtering system to enable sophisticated use cases like document filtering by date/type, code search by file extension, and content management based on custom criteria.
 
 ```python
-from leann.api import LeannBuilder, LeannSearcher
+# Add metadata during indexing
+builder.add_text(
+    "def authenticate_user(token): ...",
+    metadata={"file_extension": ".py", "lines_of_code": 25}
+)
 
-# 1. Build index (no embeddings stored!)
-builder = LeannBuilder(backend_name="diskann")
-builder.add_text("Python is a powerful programming language")
-builder.add_text("Machine learning transforms industries")  
-builder.add_text("Neural networks process complex data")
-builder.build_index("knowledge.leann")
-
-# 2. Search with real-time embeddings
-searcher = LeannSearcher("knowledge.leann")
-results = searcher.search("programming languages", top_k=2)
-
-for result in results:
-    print(f"Score: {result['score']:.3f} - {result['text']}")
+# Search with filters
+results = searcher.search(
+    query="authentication function",
+    metadata_filters={
+        "file_extension": {"==": ".py"},
+        "lines_of_code": {"<": 100}
+    }
+)
 ```
 
-### Run the Demo
+**Supported operators**: `==`, `!=`, `<`, `<=`, `>`, `>=`, `in`, `not_in`, `contains`, `starts_with`, `ends_with`, `is_true`, `is_false`
+
+📖 **[Complete Metadata filtering guide →](docs/metadata_filtering.md)**
+
+## 🏗️ Architecture & How It Works
+
+<p align="center">
+  <img src="assets/arch.png" alt="LEANN Architecture" width="800">
+</p>
+
+**The magic:** Most vector DBs store every single embedding (expensive). LEANN stores a pruned graph structure (cheap) and recomputes embeddings only when needed (fast).
+
+**Core techniques:**
+- **Graph-based selective recomputation:** Only compute embeddings for nodes in the search path
+- **High-degree preserving pruning:** Keep important "hub" nodes while removing redundant connections
+- **Dynamic batching:** Efficiently batch embedding computations for GPU utilization
+- **Two-level search:** Smart graph traversal that prioritizes promising nodes
+
+**Backends:**
+- **HNSW** (default): Ideal for most datasets with maximum storage savings through full recomputation
+- **DiskANN**: Advanced option with superior search performance, using PQ-based graph traversal with real-time reranking for the best speed-accuracy trade-off
+
+## Benchmarks
+
+**[DiskANN vs HNSW Performance Comparison →](benchmarks/diskann_vs_hnsw_speed_comparison.py)** - Compare search performance between both backends
+
+**[Simple Example: Compare LEANN vs FAISS →](benchmarks/compare_faiss_vs_leann.py)** - See storage savings in action
+
+### 📊 Storage Comparison
+
+| System | DPR (2.1M) | Wiki (60M) | Chat (400K) | Email (780K) | Browser (38K) |
+|--------|-------------|------------|-------------|--------------|---------------|
+| Traditional vector database (e.g., FAISS) | 3.8 GB      | 201 GB     | 1.8 GB     | 2.4 GB      | 130 MB        |
+| LEANN  | 324 MB      | 6 GB       | 64 MB       | 79 MB       | 6.4 MB        |
+| Savings| 91%         | 97%        | 97%         | 97%         | 95%           |
+
+
+
+## Reproduce Our Results
 
 ```bash
-uv run examples/document_search.py
+uv pip install -e ".[dev]"  # Install dev dependencies
+python benchmarks/run_evaluation.py    # Will auto-download evaluation data and run benchmarks
+python benchmarks/run_evaluation.py benchmarks/data/indices/rpj_wiki/rpj_wiki --num-queries 2000    # After downloading data, you can run the benchmark with our biggest index
 ```
 
-**PDF RAG Demo (using LlamaIndex for document parsing and Leann for indexing/search)**
-
-This demo showcases how to build a RAG system for PDF documents using Leann.
-1.  Place your PDF files (and other supported formats like .docx, .pptx, .xlsx) into the `examples/data/` directory.
-2.  Ensure you have an `OPENAI_API_KEY` set in your environment variables or in a `.env` file for the LLM to function.
-
-```bash
-uv run examples/main_cli_example.py
-```
-
-## ✨ Features
-
-### 🔥 Core Features
-- **📊 Multiple Distance Functions**: L2, Cosine, MIPS (Maximum Inner Product Search)
-- **🏗️ Pluggable Backends**: DiskANN, HNSW/FAISS with unified API
-- **🔄 Real-time Embeddings**: Dynamic computation using optimized ZMQ servers
-- **📈 Scalable Architecture**: Handles millions of documents on consumer hardware
-- **🎯 Graph Pruning**: Advanced techniques for memory-efficient search
-
-### 🛠️ Technical Highlights
-- **Zero-copy operations** for maximum performance
-- **SIMD-optimized** distance computations (AVX2/AVX512)
-- **Async embedding pipeline** with batched processing
-- **Memory-mapped indices** for fast startup
-- **Recompute mode** for highest accuracy scenarios
-
-### 🎨 Developer Experience
-- **Simple Python API** - Get started in minutes
-- **Extensible backend system** - Easy to add new algorithms
-- **Comprehensive examples** - From basic usage to production deployment
-- **Rich debugging tools** - Built-in performance profiling
-
-## 📊 Benchmarks
-
-### Memory Usage Comparison
-
-| System | 1M Documents | 10M Documents | 100M Documents |
-|--------|-------------|---------------|----------------|
-| Traditional Vector DB | 3.1 GB | 31 GB | 310 GB |
-| **Leann** | **180 MB** | **1.2 GB** | **8.4 GB** |
-| **Reduction** | **94.2%** | **96.1%** | **97.3%** |
-
-### Query Performance
-
-| Backend | Index Size | Query Time | Recall@10 |
-|---------|------------|------------|-----------|
-| DiskANN | 1M docs | 12ms | 0.95 |
-| DiskANN + Recompute | 1M docs | 145ms | 0.98 |
-| HNSW | 1M docs | 8ms | 0.93 |
-
-*Benchmarks run on AMD Ryzen 7 with 32GB RAM*
-
-## 🏗️ Architecture
-
-```
-┌─────────────────┐    ┌──────────────────┐    ┌─────────────────┐
-│   Query Text    │───▶│  Embedding       │───▶│   Graph-based   │
-│                 │    │  Computation     │    │     Search      │
-└─────────────────┘    └──────────────────┘    └─────────────────┘
-                              │                         │
-                              ▼                         ▼
-                       ┌──────────────┐         ┌──────────────┐
-                       │ ZMQ Server   │         │ Pruned Graph │
-                       │ (Cached)     │         │ Index        │
-                       └──────────────┘         └──────────────┘
-```
-
-### Key Components
-
-1. **🧠 Embedding Engine**: Real-time transformer inference with caching
-2. **📊 Graph Index**: Memory-efficient navigation structures  
-3. **🔄 Search Coordinator**: Orchestrates embedding + graph search
-4. **⚡ Backend Adapters**: Pluggable algorithm implementations
-
-## 🎓 Supported Models & Backends
-
-### 🤖 Embedding Models
-- **sentence-transformers/all-mpnet-base-v2** (default)
-- **sentence-transformers/all-MiniLM-L6-v2** (lightweight)
-- Any HuggingFace sentence-transformer model
-- Custom model support via API
-
-### 🔧 Search Backends  
-- **DiskANN**: Microsoft's billion-scale ANN algorithm
-- **HNSW**: Hierarchical Navigable Small World graphs
-- **Coming soon**: ScaNN, Faiss-IVF, NGT
-
-### 📏 Distance Functions
-- **L2**: Euclidean distance for precise similarity
-- **Cosine**: Angular similarity for normalized vectors  
-- **MIPS**: Maximum Inner Product Search for recommendation systems
-
+The evaluation script downloads data automatically on first run. The last three results were tested with partial personal data, and you can reproduce them with your own data!
 ## 🔬 Paper
 
 If you find Leann useful, please cite:
@@ -185,110 +715,15 @@ If you find Leann useful, please cite:
 }
 ```
 
-## 🌍 Use Cases
+## ✨ [Detailed Features →](docs/features.md)
 
-### 💼 Enterprise RAG
-```python
-# Handle millions of documents with limited resources
-builder = LeannBuilder(
-    backend_name="diskann",
-    distance_metric="cosine",
-    graph_degree=64,
-    memory_budget="4GB"
-)
-```
-
-### 🔬 Research & Experimentation  
-```python
-# Quick prototyping with different algorithms
-for backend in ["diskann", "hnsw"]:
-    searcher = LeannSearcher(index_path, backend=backend)
-    evaluate_recall(searcher, queries, ground_truth)
-```
-
-### 🚀 Real-time Applications
-```python
-# Sub-second response times
-chat = LeannChat("knowledge.leann")
-response = chat.ask("What is quantum computing?")
-# Returns in <100ms with recompute mode
-```
-
-## 🤝 Contributing
-
-We welcome contributions! Leann is built by the community, for the community.
-
-### Ways to Contribute
-- 🐛 **Bug Reports**: Found an issue? Let us know!
-- 💡 **Feature Requests**: Have an idea? We'd love to hear it!
-- 🔧 **Code Contributions**: PRs welcome for all skill levels
-- 📖 **Documentation**: Help make Leann more accessible
-- 🧪 **Benchmarks**: Share your performance results
-
-### Development Setup
-```bash
-git clone https://github.com/yourname/leann
-cd leann
-uv sync --dev
-uv run pytest tests/
-```
-
-### Quick Tests
-```bash
-# Sanity check all distance functions
-uv run python tests/sanity_checks/test_distance_functions.py
-
-# Verify L2 implementation
-uv run python tests/sanity_checks/test_l2_verification.py
-```
-## ❓ FAQ
-
-### Common Issues
-
-#### NCCL Topology Error
-**Problem**: You encounter `ncclTopoComputePaths` error during document processing:
-```
-ncclTopoComputePaths (system=<optimized out>, comm=comm@entry=0x5555a82fa3c0) at graph/paths.cc:688
-```
-
-**Solution**: Set these environment variables before running your script:
-```bash
-export NCCL_TOPO_DUMP_FILE=/tmp/nccl_topo.xml
-export NCCL_DEBUG=INFO
-export NCCL_DEBUG_SUBSYS=INIT,GRAPH
-export NCCL_IB_DISABLE=1
-export NCCL_NET_PLUGIN=none
-export NCCL_SOCKET_IFNAME=ens5
+## 🤝 [CONTRIBUTING →](docs/CONTRIBUTING.md)
 
 
-## 📈 Roadmap
+## ❓ [FAQ →](docs/faq.md)
 
-### 🎯 Q1 2024
-- [x] DiskANN backend with MIPS/L2/Cosine support
-- [x] HNSW backend integration  
-- [x] Real-time embedding pipeline
-- [x] Memory-efficient graph pruning
 
-### 🚀 Q2 2024
-- [ ] Distributed search across multiple nodes
-- [ ] ScaNN backend support
-- [ ] Advanced caching strategies
-- [ ] Kubernetes deployment guides
-
-### 🌟 Q3 2024
-- [ ] GPU-accelerated embedding computation
-- [ ] Approximate distance functions
-- [ ] Integration with LangChain/LlamaIndex
-- [ ] Visual similarity search
-
-## 💬 Community
-
-Join our growing community of researchers and engineers!
-
-- 🐦 **Twitter**: [@LeannAI](https://twitter.com/LeannAI)
-- 💬 **Discord**: [Join our server](https://discord.gg/leann)
-- 📧 **Email**: leann@yourcompany.com
-- 🐙 **GitHub Discussions**: [Ask questions here](https://github.com/yourname/leann/discussions)
+## 📈 [Roadmap →](docs/roadmap.md)
 
 ## 📄 License
 
@@ -296,13 +731,18 @@ MIT License - see [LICENSE](LICENSE) for details.
 
 ## 🙏 Acknowledgments
 
-- **Microsoft Research** for the DiskANN algorithm
-- **Meta AI** for FAISS and optimization insights  
-- **HuggingFace** for the transformer ecosystem
-- **Our amazing contributors** who make this possible
+Core Contributors: [Yichuan Wang](https://yichuan-w.github.io/) & [Zhifei Li](https://github.com/andylizf).
 
----
+Active Contributors: [Gabriel Dehan](https://github.com/gabriel-dehan)
 
+
+We welcome more contributors! Feel free to open issues or submit PRs.
+
+This work is done at [**Berkeley Sky Computing Lab**](https://sky.cs.berkeley.edu/).
+
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=yichuan-w/LEANN&type=Date)](https://www.star-history.com/#yichuan-w/LEANN&Date)
 <p align="center">
   <strong>⭐ Star us on GitHub if Leann is useful for your research or applications!</strong>
 </p>

apps/base_rag_example.py

@@ -0,0 +1,342 @@
+"""
+Base class for unified RAG examples interface.
+Provides common parameters and functionality for all RAG examples.
+"""
+
+import argparse
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Any
+
+import dotenv
+from leann.api import LeannBuilder, LeannChat
+from leann.registry import register_project_directory
+
+dotenv.load_dotenv()
+
+
+class BaseRAGExample(ABC):
+    """Base class for all RAG examples with unified interface."""
+
+    def __init__(
+        self,
+        name: str,
+        description: str,
+        default_index_name: str,
+    ):
+        self.name = name
+        self.description = description
+        self.default_index_name = default_index_name
+        self.parser = self._create_parser()
+
+    def _create_parser(self) -> argparse.ArgumentParser:
+        """Create argument parser with common parameters."""
+        parser = argparse.ArgumentParser(
+            description=self.description, formatter_class=argparse.RawDescriptionHelpFormatter
+        )
+
+        # Core parameters (all examples share these)
+        core_group = parser.add_argument_group("Core Parameters")
+        core_group.add_argument(
+            "--index-dir",
+            type=str,
+            default=f"./{self.default_index_name}",
+            help=f"Directory to store the index (default: ./{self.default_index_name})",
+        )
+        core_group.add_argument(
+            "--query",
+            type=str,
+            default=None,
+            help="Query to run (if not provided, will run in interactive mode)",
+        )
+        # Allow subclasses to override default max_items
+        max_items_default = getattr(self, "max_items_default", -1)
+        core_group.add_argument(
+            "--max-items",
+            type=int,
+            default=max_items_default,
+            help="Maximum number of items to process  -1 for all, means index all documents, and you should set it to a reasonable number if you have a large dataset and try at the first time)",
+        )
+        core_group.add_argument(
+            "--force-rebuild", action="store_true", help="Force rebuild index even if it exists"
+        )
+
+        # Embedding parameters
+        embedding_group = parser.add_argument_group("Embedding Parameters")
+        # Allow subclasses to override default embedding_model
+        embedding_model_default = getattr(self, "embedding_model_default", "facebook/contriever")
+        embedding_group.add_argument(
+            "--embedding-model",
+            type=str,
+            default=embedding_model_default,
+            help=f"Embedding model to use (default: {embedding_model_default}), we provide facebook/contriever, text-embedding-3-small,mlx-community/Qwen3-Embedding-0.6B-8bit or nomic-embed-text",
+        )
+        embedding_group.add_argument(
+            "--embedding-mode",
+            type=str,
+            default="sentence-transformers",
+            choices=["sentence-transformers", "openai", "mlx", "ollama"],
+            help="Embedding backend mode (default: sentence-transformers), we provide sentence-transformers, openai, mlx, or ollama",
+        )
+
+        # LLM parameters
+        llm_group = parser.add_argument_group("LLM Parameters")
+        llm_group.add_argument(
+            "--llm",
+            type=str,
+            default="openai",
+            choices=["openai", "ollama", "hf", "simulated"],
+            help="LLM backend: openai, ollama, or hf (default: openai)",
+        )
+        llm_group.add_argument(
+            "--llm-model",
+            type=str,
+            default=None,
+            help="Model name (default: gpt-4o) e.g., gpt-4o-mini, llama3.2:1b, Qwen/Qwen2.5-1.5B-Instruct",
+        )
+        llm_group.add_argument(
+            "--llm-host",
+            type=str,
+            default="http://localhost:11434",
+            help="Host for Ollama API (default: http://localhost:11434)",
+        )
+        llm_group.add_argument(
+            "--thinking-budget",
+            type=str,
+            choices=["low", "medium", "high"],
+            default=None,
+            help="Thinking budget for reasoning models (low/medium/high). Supported by GPT-Oss:20b and other reasoning models.",
+        )
+
+        # AST Chunking parameters
+        ast_group = parser.add_argument_group("AST Chunking Parameters")
+        ast_group.add_argument(
+            "--use-ast-chunking",
+            action="store_true",
+            help="Enable AST-aware chunking for code files (requires astchunk)",
+        )
+        ast_group.add_argument(
+            "--ast-chunk-size",
+            type=int,
+            default=512,
+            help="Maximum characters per AST chunk (default: 512)",
+        )
+        ast_group.add_argument(
+            "--ast-chunk-overlap",
+            type=int,
+            default=64,
+            help="Overlap between AST chunks (default: 64)",
+        )
+        ast_group.add_argument(
+            "--code-file-extensions",
+            nargs="+",
+            default=None,
+            help="Additional code file extensions to process with AST chunking (e.g., .py .java .cs .ts)",
+        )
+        ast_group.add_argument(
+            "--ast-fallback-traditional",
+            action="store_true",
+            default=True,
+            help="Fall back to traditional chunking if AST chunking fails (default: True)",
+        )
+
+        # Search parameters
+        search_group = parser.add_argument_group("Search Parameters")
+        search_group.add_argument(
+            "--top-k", type=int, default=20, help="Number of results to retrieve (default: 20)"
+        )
+        search_group.add_argument(
+            "--search-complexity",
+            type=int,
+            default=32,
+            help="Search complexity for graph traversal (default: 64)",
+        )
+
+        # Index building parameters
+        index_group = parser.add_argument_group("Index Building Parameters")
+        index_group.add_argument(
+            "--backend-name",
+            type=str,
+            default="hnsw",
+            choices=["hnsw", "diskann"],
+            help="Backend to use for index (default: hnsw)",
+        )
+        index_group.add_argument(
+            "--graph-degree",
+            type=int,
+            default=32,
+            help="Graph degree for index construction (default: 32)",
+        )
+        index_group.add_argument(
+            "--build-complexity",
+            type=int,
+            default=64,
+            help="Build complexity for index construction (default: 64)",
+        )
+        index_group.add_argument(
+            "--no-compact",
+            action="store_true",
+            help="Disable compact index storage",
+        )
+        index_group.add_argument(
+            "--no-recompute",
+            action="store_true",
+            help="Disable embedding recomputation",
+        )
+
+        # Add source-specific parameters
+        self._add_specific_arguments(parser)
+
+        return parser
+
+    @abstractmethod
+    def _add_specific_arguments(self, parser: argparse.ArgumentParser):
+        """Add source-specific arguments. Override in subclasses."""
+        pass
+
+    @abstractmethod
+    async def load_data(self, args) -> list[str]:
+        """Load data from the source. Returns list of text chunks."""
+        pass
+
+    def get_llm_config(self, args) -> dict[str, Any]:
+        """Get LLM configuration based on arguments."""
+        config = {"type": args.llm}
+
+        if args.llm == "openai":
+            config["model"] = args.llm_model or "gpt-4o"
+        elif args.llm == "ollama":
+            config["model"] = args.llm_model or "llama3.2:1b"
+            config["host"] = args.llm_host
+        elif args.llm == "hf":
+            config["model"] = args.llm_model or "Qwen/Qwen2.5-1.5B-Instruct"
+        elif args.llm == "simulated":
+            # Simulated LLM doesn't need additional configuration
+            pass
+
+        return config
+
+    async def build_index(self, args, texts: list[str]) -> str:
+        """Build LEANN index from texts."""
+        index_path = str(Path(args.index_dir) / f"{self.default_index_name}.leann")
+
+        print(f"\n[Building Index] Creating {self.name} index...")
+        print(f"Total text chunks: {len(texts)}")
+
+        builder = LeannBuilder(
+            backend_name=args.backend_name,
+            embedding_model=args.embedding_model,
+            embedding_mode=args.embedding_mode,
+            graph_degree=args.graph_degree,
+            complexity=args.build_complexity,
+            is_compact=not args.no_compact,
+            is_recompute=not args.no_recompute,
+            num_threads=1,  # Force single-threaded mode
+        )
+
+        # Add texts in batches for better progress tracking
+        batch_size = 1000
+        for i in range(0, len(texts), batch_size):
+            batch = texts[i : i + batch_size]
+            for text in batch:
+                builder.add_text(text)
+            print(f"Added {min(i + batch_size, len(texts))}/{len(texts)} texts...")
+
+        print("Building index structure...")
+        builder.build_index(index_path)
+        print(f"Index saved to: {index_path}")
+
+        # Register project directory so leann list can discover this index
+        # The index is saved as args.index_dir/index_name.leann
+        # We want to register the current working directory where the app is run
+        register_project_directory(Path.cwd())
+
+        return index_path
+
+    async def run_interactive_chat(self, args, index_path: str):
+        """Run interactive chat with the index."""
+        chat = LeannChat(
+            index_path,
+            llm_config=self.get_llm_config(args),
+            system_prompt=f"You are a helpful assistant that answers questions about {self.name} data.",
+            complexity=args.search_complexity,
+        )
+
+        print(f"\n[Interactive Mode] Chat with your {self.name} data!")
+        print("Type 'quit' or 'exit' to stop.\n")
+
+        while True:
+            try:
+                query = input("You: ").strip()
+                if query.lower() in ["quit", "exit", "q"]:
+                    print("Goodbye!")
+                    break
+
+                if not query:
+                    continue
+
+                # Prepare LLM kwargs with thinking budget if specified
+                llm_kwargs = {}
+                if hasattr(args, "thinking_budget") and args.thinking_budget:
+                    llm_kwargs["thinking_budget"] = args.thinking_budget
+
+                response = chat.ask(
+                    query,
+                    top_k=args.top_k,
+                    complexity=args.search_complexity,
+                    llm_kwargs=llm_kwargs,
+                )
+                print(f"\nAssistant: {response}\n")
+
+            except KeyboardInterrupt:
+                print("\nGoodbye!")
+                break
+            except Exception as e:
+                print(f"Error: {e}")
+
+    async def run_single_query(self, args, index_path: str, query: str):
+        """Run a single query against the index."""
+        chat = LeannChat(
+            index_path,
+            llm_config=self.get_llm_config(args),
+            complexity=args.search_complexity,
+        )
+
+        print(f"\n[Query]: \033[36m{query}\033[0m")
+
+        # Prepare LLM kwargs with thinking budget if specified
+        llm_kwargs = {}
+        if hasattr(args, "thinking_budget") and args.thinking_budget:
+            llm_kwargs["thinking_budget"] = args.thinking_budget
+
+        response = chat.ask(
+            query, top_k=args.top_k, complexity=args.search_complexity, llm_kwargs=llm_kwargs
+        )
+        print(f"\n[Response]: \033[36m{response}\033[0m")
+
+    async def run(self):
+        """Main entry point for the example."""
+        args = self.parser.parse_args()
+
+        # Check if index exists
+        index_path = str(Path(args.index_dir) / f"{self.default_index_name}.leann")
+        index_exists = Path(args.index_dir).exists()
+
+        if not index_exists or args.force_rebuild:
+            # Load data and build index
+            print(f"\n{'Rebuilding' if index_exists else 'Building'} index...")
+            texts = await self.load_data(args)
+
+            if not texts:
+                print("No data found to index!")
+                return
+
+            index_path = await self.build_index(args, texts)
+        else:
+            print(f"\nUsing existing index in {args.index_dir}")
+
+        # Run query or interactive mode
+        if args.query:
+            await self.run_single_query(args, index_path, args.query)
+        else:
+            await self.run_interactive_chat(args, index_path)

apps/browser_rag.py

@@ -0,0 +1,170 @@
+"""
+Browser History RAG example using the unified interface.
+Supports Chrome browser history.
+"""
+
+import os
+import sys
+from pathlib import Path
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent))
+
+from base_rag_example import BaseRAGExample, create_text_chunks
+
+from .history_data.history import ChromeHistoryReader
+
+
+class BrowserRAG(BaseRAGExample):
+    """RAG example for Chrome browser history."""
+
+    def __init__(self):
+        # Set default values BEFORE calling super().__init__
+        self.embedding_model_default = (
+            "sentence-transformers/all-MiniLM-L6-v2"  # Fast 384-dim model
+        )
+
+        super().__init__(
+            name="Browser History",
+            description="Process and query Chrome browser history with LEANN",
+            default_index_name="google_history_index",
+        )
+
+    def _add_specific_arguments(self, parser):
+        """Add browser-specific arguments."""
+        browser_group = parser.add_argument_group("Browser Parameters")
+        browser_group.add_argument(
+            "--chrome-profile",
+            type=str,
+            default=None,
+            help="Path to Chrome profile directory (auto-detected if not specified)",
+        )
+        browser_group.add_argument(
+            "--auto-find-profiles",
+            action="store_true",
+            default=True,
+            help="Automatically find all Chrome profiles (default: True)",
+        )
+        browser_group.add_argument(
+            "--chunk-size", type=int, default=256, help="Text chunk size (default: 256)"
+        )
+        browser_group.add_argument(
+            "--chunk-overlap", type=int, default=128, help="Text chunk overlap (default: 128)"
+        )
+
+    def _get_chrome_base_path(self) -> Path:
+        """Get the base Chrome profile path based on OS."""
+        if sys.platform == "darwin":
+            return Path.home() / "Library" / "Application Support" / "Google" / "Chrome"
+        elif sys.platform.startswith("linux"):
+            return Path.home() / ".config" / "google-chrome"
+        elif sys.platform == "win32":
+            return Path(os.environ["LOCALAPPDATA"]) / "Google" / "Chrome" / "User Data"
+        else:
+            raise ValueError(f"Unsupported platform: {sys.platform}")
+
+    def _find_chrome_profiles(self) -> list[Path]:
+        """Auto-detect all Chrome profiles."""
+        base_path = self._get_chrome_base_path()
+        if not base_path.exists():
+            return []
+
+        profiles = []
+
+        # Check Default profile
+        default_profile = base_path / "Default"
+        if default_profile.exists() and (default_profile / "History").exists():
+            profiles.append(default_profile)
+
+        # Check numbered profiles
+        for item in base_path.iterdir():
+            if item.is_dir() and item.name.startswith("Profile "):
+                if (item / "History").exists():
+                    profiles.append(item)
+
+        return profiles
+
+    async def load_data(self, args) -> list[str]:
+        """Load browser history and convert to text chunks."""
+        # Determine Chrome profiles
+        if args.chrome_profile and not args.auto_find_profiles:
+            profile_dirs = [Path(args.chrome_profile)]
+        else:
+            print("Auto-detecting Chrome profiles...")
+            profile_dirs = self._find_chrome_profiles()
+
+            # If specific profile given, filter to just that one
+            if args.chrome_profile:
+                profile_path = Path(args.chrome_profile)
+                profile_dirs = [p for p in profile_dirs if p == profile_path]
+
+        if not profile_dirs:
+            print("No Chrome profiles found!")
+            print("Please specify --chrome-profile manually")
+            return []
+
+        print(f"Found {len(profile_dirs)} Chrome profiles")
+
+        # Create reader
+        reader = ChromeHistoryReader()
+
+        # Process each profile
+        all_documents = []
+        total_processed = 0
+
+        for i, profile_dir in enumerate(profile_dirs):
+            print(f"\nProcessing profile {i + 1}/{len(profile_dirs)}: {profile_dir.name}")
+
+            try:
+                # Apply max_items limit per profile
+                max_per_profile = -1
+                if args.max_items > 0:
+                    remaining = args.max_items - total_processed
+                    if remaining <= 0:
+                        break
+                    max_per_profile = remaining
+
+                # Load history
+                documents = reader.load_data(
+                    chrome_profile_path=str(profile_dir),
+                    max_count=max_per_profile,
+                )
+
+                if documents:
+                    all_documents.extend(documents)
+                    total_processed += len(documents)
+                    print(f"Processed {len(documents)} history entries from this profile")
+
+            except Exception as e:
+                print(f"Error processing {profile_dir}: {e}")
+                continue
+
+        if not all_documents:
+            print("No browser history found to process!")
+            return []
+
+        print(f"\nTotal history entries processed: {len(all_documents)}")
+
+        # Convert to text chunks
+        all_texts = create_text_chunks(
+            all_documents, chunk_size=args.chunk_size, chunk_overlap=args.chunk_overlap
+        )
+
+        return all_texts
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    # Example queries for browser history RAG
+    print("\n🌐 Browser History RAG Example")
+    print("=" * 50)
+    print("\nExample queries you can try:")
+    print("- 'What websites did I visit about machine learning?'")
+    print("- 'Find my search history about programming'")
+    print("- 'What YouTube videos did I watch recently?'")
+    print("- 'Show me websites about travel planning'")
+    print("\nNote: Make sure Chrome is closed before running\n")
+
+    rag = BrowserRAG()
+    asyncio.run(rag.run())

apps/chunking/__init__.py

@@ -0,0 +1,22 @@
+"""
+Chunking utilities for LEANN RAG applications.
+Provides AST-aware and traditional text chunking functionality.
+"""
+
+from .utils import (
+    CODE_EXTENSIONS,
+    create_ast_chunks,
+    create_text_chunks,
+    create_traditional_chunks,
+    detect_code_files,
+    get_language_from_extension,
+)
+
+__all__ = [
+    "CODE_EXTENSIONS",
+    "create_ast_chunks",
+    "create_text_chunks",
+    "create_traditional_chunks",
+    "detect_code_files",
+    "get_language_from_extension",
+]

apps/chunking/utils.py

@@ -0,0 +1,320 @@
+"""
+Enhanced chunking utilities with AST-aware code chunking support.
+Provides unified interface for both traditional and AST-based text chunking.
+"""
+
+import logging
+from pathlib import Path
+from typing import Optional
+
+from llama_index.core.node_parser import SentenceSplitter
+
+logger = logging.getLogger(__name__)
+
+# Code file extensions supported by astchunk
+CODE_EXTENSIONS = {
+    ".py": "python",
+    ".java": "java",
+    ".cs": "csharp",
+    ".ts": "typescript",
+    ".tsx": "typescript",
+    ".js": "typescript",
+    ".jsx": "typescript",
+}
+
+# Default chunk parameters for different content types
+DEFAULT_CHUNK_PARAMS = {
+    "code": {
+        "max_chunk_size": 512,
+        "chunk_overlap": 64,
+    },
+    "text": {
+        "chunk_size": 256,
+        "chunk_overlap": 128,
+    },
+}
+
+
+def detect_code_files(documents, code_extensions=None) -> tuple[list, list]:
+    """
+    Separate documents into code files and regular text files.
+
+    Args:
+        documents: List of LlamaIndex Document objects
+        code_extensions: Dict mapping file extensions to languages (defaults to CODE_EXTENSIONS)
+
+    Returns:
+        Tuple of (code_documents, text_documents)
+    """
+    if code_extensions is None:
+        code_extensions = CODE_EXTENSIONS
+
+    code_docs = []
+    text_docs = []
+
+    for doc in documents:
+        # Get file path from metadata
+        file_path = doc.metadata.get("file_path", "")
+        if not file_path:
+            # Fallback to file_name
+            file_path = doc.metadata.get("file_name", "")
+
+        if file_path:
+            file_ext = Path(file_path).suffix.lower()
+            if file_ext in code_extensions:
+                # Add language info to metadata
+                doc.metadata["language"] = code_extensions[file_ext]
+                doc.metadata["is_code"] = True
+                code_docs.append(doc)
+            else:
+                doc.metadata["is_code"] = False
+                text_docs.append(doc)
+        else:
+            # If no file path, treat as text
+            doc.metadata["is_code"] = False
+            text_docs.append(doc)
+
+    logger.info(f"Detected {len(code_docs)} code files and {len(text_docs)} text files")
+    return code_docs, text_docs
+
+
+def get_language_from_extension(file_path: str) -> Optional[str]:
+    """Get the programming language from file extension."""
+    ext = Path(file_path).suffix.lower()
+    return CODE_EXTENSIONS.get(ext)
+
+
+def create_ast_chunks(
+    documents,
+    max_chunk_size: int = 512,
+    chunk_overlap: int = 64,
+    metadata_template: str = "default",
+) -> list[str]:
+    """
+    Create AST-aware chunks from code documents using astchunk.
+
+    Args:
+        documents: List of code documents
+        max_chunk_size: Maximum characters per chunk
+        chunk_overlap: Number of AST nodes to overlap between chunks
+        metadata_template: Template for chunk metadata
+
+    Returns:
+        List of text chunks with preserved code structure
+    """
+    try:
+        from astchunk import ASTChunkBuilder
+    except ImportError as e:
+        logger.error(f"astchunk not available: {e}")
+        logger.info("Falling back to traditional chunking for code files")
+        return create_traditional_chunks(documents, max_chunk_size, chunk_overlap)
+
+    all_chunks = []
+
+    for doc in documents:
+        # Get language from metadata (set by detect_code_files)
+        language = doc.metadata.get("language")
+        if not language:
+            logger.warning(
+                "No language detected for document, falling back to traditional chunking"
+            )
+            traditional_chunks = create_traditional_chunks([doc], max_chunk_size, chunk_overlap)
+            all_chunks.extend(traditional_chunks)
+            continue
+
+        try:
+            # Configure astchunk
+            configs = {
+                "max_chunk_size": max_chunk_size,
+                "language": language,
+                "metadata_template": metadata_template,
+                "chunk_overlap": chunk_overlap if chunk_overlap > 0 else 0,
+            }
+
+            # Add repository-level metadata if available
+            repo_metadata = {
+                "file_path": doc.metadata.get("file_path", ""),
+                "file_name": doc.metadata.get("file_name", ""),
+                "creation_date": doc.metadata.get("creation_date", ""),
+                "last_modified_date": doc.metadata.get("last_modified_date", ""),
+            }
+            configs["repo_level_metadata"] = repo_metadata
+
+            # Create chunk builder and process
+            chunk_builder = ASTChunkBuilder(**configs)
+            code_content = doc.get_content()
+
+            if not code_content or not code_content.strip():
+                logger.warning("Empty code content, skipping")
+                continue
+
+            chunks = chunk_builder.chunkify(code_content)
+
+            # Extract text content from chunks
+            for chunk in chunks:
+                if hasattr(chunk, "text"):
+                    chunk_text = chunk.text
+                elif isinstance(chunk, dict) and "text" in chunk:
+                    chunk_text = chunk["text"]
+                elif isinstance(chunk, str):
+                    chunk_text = chunk
+                else:
+                    # Try to convert to string
+                    chunk_text = str(chunk)
+
+                if chunk_text and chunk_text.strip():
+                    all_chunks.append(chunk_text.strip())
+
+            logger.info(
+                f"Created {len(chunks)} AST chunks from {language} file: {doc.metadata.get('file_name', 'unknown')}"
+            )
+
+        except Exception as e:
+            logger.warning(f"AST chunking failed for {language} file: {e}")
+            logger.info("Falling back to traditional chunking")
+            traditional_chunks = create_traditional_chunks([doc], max_chunk_size, chunk_overlap)
+            all_chunks.extend(traditional_chunks)
+
+    return all_chunks
+
+
+def create_traditional_chunks(
+    documents, chunk_size: int = 256, chunk_overlap: int = 128
+) -> list[str]:
+    """
+    Create traditional text chunks using LlamaIndex SentenceSplitter.
+
+    Args:
+        documents: List of documents to chunk
+        chunk_size: Size of each chunk in characters
+        chunk_overlap: Overlap between chunks
+
+    Returns:
+        List of text chunks
+    """
+    # Handle invalid chunk_size values
+    if chunk_size <= 0:
+        logger.warning(f"Invalid chunk_size={chunk_size}, using default value of 256")
+        chunk_size = 256
+
+    # Ensure chunk_overlap is not negative and not larger than chunk_size
+    if chunk_overlap < 0:
+        chunk_overlap = 0
+    if chunk_overlap >= chunk_size:
+        chunk_overlap = chunk_size // 2
+
+    node_parser = SentenceSplitter(
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+        separator=" ",
+        paragraph_separator="\n\n",
+    )
+
+    all_texts = []
+    for doc in documents:
+        try:
+            nodes = node_parser.get_nodes_from_documents([doc])
+            if nodes:
+                chunk_texts = [node.get_content() for node in nodes]
+                all_texts.extend(chunk_texts)
+                logger.debug(f"Created {len(chunk_texts)} traditional chunks from document")
+        except Exception as e:
+            logger.error(f"Traditional chunking failed for document: {e}")
+            # As last resort, add the raw content
+            content = doc.get_content()
+            if content and content.strip():
+                all_texts.append(content.strip())
+
+    return all_texts
+
+
+def create_text_chunks(
+    documents,
+    chunk_size: int = 256,
+    chunk_overlap: int = 128,
+    use_ast_chunking: bool = False,
+    ast_chunk_size: int = 512,
+    ast_chunk_overlap: int = 64,
+    code_file_extensions: Optional[list[str]] = None,
+    ast_fallback_traditional: bool = True,
+) -> list[str]:
+    """
+    Create text chunks from documents with optional AST support for code files.
+
+    Args:
+        documents: List of LlamaIndex Document objects
+        chunk_size: Size for traditional text chunks
+        chunk_overlap: Overlap for traditional text chunks
+        use_ast_chunking: Whether to use AST chunking for code files
+        ast_chunk_size: Size for AST chunks
+        ast_chunk_overlap: Overlap for AST chunks
+        code_file_extensions: Custom list of code file extensions
+        ast_fallback_traditional: Fall back to traditional chunking on AST errors
+
+    Returns:
+        List of text chunks
+    """
+    if not documents:
+        logger.warning("No documents provided for chunking")
+        return []
+
+    # Create a local copy of supported extensions for this function call
+    local_code_extensions = CODE_EXTENSIONS.copy()
+
+    # Update supported extensions if provided
+    if code_file_extensions:
+        # Map extensions to languages (simplified mapping)
+        ext_mapping = {
+            ".py": "python",
+            ".java": "java",
+            ".cs": "c_sharp",
+            ".ts": "typescript",
+            ".tsx": "typescript",
+        }
+        for ext in code_file_extensions:
+            if ext.lower() not in local_code_extensions:
+                # Try to guess language from extension
+                if ext.lower() in ext_mapping:
+                    local_code_extensions[ext.lower()] = ext_mapping[ext.lower()]
+                else:
+                    logger.warning(f"Unsupported extension {ext}, will use traditional chunking")
+
+    all_chunks = []
+
+    if use_ast_chunking:
+        # Separate code and text documents using local extensions
+        code_docs, text_docs = detect_code_files(documents, local_code_extensions)
+
+        # Process code files with AST chunking
+        if code_docs:
+            logger.info(f"Processing {len(code_docs)} code files with AST chunking")
+            try:
+                ast_chunks = create_ast_chunks(
+                    code_docs, max_chunk_size=ast_chunk_size, chunk_overlap=ast_chunk_overlap
+                )
+                all_chunks.extend(ast_chunks)
+                logger.info(f"Created {len(ast_chunks)} AST chunks from code files")
+            except Exception as e:
+                logger.error(f"AST chunking failed: {e}")
+                if ast_fallback_traditional:
+                    logger.info("Falling back to traditional chunking for code files")
+                    traditional_code_chunks = create_traditional_chunks(
+                        code_docs, chunk_size, chunk_overlap
+                    )
+                    all_chunks.extend(traditional_code_chunks)
+                else:
+                    raise
+
+        # Process text files with traditional chunking
+        if text_docs:
+            logger.info(f"Processing {len(text_docs)} text files with traditional chunking")
+            text_chunks = create_traditional_chunks(text_docs, chunk_size, chunk_overlap)
+            all_chunks.extend(text_chunks)
+            logger.info(f"Created {len(text_chunks)} traditional chunks from text files")
+    else:
+        # Use traditional chunking for all files
+        logger.info(f"Processing {len(documents)} documents with traditional chunking")
+        all_chunks = create_traditional_chunks(documents, chunk_size, chunk_overlap)
+
+    logger.info(f"Total chunks created: {len(all_chunks)}")
+    return all_chunks

apps/code_rag.py

@@ -0,0 +1,211 @@
+"""
+Code RAG example using AST-aware chunking for optimal code understanding.
+Specialized for code repositories with automatic language detection and
+optimized chunking parameters.
+"""
+
+import sys
+from pathlib import Path
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent))
+
+from base_rag_example import BaseRAGExample
+from chunking import CODE_EXTENSIONS, create_text_chunks
+from llama_index.core import SimpleDirectoryReader
+
+
+class CodeRAG(BaseRAGExample):
+    """Specialized RAG example for code repositories with AST-aware chunking."""
+
+    def __init__(self):
+        super().__init__(
+            name="Code",
+            description="Process and query code repositories with AST-aware chunking",
+            default_index_name="code_index",
+        )
+        # Override defaults for code-specific usage
+        self.embedding_model_default = "facebook/contriever"  # Good for code
+        self.max_items_default = -1  # Process all code files by default
+
+    def _add_specific_arguments(self, parser):
+        """Add code-specific arguments."""
+        code_group = parser.add_argument_group("Code Repository Parameters")
+
+        code_group.add_argument(
+            "--repo-dir",
+            type=str,
+            default=".",
+            help="Code repository directory to index (default: current directory)",
+        )
+        code_group.add_argument(
+            "--include-extensions",
+            nargs="+",
+            default=list(CODE_EXTENSIONS.keys()),
+            help="File extensions to include (default: supported code extensions)",
+        )
+        code_group.add_argument(
+            "--exclude-dirs",
+            nargs="+",
+            default=[
+                ".git",
+                "__pycache__",
+                "node_modules",
+                "venv",
+                ".venv",
+                "build",
+                "dist",
+                "target",
+            ],
+            help="Directories to exclude from indexing",
+        )
+        code_group.add_argument(
+            "--max-file-size",
+            type=int,
+            default=1000000,  # 1MB
+            help="Maximum file size in bytes to process (default: 1MB)",
+        )
+        code_group.add_argument(
+            "--include-comments",
+            action="store_true",
+            help="Include comments in chunking (useful for documentation)",
+        )
+        code_group.add_argument(
+            "--preserve-imports",
+            action="store_true",
+            default=True,
+            help="Try to preserve import statements in chunks (default: True)",
+        )
+
+    async def load_data(self, args) -> list[str]:
+        """Load code files and convert to AST-aware chunks."""
+        print(f"🔍 Scanning code repository: {args.repo_dir}")
+        print(f"📁 Including extensions: {args.include_extensions}")
+        print(f"🚫 Excluding directories: {args.exclude_dirs}")
+
+        # Check if repository directory exists
+        repo_path = Path(args.repo_dir)
+        if not repo_path.exists():
+            raise ValueError(f"Repository directory not found: {args.repo_dir}")
+
+        # Load code files with filtering
+        reader_kwargs = {
+            "recursive": True,
+            "encoding": "utf-8",
+            "required_exts": args.include_extensions,
+            "exclude_hidden": True,
+        }
+
+        # Create exclusion filter
+        def file_filter(file_path: str) -> bool:
+            """Filter out unwanted files and directories."""
+            path = Path(file_path)
+
+            # Check file size
+            try:
+                if path.stat().st_size > args.max_file_size:
+                    print(f"⚠️ Skipping large file: {path.name} ({path.stat().st_size} bytes)")
+                    return False
+            except Exception:
+                return False
+
+            # Check if in excluded directory
+            for exclude_dir in args.exclude_dirs:
+                if exclude_dir in path.parts:
+                    return False
+
+            return True
+
+        try:
+            # Load documents with file filtering
+            documents = SimpleDirectoryReader(
+                args.repo_dir,
+                file_extractor=None,  # Use default extractors
+                **reader_kwargs,
+            ).load_data(show_progress=True)
+
+            # Apply custom filtering
+            filtered_docs = []
+            for doc in documents:
+                file_path = doc.metadata.get("file_path", "")
+                if file_filter(file_path):
+                    filtered_docs.append(doc)
+
+            documents = filtered_docs
+
+        except Exception as e:
+            print(f"❌ Error loading code files: {e}")
+            return []
+
+        if not documents:
+            print(
+                f"❌ No code files found in {args.repo_dir} with extensions {args.include_extensions}"
+            )
+            return []
+
+        print(f"✅ Loaded {len(documents)} code files")
+
+        # Show breakdown by language/extension
+        ext_counts = {}
+        for doc in documents:
+            file_path = doc.metadata.get("file_path", "")
+            if file_path:
+                ext = Path(file_path).suffix.lower()
+                ext_counts[ext] = ext_counts.get(ext, 0) + 1
+
+        print("📊 Files by extension:")
+        for ext, count in sorted(ext_counts.items()):
+            print(f"   {ext}: {count} files")
+
+        # Use AST-aware chunking by default for code
+        print(
+            f"🧠 Using AST-aware chunking (chunk_size: {args.ast_chunk_size}, overlap: {args.ast_chunk_overlap})"
+        )
+
+        all_texts = create_text_chunks(
+            documents,
+            chunk_size=256,  # Fallback for non-code files
+            chunk_overlap=64,
+            use_ast_chunking=True,  # Always use AST for code RAG
+            ast_chunk_size=args.ast_chunk_size,
+            ast_chunk_overlap=args.ast_chunk_overlap,
+            code_file_extensions=args.include_extensions,
+            ast_fallback_traditional=True,
+        )
+
+        # Apply max_items limit if specified
+        if args.max_items > 0 and len(all_texts) > args.max_items:
+            print(f"⏳ Limiting to {args.max_items} chunks (from {len(all_texts)})")
+            all_texts = all_texts[: args.max_items]
+
+        print(f"✅ Generated {len(all_texts)} code chunks")
+        return all_texts
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    # Example queries for code RAG
+    print("\n💻 Code RAG Example")
+    print("=" * 50)
+    print("\nExample queries you can try:")
+    print("- 'How does the embedding computation work?'")
+    print("- 'What are the main classes in this codebase?'")
+    print("- 'Show me the search implementation'")
+    print("- 'How is error handling implemented?'")
+    print("- 'What design patterns are used?'")
+    print("- 'Explain the chunking logic'")
+    print("\n🚀 Features:")
+    print("- ✅ AST-aware chunking preserves code structure")
+    print("- ✅ Automatic language detection")
+    print("- ✅ Smart filtering of large files and common excludes")
+    print("- ✅ Optimized for code understanding")
+    print("\nUsage examples:")
+    print("  python -m apps.code_rag --repo-dir ./my_project")
+    print(
+        "  python -m apps.code_rag --include-extensions .py .js --query 'How does authentication work?'"
+    )
+    print("\nOr run without --query for interactive mode\n")
+
+    rag = CodeRAG()
+    asyncio.run(rag.run())

apps/document_rag.py

@@ -0,0 +1,131 @@
+"""
+Document RAG example using the unified interface.
+Supports PDF, TXT, MD, and other document formats.
+"""
+
+import sys
+from pathlib import Path
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent))
+
+from base_rag_example import BaseRAGExample
+from chunking import create_text_chunks
+from llama_index.core import SimpleDirectoryReader
+
+
+class DocumentRAG(BaseRAGExample):
+    """RAG example for document processing (PDF, TXT, MD, etc.)."""
+
+    def __init__(self):
+        super().__init__(
+            name="Document",
+            description="Process and query documents (PDF, TXT, MD, etc.) with LEANN",
+            default_index_name="test_doc_files",
+        )
+
+    def _add_specific_arguments(self, parser):
+        """Add document-specific arguments."""
+        doc_group = parser.add_argument_group("Document Parameters")
+        doc_group.add_argument(
+            "--data-dir",
+            type=str,
+            default="data",
+            help="Directory containing documents to index (default: data)",
+        )
+        doc_group.add_argument(
+            "--file-types",
+            nargs="+",
+            default=None,
+            help="Filter by file types (e.g., .pdf .txt .md). If not specified, all supported types are processed",
+        )
+        doc_group.add_argument(
+            "--chunk-size", type=int, default=256, help="Text chunk size (default: 256)"
+        )
+        doc_group.add_argument(
+            "--chunk-overlap", type=int, default=128, help="Text chunk overlap (default: 128)"
+        )
+        doc_group.add_argument(
+            "--enable-code-chunking",
+            action="store_true",
+            help="Enable AST-aware chunking for code files in the data directory",
+        )
+
+    async def load_data(self, args) -> list[str]:
+        """Load documents and convert to text chunks."""
+        print(f"Loading documents from: {args.data_dir}")
+        if args.file_types:
+            print(f"Filtering by file types: {args.file_types}")
+        else:
+            print("Processing all supported file types")
+
+        # Check if data directory exists
+        data_path = Path(args.data_dir)
+        if not data_path.exists():
+            raise ValueError(f"Data directory not found: {args.data_dir}")
+
+        # Load documents
+        reader_kwargs = {
+            "recursive": True,
+            "encoding": "utf-8",
+        }
+        if args.file_types:
+            reader_kwargs["required_exts"] = args.file_types
+
+        documents = SimpleDirectoryReader(args.data_dir, **reader_kwargs).load_data(
+            show_progress=True
+        )
+
+        if not documents:
+            print(f"No documents found in {args.data_dir} with extensions {args.file_types}")
+            return []
+
+        print(f"Loaded {len(documents)} documents")
+
+        # Determine chunking strategy
+        use_ast = args.enable_code_chunking or getattr(args, "use_ast_chunking", False)
+
+        if use_ast:
+            print("Using AST-aware chunking for code files")
+
+        # Convert to text chunks with optional AST support
+        all_texts = create_text_chunks(
+            documents,
+            chunk_size=args.chunk_size,
+            chunk_overlap=args.chunk_overlap,
+            use_ast_chunking=use_ast,
+            ast_chunk_size=getattr(args, "ast_chunk_size", 512),
+            ast_chunk_overlap=getattr(args, "ast_chunk_overlap", 64),
+            code_file_extensions=getattr(args, "code_file_extensions", None),
+            ast_fallback_traditional=getattr(args, "ast_fallback_traditional", True),
+        )
+
+        # Apply max_items limit if specified
+        if args.max_items > 0 and len(all_texts) > args.max_items:
+            print(f"Limiting to {args.max_items} chunks (from {len(all_texts)})")
+            all_texts = all_texts[: args.max_items]
+
+        return all_texts
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    # Example queries for document RAG
+    print("\n📄 Document RAG Example")
+    print("=" * 50)
+    print("\nExample queries you can try:")
+    print("- 'What are the main techniques LEANN uses?'")
+    print("- 'What is the technique DLPM?'")
+    print("- 'Who does Elizabeth Bennet marry?'")
+    print(
+        "- 'What is the problem of developing pan gu model Huawei meets? (盘古大模型开发中遇到什么问题?)'"
+    )
+    print("\n🚀 NEW: Code-aware chunking available!")
+    print("- Use --enable-code-chunking to enable AST-aware chunking for code files")
+    print("- Supports Python, Java, C#, TypeScript files")
+    print("- Better semantic understanding of code structure")
+    print("\nOr run without --query for interactive mode\n")
+
+    rag = DocumentRAG()
+    asyncio.run(rag.run())

apps/email_data/LEANN_email_reader.py

@@ -0,0 +1,167 @@
+import email
+import os
+from pathlib import Path
+from typing import Any
+
+from llama_index.core import Document
+from llama_index.core.readers.base import BaseReader
+
+
+def find_all_messages_directories(root: str | None = None) -> list[Path]:
+    """
+    Recursively find all 'Messages' directories under the given root.
+    Returns a list of Path objects.
+    """
+    if root is None:
+        # Auto-detect user's mail path
+        home_dir = os.path.expanduser("~")
+        root = os.path.join(home_dir, "Library", "Mail")
+
+    messages_dirs = []
+    for dirpath, _dirnames, _filenames in os.walk(root):
+        if os.path.basename(dirpath) == "Messages":
+            messages_dirs.append(Path(dirpath))
+    return messages_dirs
+
+
+class EmlxReader(BaseReader):
+    """
+    Apple Mail .emlx file reader with embedded metadata.
+
+    Reads individual .emlx files from Apple Mail's storage format.
+    """
+
+    def __init__(self, include_html: bool = False) -> None:
+        """
+        Initialize.
+
+        Args:
+            include_html: Whether to include HTML content in the email body (default: False)
+        """
+        self.include_html = include_html
+
+    def load_data(self, input_dir: str, **load_kwargs: Any) -> list[Document]:
+        """
+        Load data from the input directory containing .emlx files.
+
+        Args:
+            input_dir: Directory containing .emlx files
+            **load_kwargs:
+                max_count (int): Maximum amount of messages to read.
+        """
+        docs: list[Document] = []
+        max_count = load_kwargs.get("max_count", 1000)
+        count = 0
+        total_files = 0
+        successful_files = 0
+        failed_files = 0
+
+        print(f"Starting to process directory: {input_dir}")
+
+        # Walk through the directory recursively
+        for dirpath, dirnames, filenames in os.walk(input_dir):
+            # Skip hidden directories
+            dirnames[:] = [d for d in dirnames if not d.startswith(".")]
+
+            for filename in filenames:
+                # Check if we've reached the max count (skip if max_count == -1)
+                if max_count > 0 and count >= max_count:
+                    break
+
+                if filename.endswith(".emlx"):
+                    total_files += 1
+                    filepath = os.path.join(dirpath, filename)
+                    try:
+                        # Read the .emlx file
+                        with open(filepath, encoding="utf-8", errors="ignore") as f:
+                            content = f.read()
+
+                        # .emlx files have a length prefix followed by the email content
+                        # The first line contains the length, followed by the email
+                        lines = content.split("\n", 1)
+                        if len(lines) >= 2:
+                            email_content = lines[1]
+
+                            # Parse the email using Python's email module
+                            try:
+                                msg = email.message_from_string(email_content)
+
+                                # Extract email metadata
+                                subject = msg.get("Subject", "No Subject")
+                                from_addr = msg.get("From", "Unknown")
+                                to_addr = msg.get("To", "Unknown")
+                                date = msg.get("Date", "Unknown")
+
+                                # Extract email body
+                                body = ""
+                                if msg.is_multipart():
+                                    for part in msg.walk():
+                                        if (
+                                            part.get_content_type() == "text/plain"
+                                            or part.get_content_type() == "text/html"
+                                        ):
+                                            if (
+                                                part.get_content_type() == "text/html"
+                                                and not self.include_html
+                                            ):
+                                                continue
+                                            try:
+                                                payload = part.get_payload(decode=True)
+                                                if payload:
+                                                    body += payload.decode("utf-8", errors="ignore")
+                                            except Exception as e:
+                                                print(f"Error decoding payload: {e}")
+                                                continue
+                                else:
+                                    try:
+                                        payload = msg.get_payload(decode=True)
+                                        if payload:
+                                            body = payload.decode("utf-8", errors="ignore")
+                                    except Exception as e:
+                                        print(f"Error decoding single part payload: {e}")
+                                        body = ""
+
+                                # Only create document if we have some content
+                                if body.strip() or subject != "No Subject":
+                                    # Create document content with metadata embedded in text
+                                    doc_content = f"""
+[File]: {filename}
+[From]: {from_addr}
+[To]: {to_addr}
+[Subject]: {subject}
+[Date]: {date}
+[EMAIL BODY Start]:
+{body}
+"""
+
+                                    # No separate metadata - everything is in the text
+                                    doc = Document(text=doc_content, metadata={})
+                                    docs.append(doc)
+                                    count += 1
+                                    successful_files += 1
+
+                                    # Print first few successful files for debugging
+                                    if successful_files <= 3:
+                                        print(
+                                            f"Successfully loaded: {filename} - Subject: {subject[:50]}..."
+                                        )
+
+                            except Exception as e:
+                                failed_files += 1
+                                if failed_files <= 5:  # Only print first few errors
+                                    print(f"Error parsing email from {filepath}: {e}")
+                                continue
+
+                    except Exception as e:
+                        failed_files += 1
+                        if failed_files <= 5:  # Only print first few errors
+                            print(f"Error reading file {filepath}: {e}")
+                        continue
+
+        print("Processing summary:")
+        print(f"  Total .emlx files found: {total_files}")
+        print(f"  Successfully loaded: {successful_files}")
+        print(f"  Failed to load: {failed_files}")
+        print(f"  Final documents: {len(docs)}")
+
+        return docs

apps/email_data/email.py

@@ -0,0 +1,186 @@
+"""
+Mbox parser.
+
+Contains simple parser for mbox files.
+
+"""
+
+import logging
+from pathlib import Path
+from typing import Any
+
+from fsspec import AbstractFileSystem
+from llama_index.core.readers.base import BaseReader
+from llama_index.core.schema import Document
+
+logger = logging.getLogger(__name__)
+
+
+class MboxReader(BaseReader):
+    """
+    Mbox parser.
+
+    Extract messages from mailbox files.
+    Returns string including date, subject, sender, receiver and
+    content for each message.
+
+    """
+
+    DEFAULT_MESSAGE_FORMAT: str = (
+        "Date: {_date}\nFrom: {_from}\nTo: {_to}\nSubject: {_subject}\nContent: {_content}"
+    )
+
+    def __init__(
+        self,
+        *args: Any,
+        max_count: int = 0,
+        message_format: str = DEFAULT_MESSAGE_FORMAT,
+        **kwargs: Any,
+    ) -> None:
+        """Init params."""
+        try:
+            from bs4 import BeautifulSoup  # noqa
+        except ImportError:
+            raise ImportError("`beautifulsoup4` package not found: `pip install beautifulsoup4`")
+
+        super().__init__(*args, **kwargs)
+        self.max_count = max_count
+        self.message_format = message_format
+
+    def load_data(
+        self,
+        file: Path,
+        extra_info: dict | None = None,
+        fs: AbstractFileSystem | None = None,
+    ) -> list[Document]:
+        """Parse file into string."""
+        # Import required libraries
+        import mailbox
+        from email.parser import BytesParser
+        from email.policy import default
+
+        from bs4 import BeautifulSoup
+
+        if fs:
+            logger.warning(
+                "fs was specified but MboxReader doesn't support loading "
+                "from fsspec filesystems. Will load from local filesystem instead."
+            )
+
+        i = 0
+        results: list[str] = []
+        # Load file using mailbox
+        bytes_parser = BytesParser(policy=default).parse
+        mbox = mailbox.mbox(file, factory=bytes_parser)  # type: ignore
+
+        # Iterate through all messages
+        for _, _msg in enumerate(mbox):
+            try:
+                msg: mailbox.mboxMessage = _msg
+                # Parse multipart messages
+                if msg.is_multipart():
+                    for part in msg.walk():
+                        ctype = part.get_content_type()
+                        cdispo = str(part.get("Content-Disposition"))
+                        if "attachment" in cdispo:
+                            print(f"Attachment found: {part.get_filename()}")
+                        if ctype == "text/plain" and "attachment" not in cdispo:
+                            content = part.get_payload(decode=True)  # decode
+                            break
+                # Get plain message payload for non-multipart messages
+                else:
+                    content = msg.get_payload(decode=True)
+
+                # Parse message HTML content and remove unneeded whitespace
+                soup = BeautifulSoup(content)
+                stripped_content = " ".join(soup.get_text().split())
+                # Format message to include date, sender, receiver and subject
+                msg_string = self.message_format.format(
+                    _date=msg["date"],
+                    _from=msg["from"],
+                    _to=msg["to"],
+                    _subject=msg["subject"],
+                    _content=stripped_content,
+                )
+                # Add message string to results
+                results.append(msg_string)
+            except Exception as e:
+                logger.warning(f"Failed to parse message:\n{_msg}\n with exception {e}")
+
+            # Increment counter and return if max count is met
+            i += 1
+            if self.max_count > 0 and i >= self.max_count:
+                break
+
+        return [Document(text=result, metadata=extra_info or {}) for result in results]
+
+
+class EmlxMboxReader(MboxReader):
+    """
+    EmlxMboxReader - Modified MboxReader that handles directories of .emlx files.
+
+    Extends MboxReader to work with Apple Mail's .emlx format by:
+    1. Reading .emlx files from a directory
+    2. Converting them to mbox format in memory
+    3. Using the parent MboxReader's parsing logic
+    """
+
+    def load_data(
+        self,
+        directory: Path,
+        extra_info: dict | None = None,
+        fs: AbstractFileSystem | None = None,
+    ) -> list[Document]:
+        """Parse .emlx files from directory into strings using MboxReader logic."""
+        import os
+        import tempfile
+
+        if fs:
+            logger.warning(
+                "fs was specified but EmlxMboxReader doesn't support loading "
+                "from fsspec filesystems. Will load from local filesystem instead."
+            )
+
+        # Find all .emlx files in the directory
+        emlx_files = list(directory.glob("*.emlx"))
+        logger.info(f"Found {len(emlx_files)} .emlx files in {directory}")
+
+        if not emlx_files:
+            logger.warning(f"No .emlx files found in {directory}")
+            return []
+
+        # Create a temporary mbox file
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".mbox", delete=False) as temp_mbox:
+            temp_mbox_path = temp_mbox.name
+
+            # Convert .emlx files to mbox format
+            for emlx_file in emlx_files:
+                try:
+                    # Read the .emlx file
+                    with open(emlx_file, encoding="utf-8", errors="ignore") as f:
+                        content = f.read()
+
+                    # .emlx format: first line is length, rest is email content
+                    lines = content.split("\n", 1)
+                    if len(lines) >= 2:
+                        email_content = lines[1]  # Skip the length line
+
+                        # Write to mbox format (each message starts with "From " and ends with blank line)
+                        temp_mbox.write(f"From {emlx_file.name} {email_content}\n\n")
+
+                except Exception as e:
+                    logger.warning(f"Failed to process {emlx_file}: {e}")
+                    continue
+
+            # Close the temporary file so MboxReader can read it
+            temp_mbox.close()
+
+            try:
+                # Use the parent MboxReader's logic to parse the mbox file
+                return super().load_data(Path(temp_mbox_path), extra_info, fs)
+            finally:
+                # Clean up temporary file
+                try:
+                    os.unlink(temp_mbox_path)
+                except OSError:
+                    pass

apps/email_rag.py

@@ -0,0 +1,156 @@
+"""
+Email RAG example using the unified interface.
+Supports Apple Mail on macOS.
+"""
+
+import sys
+from pathlib import Path
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent))
+
+from base_rag_example import BaseRAGExample, create_text_chunks
+
+from .email_data.LEANN_email_reader import EmlxReader
+
+
+class EmailRAG(BaseRAGExample):
+    """RAG example for Apple Mail processing."""
+
+    def __init__(self):
+        # Set default values BEFORE calling super().__init__
+        self.max_items_default = -1  # Process all emails by default
+        self.embedding_model_default = (
+            "sentence-transformers/all-MiniLM-L6-v2"  # Fast 384-dim model
+        )
+
+        super().__init__(
+            name="Email",
+            description="Process and query Apple Mail emails with LEANN",
+            default_index_name="mail_index",
+        )
+
+    def _add_specific_arguments(self, parser):
+        """Add email-specific arguments."""
+        email_group = parser.add_argument_group("Email Parameters")
+        email_group.add_argument(
+            "--mail-path",
+            type=str,
+            default=None,
+            help="Path to Apple Mail directory (auto-detected if not specified)",
+        )
+        email_group.add_argument(
+            "--include-html", action="store_true", help="Include HTML content in email processing"
+        )
+        email_group.add_argument(
+            "--chunk-size", type=int, default=256, help="Text chunk size (default: 256)"
+        )
+        email_group.add_argument(
+            "--chunk-overlap", type=int, default=25, help="Text chunk overlap (default: 25)"
+        )
+
+    def _find_mail_directories(self) -> list[Path]:
+        """Auto-detect all Apple Mail directories."""
+        mail_base = Path.home() / "Library" / "Mail"
+        if not mail_base.exists():
+            return []
+
+        # Find all Messages directories
+        messages_dirs = []
+        for item in mail_base.rglob("Messages"):
+            if item.is_dir():
+                messages_dirs.append(item)
+
+        return messages_dirs
+
+    async def load_data(self, args) -> list[str]:
+        """Load emails and convert to text chunks."""
+        # Determine mail directories
+        if args.mail_path:
+            messages_dirs = [Path(args.mail_path)]
+        else:
+            print("Auto-detecting Apple Mail directories...")
+            messages_dirs = self._find_mail_directories()
+
+        if not messages_dirs:
+            print("No Apple Mail directories found!")
+            print("Please specify --mail-path manually")
+            return []
+
+        print(f"Found {len(messages_dirs)} mail directories")
+
+        # Create reader
+        reader = EmlxReader(include_html=args.include_html)
+
+        # Process each directory
+        all_documents = []
+        total_processed = 0
+
+        for i, messages_dir in enumerate(messages_dirs):
+            print(f"\nProcessing directory {i + 1}/{len(messages_dirs)}: {messages_dir}")
+
+            try:
+                # Count emlx files
+                emlx_files = list(messages_dir.glob("*.emlx"))
+                print(f"Found {len(emlx_files)} email files")
+
+                # Apply max_items limit per directory
+                max_per_dir = -1  # Default to process all
+                if args.max_items > 0:
+                    remaining = args.max_items - total_processed
+                    if remaining <= 0:
+                        break
+                    max_per_dir = remaining
+                # If args.max_items == -1, max_per_dir stays -1 (process all)
+
+                # Load emails - fix the parameter passing
+                documents = reader.load_data(
+                    input_dir=str(messages_dir),
+                    max_count=max_per_dir,
+                )
+
+                if documents:
+                    all_documents.extend(documents)
+                    total_processed += len(documents)
+                    print(f"Processed {len(documents)} emails from this directory")
+
+            except Exception as e:
+                print(f"Error processing {messages_dir}: {e}")
+                continue
+
+        if not all_documents:
+            print("No emails found to process!")
+            return []
+
+        print(f"\nTotal emails processed: {len(all_documents)}")
+        print("now starting to split into text chunks ... take some time")
+
+        # Convert to text chunks
+        # Email reader uses chunk_overlap=25 as in original
+        all_texts = create_text_chunks(
+            all_documents, chunk_size=args.chunk_size, chunk_overlap=args.chunk_overlap
+        )
+
+        return all_texts
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    # Check platform
+    if sys.platform != "darwin":
+        print("\n⚠️  Warning: This example is designed for macOS (Apple Mail)")
+        print("   Windows/Linux support coming soon!\n")
+
+    # Example queries for email RAG
+    print("\n📧 Email RAG Example")
+    print("=" * 50)
+    print("\nExample queries you can try:")
+    print("- 'What did my boss say about deadlines?'")
+    print("- 'Find emails about travel expenses'")
+    print("- 'Show me emails from last month about the project'")
+    print("- 'What food did I order from DoorDash?'")
+    print("\nNote: You may need to grant Full Disk Access to your terminal\n")
+
+    rag = EmailRAG()
+    asyncio.run(rag.run())

apps/history_data/__init__.py

@@ -0,0 +1,3 @@
+from .history import ChromeHistoryReader
+
+__all__ = ["ChromeHistoryReader"]

apps/history_data/history.py

@@ -0,0 +1,186 @@
+import os
+import sqlite3
+from pathlib import Path
+from typing import Any
+
+from llama_index.core import Document
+from llama_index.core.readers.base import BaseReader
+
+
+class ChromeHistoryReader(BaseReader):
+    """
+    Chrome browser history reader that extracts browsing data from SQLite database.
+
+    Reads Chrome history from the default Chrome profile location and creates documents
+    with embedded metadata similar to the email reader structure.
+    """
+
+    def __init__(self) -> None:
+        """Initialize."""
+        pass
+
+    def load_data(self, input_dir: str | None = None, **load_kwargs: Any) -> list[Document]:
+        """
+        Load Chrome history data from the default Chrome profile location.
+
+        Args:
+            input_dir: Not used for Chrome history (kept for compatibility)
+            **load_kwargs:
+                max_count (int): Maximum amount of history entries to read.
+                chrome_profile_path (str): Custom path to Chrome profile directory.
+        """
+        docs: list[Document] = []
+        max_count = load_kwargs.get("max_count", 1000)
+        chrome_profile_path = load_kwargs.get("chrome_profile_path", None)
+
+        # Default Chrome profile path on macOS
+        if chrome_profile_path is None:
+            chrome_profile_path = os.path.expanduser(
+                "~/Library/Application Support/Google/Chrome/Default"
+            )
+
+        history_db_path = os.path.join(chrome_profile_path, "History")
+
+        if not os.path.exists(history_db_path):
+            print(f"Chrome history database not found at: {history_db_path}")
+            return docs
+
+        try:
+            # Connect to the Chrome history database
+            print(f"Connecting to database: {history_db_path}")
+            conn = sqlite3.connect(history_db_path)
+            cursor = conn.cursor()
+
+            # Query to get browsing history with metadata (removed created_time column)
+            query = """
+            SELECT
+                datetime(last_visit_time/1000000-11644473600,'unixepoch','localtime') as last_visit,
+                url,
+                title,
+                visit_count,
+                typed_count,
+                hidden
+            FROM urls
+            ORDER BY last_visit_time DESC
+            """
+
+            print(f"Executing query on database: {history_db_path}")
+            cursor.execute(query)
+            rows = cursor.fetchall()
+            print(f"Query returned {len(rows)} rows")
+
+            count = 0
+            for row in rows:
+                if count >= max_count and max_count > 0:
+                    break
+
+                last_visit, url, title, visit_count, typed_count, hidden = row
+
+                # Create document content with metadata embedded in text
+                doc_content = f"""
+[Title]: {title}
+[URL of the page]: {url}
+[Last visited time]: {last_visit}
+[Visit times]: {visit_count}
+[Typed times]: {typed_count}
+"""
+
+                # Create document with embedded metadata
+                doc = Document(text=doc_content, metadata={"title": title[0:150]})
+                # if len(title) > 150:
+                #     print(f"Title is too long: {title}")
+                docs.append(doc)
+                count += 1
+
+            conn.close()
+            print(f"Loaded {len(docs)} Chrome history documents")
+
+        except Exception as e:
+            print(f"Error reading Chrome history: {e}")
+            # add you may need to close your browser to make the database file available
+            # also highlight in red
+            print(
+                "\033[91mYou may need to close your browser to make the database file available\033[0m"
+            )
+            return docs
+
+        return docs
+
+    @staticmethod
+    def find_chrome_profiles() -> list[Path]:
+        """
+        Find all Chrome profile directories.
+
+        Returns:
+            List of Path objects pointing to Chrome profile directories
+        """
+        chrome_base_path = Path(os.path.expanduser("~/Library/Application Support/Google/Chrome"))
+        profile_dirs = []
+
+        if not chrome_base_path.exists():
+            print(f"Chrome directory not found at: {chrome_base_path}")
+            return profile_dirs
+
+        # Find all profile directories
+        for profile_dir in chrome_base_path.iterdir():
+            if profile_dir.is_dir() and profile_dir.name != "System Profile":
+                history_path = profile_dir / "History"
+                if history_path.exists():
+                    profile_dirs.append(profile_dir)
+                    print(f"Found Chrome profile: {profile_dir}")
+
+        print(f"Found {len(profile_dirs)} Chrome profiles")
+        return profile_dirs
+
+    @staticmethod
+    def export_history_to_file(
+        output_file: str = "chrome_history_export.txt", max_count: int = 1000
+    ):
+        """
+        Export Chrome history to a text file using the same SQL query format.
+
+        Args:
+            output_file: Path to the output file
+            max_count: Maximum number of entries to export
+        """
+        chrome_profile_path = os.path.expanduser(
+            "~/Library/Application Support/Google/Chrome/Default"
+        )
+        history_db_path = os.path.join(chrome_profile_path, "History")
+
+        if not os.path.exists(history_db_path):
+            print(f"Chrome history database not found at: {history_db_path}")
+            return
+
+        try:
+            conn = sqlite3.connect(history_db_path)
+            cursor = conn.cursor()
+
+            query = """
+            SELECT
+                datetime(last_visit_time/1000000-11644473600,'unixepoch','localtime') as last_visit,
+                url,
+                title,
+                visit_count,
+                typed_count,
+                hidden
+            FROM urls
+            ORDER BY last_visit_time DESC
+            LIMIT ?
+            """
+
+            cursor.execute(query, (max_count,))
+            rows = cursor.fetchall()
+
+            with open(output_file, "w", encoding="utf-8") as f:
+                for row in rows:
+                    last_visit, url, title, visit_count, typed_count, hidden = row
+                    f.write(
+                        f"{last_visit}\t{url}\t{title}\t{visit_count}\t{typed_count}\t{hidden}\n"
+                    )
+
+            conn.close()
+            print(f"Exported {len(rows)} history entries to {output_file}")
+
+        except Exception as e:
+            print(f"Error exporting Chrome history: {e}")

apps/history_data/wechat_history.py

@@ -0,0 +1,774 @@
+import json
+import os
+import re
+import subprocess
+import time
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+from llama_index.core import Document
+from llama_index.core.readers.base import BaseReader
+
+
+class WeChatHistoryReader(BaseReader):
+    """
+    WeChat chat history reader that extracts chat data from exported JSON files.
+
+    Reads WeChat chat history from exported JSON files (from wechat-exporter tool)
+    and creates documents with embedded metadata similar to the Chrome history reader structure.
+
+    Also includes utilities for automatic WeChat chat history export.
+    """
+
+    def __init__(self) -> None:
+        """Initialize."""
+        self.packages_dir = Path(__file__).parent.parent.parent / "packages"
+        self.wechat_exporter_dir = self.packages_dir / "wechat-exporter"
+        self.wechat_decipher_dir = self.packages_dir / "wechat-decipher-macos"
+
+    def check_wechat_running(self) -> bool:
+        """Check if WeChat is currently running."""
+        try:
+            result = subprocess.run(["pgrep", "-f", "WeChat"], capture_output=True, text=True)
+            return result.returncode == 0
+        except Exception:
+            return False
+
+    def install_wechattweak(self) -> bool:
+        """Install WeChatTweak CLI tool."""
+        try:
+            # Create wechat-exporter directory if it doesn't exist
+            self.wechat_exporter_dir.mkdir(parents=True, exist_ok=True)
+
+            wechattweak_path = self.wechat_exporter_dir / "wechattweak-cli"
+            if not wechattweak_path.exists():
+                print("Downloading WeChatTweak CLI...")
+                subprocess.run(
+                    [
+                        "curl",
+                        "-L",
+                        "-o",
+                        str(wechattweak_path),
+                        "https://github.com/JettChenT/WeChatTweak-CLI/releases/latest/download/wechattweak-cli",
+                    ],
+                    check=True,
+                )
+
+            # Make executable
+            wechattweak_path.chmod(0o755)
+
+            # Install WeChatTweak
+            print("Installing WeChatTweak...")
+            subprocess.run(["sudo", str(wechattweak_path), "install"], check=True)
+            return True
+        except Exception as e:
+            print(f"Error installing WeChatTweak: {e}")
+            return False
+
+    def restart_wechat(self):
+        """Restart WeChat to apply WeChatTweak."""
+        try:
+            print("Restarting WeChat...")
+            subprocess.run(["pkill", "-f", "WeChat"], check=False)
+            time.sleep(2)
+            subprocess.run(["open", "-a", "WeChat"], check=True)
+            time.sleep(5)  # Wait for WeChat to start
+        except Exception as e:
+            print(f"Error restarting WeChat: {e}")
+
+    def check_api_available(self) -> bool:
+        """Check if WeChatTweak API is available."""
+        try:
+            result = subprocess.run(
+                ["curl", "-s", "http://localhost:48065/wechat/allcontacts"],
+                capture_output=True,
+                text=True,
+                timeout=5,
+            )
+            return result.returncode == 0 and result.stdout.strip()
+        except Exception:
+            return False
+
+    def _extract_readable_text(self, content: str) -> str:
+        """
+        Extract readable text from message content, removing XML and system messages.
+
+        Args:
+            content: The raw message content (can be string or dict)
+
+        Returns:
+            Cleaned, readable text
+        """
+        if not content:
+            return ""
+
+        # Handle dictionary content (like quoted messages)
+        if isinstance(content, dict):
+            # Extract text from dictionary structure
+            text_parts = []
+            if "title" in content:
+                text_parts.append(str(content["title"]))
+            if "quoted" in content:
+                text_parts.append(str(content["quoted"]))
+            if "content" in content:
+                text_parts.append(str(content["content"]))
+            if "text" in content:
+                text_parts.append(str(content["text"]))
+
+            if text_parts:
+                return " | ".join(text_parts)
+            else:
+                # If we can't extract meaningful text from dict, return empty
+                return ""
+
+        # Handle string content
+        if not isinstance(content, str):
+            return ""
+
+        # Remove common prefixes like "wxid_xxx:\n"
+        clean_content = re.sub(r"^wxid_[^:]+:\s*", "", content)
+        clean_content = re.sub(r"^[^:]+:\s*", "", clean_content)
+
+        # If it's just XML or system message, return empty
+        if clean_content.strip().startswith("<") or "recalled a message" in clean_content:
+            return ""
+
+        return clean_content.strip()
+
+    def _is_text_message(self, content: str) -> bool:
+        """
+        Check if a message contains readable text content.
+
+        Args:
+            content: The message content (can be string or dict)
+
+        Returns:
+            True if the message contains readable text, False otherwise
+        """
+        if not content:
+            return False
+
+        # Handle dictionary content
+        if isinstance(content, dict):
+            # Check if dict has any readable text fields
+            text_fields = ["title", "quoted", "content", "text"]
+            for field in text_fields:
+                if content.get(field):
+                    return True
+            return False
+
+        # Handle string content
+        if not isinstance(content, str):
+            return False
+
+        # Skip image messages (contain XML with img tags)
+        if "<img" in content and "cdnurl" in content:
+            return False
+
+        # Skip emoji messages (contain emoji XML tags)
+        if "<emoji" in content and "productid" in content:
+            return False
+
+        # Skip voice messages
+        if "<voice" in content:
+            return False
+
+        # Skip video messages
+        if "<video" in content:
+            return False
+
+        # Skip file messages
+        if "<appmsg" in content and "appid" in content:
+            return False
+
+        # Skip system messages (like "recalled a message")
+        if "recalled a message" in content:
+            return False
+
+        # Check if there's actual readable text (not just XML or system messages)
+        # Remove common prefixes like "wxid_xxx:\n" and check for actual content
+        clean_content = re.sub(r"^wxid_[^:]+:\s*", "", content)
+        clean_content = re.sub(r"^[^:]+:\s*", "", clean_content)
+
+        # If after cleaning we have meaningful text, consider it readable
+        if len(clean_content.strip()) > 0 and not clean_content.strip().startswith("<"):
+            return True
+
+        return False
+
+    def _concatenate_messages(
+        self,
+        messages: list[dict],
+        max_length: int = 128,
+        time_window_minutes: int = 30,
+        overlap_messages: int = 0,
+    ) -> list[dict]:
+        """
+        Concatenate messages based on length and time rules.
+
+        Args:
+            messages: List of message dictionaries
+            max_length: Maximum length for concatenated message groups. Use -1 to disable length constraint.
+            time_window_minutes: Time window in minutes to group messages together. Use -1 to disable time constraint.
+            overlap_messages: Number of messages to overlap between consecutive groups
+
+        Returns:
+            List of concatenated message groups
+        """
+        if not messages:
+            return []
+
+        concatenated_groups = []
+        current_group = []
+        current_length = 0
+        last_timestamp = None
+
+        for message in messages:
+            # Extract message info
+            content = message.get("content", "")
+            message_text = message.get("message", "")
+            create_time = message.get("createTime", 0)
+            message.get("fromUser", "")
+            message.get("toUser", "")
+            message.get("isSentFromSelf", False)
+
+            # Extract readable text
+            readable_text = self._extract_readable_text(content)
+            if not readable_text:
+                readable_text = message_text
+
+            # Skip empty messages
+            if not readable_text.strip():
+                continue
+
+            # Check time window constraint (only if time_window_minutes != -1)
+            if time_window_minutes != -1 and last_timestamp is not None and create_time > 0:
+                time_diff_minutes = (create_time - last_timestamp) / 60
+                if time_diff_minutes > time_window_minutes:
+                    # Time gap too large, start new group
+                    if current_group:
+                        concatenated_groups.append(
+                            {
+                                "messages": current_group,
+                                "total_length": current_length,
+                                "start_time": current_group[0].get("createTime", 0),
+                                "end_time": current_group[-1].get("createTime", 0),
+                            }
+                        )
+                        # Keep last few messages for overlap
+                        if overlap_messages > 0 and len(current_group) > overlap_messages:
+                            current_group = current_group[-overlap_messages:]
+                            current_length = sum(
+                                len(
+                                    self._extract_readable_text(msg.get("content", ""))
+                                    or msg.get("message", "")
+                                )
+                                for msg in current_group
+                            )
+                        else:
+                            current_group = []
+                            current_length = 0
+
+            # Check length constraint (only if max_length != -1)
+            message_length = len(readable_text)
+            if max_length != -1 and current_length + message_length > max_length and current_group:
+                # Current group would exceed max length, save it and start new
+                concatenated_groups.append(
+                    {
+                        "messages": current_group,
+                        "total_length": current_length,
+                        "start_time": current_group[0].get("createTime", 0),
+                        "end_time": current_group[-1].get("createTime", 0),
+                    }
+                )
+                # Keep last few messages for overlap
+                if overlap_messages > 0 and len(current_group) > overlap_messages:
+                    current_group = current_group[-overlap_messages:]
+                    current_length = sum(
+                        len(
+                            self._extract_readable_text(msg.get("content", ""))
+                            or msg.get("message", "")
+                        )
+                        for msg in current_group
+                    )
+                else:
+                    current_group = []
+                    current_length = 0
+
+            # Add message to current group
+            current_group.append(message)
+            current_length += message_length
+            last_timestamp = create_time
+
+        # Add the last group if it exists
+        if current_group:
+            concatenated_groups.append(
+                {
+                    "messages": current_group,
+                    "total_length": current_length,
+                    "start_time": current_group[0].get("createTime", 0),
+                    "end_time": current_group[-1].get("createTime", 0),
+                }
+            )
+
+        return concatenated_groups
+
+    def _create_concatenated_content(self, message_group: dict, contact_name: str) -> str:
+        """
+        Create concatenated content from a group of messages.
+
+        Args:
+            message_group: Dictionary containing messages and metadata
+            contact_name: Name of the contact
+
+        Returns:
+            Formatted concatenated content
+        """
+        messages = message_group["messages"]
+        start_time = message_group["start_time"]
+        end_time = message_group["end_time"]
+
+        # Format timestamps
+        if start_time:
+            try:
+                start_timestamp = datetime.fromtimestamp(start_time)
+                start_time_str = start_timestamp.strftime("%Y-%m-%d %H:%M:%S")
+            except (ValueError, OSError):
+                start_time_str = str(start_time)
+        else:
+            start_time_str = "Unknown"
+
+        if end_time:
+            try:
+                end_timestamp = datetime.fromtimestamp(end_time)
+                end_time_str = end_timestamp.strftime("%Y-%m-%d %H:%M:%S")
+            except (ValueError, OSError):
+                end_time_str = str(end_time)
+        else:
+            end_time_str = "Unknown"
+
+        # Build concatenated message content
+        message_parts = []
+        for message in messages:
+            content = message.get("content", "")
+            message_text = message.get("message", "")
+            create_time = message.get("createTime", 0)
+            is_sent_from_self = message.get("isSentFromSelf", False)
+
+            # Extract readable text
+            readable_text = self._extract_readable_text(content)
+            if not readable_text:
+                readable_text = message_text
+
+            # Format individual message
+            if create_time:
+                try:
+                    timestamp = datetime.fromtimestamp(create_time)
+                    # change to YYYY-MM-DD HH:MM:SS
+                    time_str = timestamp.strftime("%Y-%m-%d %H:%M:%S")
+                except (ValueError, OSError):
+                    time_str = str(create_time)
+            else:
+                time_str = "Unknown"
+
+            sender = "[Me]" if is_sent_from_self else "[Contact]"
+            message_parts.append(f"({time_str}) {sender}: {readable_text}")
+
+        concatenated_text = "\n".join(message_parts)
+
+        # Create final document content
+        doc_content = f"""
+Contact: {contact_name}
+Time Range: {start_time_str} - {end_time_str}
+Messages ({len(messages)} messages, {message_group["total_length"]} chars):
+
+{concatenated_text}
+"""
+        # TODO @yichuan give better format and rich info here!
+        doc_content = f"""
+{concatenated_text}
+"""
+        return doc_content, contact_name
+
+    def load_data(self, input_dir: str | None = None, **load_kwargs: Any) -> list[Document]:
+        """
+        Load WeChat chat history data from exported JSON files.
+
+        Args:
+            input_dir: Directory containing exported WeChat JSON files
+            **load_kwargs:
+                max_count (int): Maximum amount of chat entries to read.
+                wechat_export_dir (str): Custom path to WeChat export directory.
+                include_non_text (bool): Whether to include non-text messages (images, emojis, etc.)
+                concatenate_messages (bool): Whether to concatenate messages based on length rules.
+                max_length (int): Maximum length for concatenated message groups (default: 1000).
+                time_window_minutes (int): Time window in minutes to group messages together (default: 30).
+                overlap_messages (int): Number of messages to overlap between consecutive groups (default: 2).
+        """
+        docs: list[Document] = []
+        max_count = load_kwargs.get("max_count", 1000)
+        wechat_export_dir = load_kwargs.get("wechat_export_dir", None)
+        include_non_text = load_kwargs.get("include_non_text", False)
+        concatenate_messages = load_kwargs.get("concatenate_messages", False)
+        max_length = load_kwargs.get("max_length", 1000)
+        time_window_minutes = load_kwargs.get("time_window_minutes", 30)
+
+        # Default WeChat export path
+        if wechat_export_dir is None:
+            wechat_export_dir = "./wechat_export_test"
+
+        if not os.path.exists(wechat_export_dir):
+            print(f"WeChat export directory not found at: {wechat_export_dir}")
+            return docs
+
+        try:
+            # Find all JSON files in the export directory
+            json_files = list(Path(wechat_export_dir).glob("*.json"))
+            print(f"Found {len(json_files)} WeChat chat history files")
+
+            count = 0
+            for json_file in json_files:
+                if count >= max_count and max_count > 0:
+                    break
+
+                try:
+                    with open(json_file, encoding="utf-8") as f:
+                        chat_data = json.load(f)
+
+                    # Extract contact name from filename
+                    contact_name = json_file.stem
+
+                    if concatenate_messages:
+                        # Filter messages to only include readable text messages
+                        readable_messages = []
+                        for message in chat_data:
+                            try:
+                                content = message.get("content", "")
+                                if not include_non_text and not self._is_text_message(content):
+                                    continue
+
+                                readable_text = self._extract_readable_text(content)
+                                if not readable_text and not include_non_text:
+                                    continue
+
+                                readable_messages.append(message)
+                            except Exception as e:
+                                print(f"Error processing message in {json_file}: {e}")
+                                continue
+
+                        # Concatenate messages based on rules
+                        message_groups = self._concatenate_messages(
+                            readable_messages,
+                            max_length=max_length,
+                            time_window_minutes=time_window_minutes,
+                            overlap_messages=0,  # No overlap between groups
+                        )
+
+                        # Create documents from concatenated groups
+                        for message_group in message_groups:
+                            if count >= max_count and max_count > 0:
+                                break
+
+                            doc_content, contact_name = self._create_concatenated_content(
+                                message_group, contact_name
+                            )
+                            doc = Document(
+                                text=doc_content,
+                                metadata={"contact_name": contact_name},
+                            )
+                            docs.append(doc)
+                            count += 1
+
+                        print(
+                            f"Created {len(message_groups)} concatenated message groups for {contact_name}"
+                        )
+
+                    else:
+                        # Original single-message processing
+                        for message in chat_data:
+                            if count >= max_count and max_count > 0:
+                                break
+
+                            # Extract message information
+                            message.get("fromUser", "")
+                            message.get("toUser", "")
+                            content = message.get("content", "")
+                            message_text = message.get("message", "")
+                            create_time = message.get("createTime", 0)
+                            is_sent_from_self = message.get("isSentFromSelf", False)
+
+                            # Handle content that might be dict or string
+                            try:
+                                # Check if this is a readable text message
+                                if not include_non_text and not self._is_text_message(content):
+                                    continue
+
+                                # Extract readable text
+                                readable_text = self._extract_readable_text(content)
+                                if not readable_text and not include_non_text:
+                                    continue
+                            except Exception as e:
+                                # Skip messages that cause processing errors
+                                print(f"Error processing message in {json_file}: {e}")
+                                continue
+
+                            # Convert timestamp to readable format
+                            if create_time:
+                                try:
+                                    timestamp = datetime.fromtimestamp(create_time)
+                                    time_str = timestamp.strftime("%Y-%m-%d %H:%M:%S")
+                                except (ValueError, OSError):
+                                    time_str = str(create_time)
+                            else:
+                                time_str = "Unknown"
+
+                            # Create document content with metadata header and contact info
+                            doc_content = f"""
+Contact: {contact_name}
+Is sent from self: {is_sent_from_self}
+Time: {time_str}
+Message: {readable_text if readable_text else message_text}
+"""
+
+                            # Create document with embedded metadata
+                            doc = Document(
+                                text=doc_content, metadata={"contact_name": contact_name}
+                            )
+                            docs.append(doc)
+                            count += 1
+
+                except Exception as e:
+                    print(f"Error reading {json_file}: {e}")
+                    continue
+
+            print(f"Loaded {len(docs)} WeChat chat documents")
+
+        except Exception as e:
+            print(f"Error reading WeChat history: {e}")
+            return docs
+
+        return docs
+
+    @staticmethod
+    def find_wechat_export_dirs() -> list[Path]:
+        """
+        Find all WeChat export directories.
+
+        Returns:
+            List of Path objects pointing to WeChat export directories
+        """
+        export_dirs = []
+
+        # Look for common export directory names
+        possible_dirs = [
+            Path("./wechat_export"),
+            Path("./wechat_export_direct"),
+            Path("./wechat_chat_history"),
+            Path("./chat_export"),
+        ]
+
+        for export_dir in possible_dirs:
+            if export_dir.exists() and export_dir.is_dir():
+                json_files = list(export_dir.glob("*.json"))
+                if json_files:
+                    export_dirs.append(export_dir)
+                    print(
+                        f"Found WeChat export directory: {export_dir} with {len(json_files)} files"
+                    )
+
+        print(f"Found {len(export_dirs)} WeChat export directories")
+        return export_dirs
+
+    @staticmethod
+    def export_chat_to_file(
+        output_file: str = "wechat_chat_export.txt",
+        max_count: int = 1000,
+        export_dir: str | None = None,
+        include_non_text: bool = False,
+    ):
+        """
+        Export WeChat chat history to a text file.
+
+        Args:
+            output_file: Path to the output file
+            max_count: Maximum number of entries to export
+            export_dir: Directory containing WeChat JSON files
+            include_non_text: Whether to include non-text messages
+        """
+        if export_dir is None:
+            export_dir = "./wechat_export_test"
+
+        if not os.path.exists(export_dir):
+            print(f"WeChat export directory not found at: {export_dir}")
+            return
+
+        try:
+            json_files = list(Path(export_dir).glob("*.json"))
+
+            with open(output_file, "w", encoding="utf-8") as f:
+                count = 0
+                for json_file in json_files:
+                    if count >= max_count and max_count > 0:
+                        break
+
+                    try:
+                        with open(json_file, encoding="utf-8") as json_f:
+                            chat_data = json.load(json_f)
+
+                        contact_name = json_file.stem
+                        f.write(f"\n=== Chat with {contact_name} ===\n")
+
+                        for message in chat_data:
+                            if count >= max_count and max_count > 0:
+                                break
+
+                            from_user = message.get("fromUser", "")
+                            content = message.get("content", "")
+                            message_text = message.get("message", "")
+                            create_time = message.get("createTime", 0)
+
+                            # Skip non-text messages unless requested
+                            if not include_non_text:
+                                reader = WeChatHistoryReader()
+                                if not reader._is_text_message(content):
+                                    continue
+                                readable_text = reader._extract_readable_text(content)
+                                if not readable_text:
+                                    continue
+                                message_text = readable_text
+
+                            if create_time:
+                                try:
+                                    timestamp = datetime.fromtimestamp(create_time)
+                                    time_str = timestamp.strftime("%Y-%m-%d %H:%M:%S")
+                                except (ValueError, OSError):
+                                    time_str = str(create_time)
+                            else:
+                                time_str = "Unknown"
+
+                            f.write(f"[{time_str}] {from_user}: {message_text}\n")
+                            count += 1
+
+                    except Exception as e:
+                        print(f"Error processing {json_file}: {e}")
+                        continue
+
+            print(f"Exported {count} chat entries to {output_file}")
+
+        except Exception as e:
+            print(f"Error exporting WeChat chat history: {e}")
+
+    def export_wechat_chat_history(self, export_dir: str = "./wechat_export_direct") -> Path | None:
+        """
+        Export WeChat chat history using wechat-exporter tool.
+
+        Args:
+            export_dir: Directory to save exported chat history
+
+        Returns:
+            Path to export directory if successful, None otherwise
+        """
+        try:
+            import subprocess
+            import sys
+
+            # Create export directory
+            export_path = Path(export_dir)
+            export_path.mkdir(exist_ok=True)
+
+            print(f"Exporting WeChat chat history to {export_path}...")
+
+            # Check if wechat-exporter directory exists
+            if not self.wechat_exporter_dir.exists():
+                print(f"wechat-exporter directory not found at: {self.wechat_exporter_dir}")
+                return None
+
+            # Install requirements if needed
+            requirements_file = self.wechat_exporter_dir / "requirements.txt"
+            if requirements_file.exists():
+                print("Installing wechat-exporter requirements...")
+                subprocess.run(["uv", "pip", "install", "-r", str(requirements_file)], check=True)
+
+            # Run the export command
+            print("Running wechat-exporter...")
+            result = subprocess.run(
+                [
+                    sys.executable,
+                    str(self.wechat_exporter_dir / "main.py"),
+                    "export-all",
+                    str(export_path),
+                ],
+                capture_output=True,
+                text=True,
+                check=True,
+            )
+
+            print("Export command output:")
+            print(result.stdout)
+            if result.stderr:
+                print("Export errors:")
+                print(result.stderr)
+
+            # Check if export was successful
+            if export_path.exists() and any(export_path.glob("*.json")):
+                json_files = list(export_path.glob("*.json"))
+                print(
+                    f"Successfully exported {len(json_files)} chat history files to {export_path}"
+                )
+                return export_path
+            else:
+                print("Export completed but no JSON files found")
+                return None
+
+        except subprocess.CalledProcessError as e:
+            print(f"Export command failed: {e}")
+            print(f"Command output: {e.stdout}")
+            print(f"Command errors: {e.stderr}")
+            return None
+        except Exception as e:
+            print(f"Export failed: {e}")
+            print("Please ensure WeChat is running and WeChatTweak is installed.")
+            return None
+
+    def find_or_export_wechat_data(self, export_dir: str = "./wechat_export_direct") -> list[Path]:
+        """
+        Find existing WeChat exports or create new ones.
+
+        Args:
+            export_dir: Directory to save exported chat history if needed
+
+        Returns:
+            List of Path objects pointing to WeChat export directories
+        """
+        export_dirs = []
+
+        # Look for existing exports in common locations
+        possible_export_dirs = [
+            Path("./wechat_database_export"),
+            Path("./wechat_export_test"),
+            Path("./wechat_export"),
+            Path("./wechat_export_direct"),
+            Path("./wechat_chat_history"),
+            Path("./chat_export"),
+        ]
+
+        for export_dir_path in possible_export_dirs:
+            if export_dir_path.exists() and any(export_dir_path.glob("*.json")):
+                export_dirs.append(export_dir_path)
+                print(f"Found existing export: {export_dir_path}")
+
+        # If no existing exports, try to export automatically
+        if not export_dirs:
+            print("No existing WeChat exports found. Starting direct export...")
+
+            # Try to export using wechat-exporter
+            exported_path = self.export_wechat_chat_history(export_dir)
+            if exported_path:
+                export_dirs = [exported_path]
+            else:
+                print(
+                    "Failed to export WeChat data. Please ensure WeChat is running and WeChatTweak is installed."
+                )
+
+        return export_dirs

apps/wechat_rag.py

@@ -0,0 +1,189 @@
+"""
+WeChat History RAG example using the unified interface.
+Supports WeChat chat history export and search.
+"""
+
+import subprocess
+import sys
+from pathlib import Path
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent))
+
+from base_rag_example import BaseRAGExample
+
+from .history_data.wechat_history import WeChatHistoryReader
+
+
+class WeChatRAG(BaseRAGExample):
+    """RAG example for WeChat chat history."""
+
+    def __init__(self):
+        # Set default values BEFORE calling super().__init__
+        self.max_items_default = -1  # Match original default
+        self.embedding_model_default = (
+            "sentence-transformers/all-MiniLM-L6-v2"  # Fast 384-dim model
+        )
+
+        super().__init__(
+            name="WeChat History",
+            description="Process and query WeChat chat history with LEANN",
+            default_index_name="wechat_history_magic_test_11Debug_new",
+        )
+
+    def _add_specific_arguments(self, parser):
+        """Add WeChat-specific arguments."""
+        wechat_group = parser.add_argument_group("WeChat Parameters")
+        wechat_group.add_argument(
+            "--export-dir",
+            type=str,
+            default="./wechat_export",
+            help="Directory to store WeChat exports (default: ./wechat_export)",
+        )
+        wechat_group.add_argument(
+            "--force-export",
+            action="store_true",
+            help="Force re-export of WeChat data even if exports exist",
+        )
+        wechat_group.add_argument(
+            "--chunk-size", type=int, default=192, help="Text chunk size (default: 192)"
+        )
+        wechat_group.add_argument(
+            "--chunk-overlap", type=int, default=64, help="Text chunk overlap (default: 64)"
+        )
+
+    def _export_wechat_data(self, export_dir: Path) -> bool:
+        """Export WeChat data using wechattweak-cli."""
+        print("Exporting WeChat data...")
+
+        # Check if WeChat is running
+        try:
+            result = subprocess.run(["pgrep", "WeChat"], capture_output=True, text=True)
+            if result.returncode != 0:
+                print("WeChat is not running. Please start WeChat first.")
+                return False
+        except Exception:
+            pass  # pgrep might not be available on all systems
+
+        # Create export directory
+        export_dir.mkdir(parents=True, exist_ok=True)
+
+        # Run export command
+        cmd = ["packages/wechat-exporter/wechattweak-cli", "export", str(export_dir)]
+
+        try:
+            print(f"Running: {' '.join(cmd)}")
+            result = subprocess.run(cmd, capture_output=True, text=True)
+
+            if result.returncode == 0:
+                print("WeChat data exported successfully!")
+                return True
+            else:
+                print(f"Export failed: {result.stderr}")
+                return False
+
+        except FileNotFoundError:
+            print("\nError: wechattweak-cli not found!")
+            print("Please install it first:")
+            print("  sudo packages/wechat-exporter/wechattweak-cli install")
+            return False
+        except Exception as e:
+            print(f"Export error: {e}")
+            return False
+
+    async def load_data(self, args) -> list[str]:
+        """Load WeChat history and convert to text chunks."""
+        # Initialize WeChat reader with export capabilities
+        reader = WeChatHistoryReader()
+
+        # Find existing exports or create new ones using the centralized method
+        export_dirs = reader.find_or_export_wechat_data(args.export_dir)
+        if not export_dirs:
+            print("Failed to find or export WeChat data. Trying to find any existing exports...")
+            # Try to find any existing exports in common locations
+            export_dirs = reader.find_wechat_export_dirs()
+            if not export_dirs:
+                print("No WeChat data found. Please ensure WeChat exports exist.")
+                return []
+
+        # Load documents from all found export directories
+        all_documents = []
+        total_processed = 0
+
+        for i, export_dir in enumerate(export_dirs):
+            print(f"\nProcessing WeChat export {i + 1}/{len(export_dirs)}: {export_dir}")
+
+            try:
+                # Apply max_items limit per export
+                max_per_export = -1
+                if args.max_items > 0:
+                    remaining = args.max_items - total_processed
+                    if remaining <= 0:
+                        break
+                    max_per_export = remaining
+
+                documents = reader.load_data(
+                    wechat_export_dir=str(export_dir),
+                    max_count=max_per_export,
+                    concatenate_messages=True,  # Enable message concatenation for better context
+                )
+
+                if documents:
+                    print(f"Loaded {len(documents)} chat documents from {export_dir}")
+                    all_documents.extend(documents)
+                    total_processed += len(documents)
+                else:
+                    print(f"No documents loaded from {export_dir}")
+
+            except Exception as e:
+                print(f"Error processing {export_dir}: {e}")
+                continue
+
+        if not all_documents:
+            print("No documents loaded from any source. Exiting.")
+            return []
+
+        print(f"\nTotal loaded {len(all_documents)} chat documents from {len(export_dirs)} exports")
+        print("now starting to split into text chunks ... take some time")
+
+        # Convert to text chunks with contact information
+        all_texts = []
+        for doc in all_documents:
+            # Split the document into chunks
+            from llama_index.core.node_parser import SentenceSplitter
+
+            text_splitter = SentenceSplitter(
+                chunk_size=args.chunk_size, chunk_overlap=args.chunk_overlap
+            )
+            nodes = text_splitter.get_nodes_from_documents([doc])
+
+            for node in nodes:
+                # Add contact information to each chunk
+                contact_name = doc.metadata.get("contact_name", "Unknown")
+                text = f"[Contact] means the message is from: {contact_name}\n" + node.get_content()
+                all_texts.append(text)
+
+        print(f"Created {len(all_texts)} text chunks from {len(all_documents)} documents")
+        return all_texts
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    # Check platform
+    if sys.platform != "darwin":
+        print("\n⚠️  Warning: WeChat export is only supported on macOS")
+        print("   You can still query existing exports on other platforms\n")
+
+    # Example queries for WeChat RAG
+    print("\n💬 WeChat History RAG Example")
+    print("=" * 50)
+    print("\nExample queries you can try:")
+    print("- 'Show me conversations about travel plans'")
+    print("- 'Find group chats about weekend activities'")
+    print("- '我想买魔术师约翰逊的球衣,给我一些对应聊天记录?'")
+    print("- 'What did we discuss about the project last month?'")
+    print("\nNote: WeChat must be running for export to work\n")
+
+    rag = WeChatRAG()
+    asyncio.run(rag.run())

benchmarks/README.md

@@ -1,9 +1,24 @@
-# 🧪 Leann Sanity Checks
+# 🧪 LEANN Benchmarks & Testing
 
-This directory contains comprehensive sanity checks for the Leann system, ensuring all components work correctly across different configurations.
+This directory contains performance benchmarks and comprehensive tests for the LEANN system, including backend comparisons and sanity checks across different configurations.
 
 ## 📁 Test Files
 
+### `diskann_vs_hnsw_speed_comparison.py`
+Performance comparison between DiskANN and HNSW backends:
+- ✅ **Search latency** comparison with both backends using recompute
+- ✅ **Index size** and **build time** measurements
+- ✅ **Score validity** testing (ensures no -inf scores)
+- ✅ **Configurable dataset sizes** for different scales
+
+```bash
+# Quick comparison with 500 docs, 10 queries
+python benchmarks/diskann_vs_hnsw_speed_comparison.py
+
+# Large-scale comparison with 2000 docs, 20 queries
+python benchmarks/diskann_vs_hnsw_speed_comparison.py 2000 20
+```
+
 ### `test_distance_functions.py`
 Tests all supported distance functions across DiskANN backend:
 - ✅ **MIPS** (Maximum Inner Product Search)

benchmarks/benchmark_embeddings.py

@@ -0,0 +1,141 @@
+import time
+
+import matplotlib.pyplot as plt
+import mlx.core as mx
+import numpy as np
+import torch
+from mlx_lm import load
+from sentence_transformers import SentenceTransformer
+
+# --- Configuration ---
+MODEL_NAME_TORCH = "Qwen/Qwen3-Embedding-0.6B"
+MODEL_NAME_MLX = "mlx-community/Qwen3-Embedding-0.6B-4bit-DWQ"
+BATCH_SIZES = [1, 8, 16, 32, 64, 128]
+NUM_RUNS = 10  # Number of runs to average for each batch size
+WARMUP_RUNS = 2  # Number of warm-up runs
+
+# --- Generate Dummy Data ---
+DUMMY_SENTENCES = ["This is a test sentence for benchmarking." * 5] * max(BATCH_SIZES)
+
+# --- Benchmark Functions ---b
+
+
+def benchmark_torch(model, sentences):
+    start_time = time.time()
+    model.encode(sentences, convert_to_numpy=True)
+    end_time = time.time()
+    return (end_time - start_time) * 1000  # Return time in ms
+
+
+def benchmark_mlx(model, tokenizer, sentences):
+    start_time = time.time()
+
+    # Tokenize sentences using MLX tokenizer
+    tokens = []
+    for sentence in sentences:
+        token_ids = tokenizer.encode(sentence)
+        tokens.append(token_ids)
+
+    # Pad sequences to the same length
+    max_len = max(len(t) for t in tokens)
+    input_ids = []
+    attention_mask = []
+
+    for token_seq in tokens:
+        # Pad sequence
+        padded = token_seq + [tokenizer.eos_token_id] * (max_len - len(token_seq))
+        input_ids.append(padded)
+        # Create attention mask (1 for real tokens, 0 for padding)
+        mask = [1] * len(token_seq) + [0] * (max_len - len(token_seq))
+        attention_mask.append(mask)
+
+    # Convert to MLX arrays
+    input_ids = mx.array(input_ids)
+    attention_mask = mx.array(attention_mask)
+
+    # Get embeddings
+    embeddings = model(input_ids)
+
+    # Mean pooling
+    mask = mx.expand_dims(attention_mask, -1)
+    sum_embeddings = (embeddings * mask).sum(axis=1)
+    sum_mask = mask.sum(axis=1)
+    _ = sum_embeddings / sum_mask
+
+    mx.eval()  # Ensure computation is finished
+    end_time = time.time()
+    return (end_time - start_time) * 1000  # Return time in ms
+
+
+# --- Main Execution ---
+def main():
+    print("--- Initializing Models ---")
+    # Load PyTorch model
+    print(f"Loading PyTorch model: {MODEL_NAME_TORCH}")
+    device = "mps" if torch.backends.mps.is_available() else "cpu"
+    model_torch = SentenceTransformer(MODEL_NAME_TORCH, device=device)
+    print(f"PyTorch model loaded on: {device}")
+
+    # Load MLX model
+    print(f"Loading MLX model: {MODEL_NAME_MLX}")
+    model_mlx, tokenizer_mlx = load(MODEL_NAME_MLX)
+    print("MLX model loaded.")
+
+    # --- Warm-up ---
+    print("\n--- Performing Warm-up Runs ---")
+    for _ in range(WARMUP_RUNS):
+        benchmark_torch(model_torch, DUMMY_SENTENCES[:1])
+        benchmark_mlx(model_mlx, tokenizer_mlx, DUMMY_SENTENCES[:1])
+    print("Warm-up complete.")
+
+    # --- Benchmarking ---
+    print("\n--- Starting Benchmark ---")
+    results_torch = []
+    results_mlx = []
+
+    for batch_size in BATCH_SIZES:
+        print(f"Benchmarking batch size: {batch_size}")
+        sentences_batch = DUMMY_SENTENCES[:batch_size]
+
+        # Benchmark PyTorch
+        torch_times = [benchmark_torch(model_torch, sentences_batch) for _ in range(NUM_RUNS)]
+        results_torch.append(np.mean(torch_times))
+
+        # Benchmark MLX
+        mlx_times = [
+            benchmark_mlx(model_mlx, tokenizer_mlx, sentences_batch) for _ in range(NUM_RUNS)
+        ]
+        results_mlx.append(np.mean(mlx_times))
+
+    print("\n--- Benchmark Results (Average time per batch in ms) ---")
+    print(f"Batch Sizes: {BATCH_SIZES}")
+    print(f"PyTorch (mps): {[f'{t:.2f}' for t in results_torch]}")
+    print(f"MLX:           {[f'{t:.2f}' for t in results_mlx]}")
+
+    # --- Plotting ---
+    print("\n--- Generating Plot ---")
+    plt.figure(figsize=(10, 6))
+    plt.plot(
+        BATCH_SIZES,
+        results_torch,
+        marker="o",
+        linestyle="-",
+        label=f"PyTorch ({device})",
+    )
+    plt.plot(BATCH_SIZES, results_mlx, marker="s", linestyle="-", label="MLX")
+
+    plt.title(f"Embedding Performance: MLX vs PyTorch\nModel: {MODEL_NAME_TORCH}")
+    plt.xlabel("Batch Size")
+    plt.ylabel("Average Time per Batch (ms)")
+    plt.xticks(BATCH_SIZES)
+    plt.grid(True)
+    plt.legend()
+
+    # Save the plot
+    output_filename = "embedding_benchmark.png"
+    plt.savefig(output_filename)
+    print(f"Plot saved to {output_filename}")
+
+
+if __name__ == "__main__":
+    main()

benchmarks/benchmark_no_recompute.py

@@ -0,0 +1,148 @@
+import argparse
+import os
+import time
+from pathlib import Path
+
+from leann import LeannBuilder, LeannSearcher
+
+
+def _meta_exists(index_path: str) -> bool:
+    p = Path(index_path)
+    return (p.parent / f"{p.stem}.meta.json").exists()
+
+
+def ensure_index(index_path: str, backend_name: str, num_docs: int, is_recompute: bool) -> None:
+    # if _meta_exists(index_path):
+    #     return
+    kwargs = {}
+    if backend_name == "hnsw":
+        kwargs["is_compact"] = is_recompute
+    builder = LeannBuilder(
+        backend_name=backend_name,
+        embedding_model=os.getenv("LEANN_EMBED_MODEL", "facebook/contriever"),
+        embedding_mode=os.getenv("LEANN_EMBED_MODE", "sentence-transformers"),
+        graph_degree=32,
+        complexity=64,
+        is_recompute=is_recompute,
+        num_threads=4,
+        **kwargs,
+    )
+    for i in range(num_docs):
+        builder.add_text(
+            f"This is a test document number {i}. It contains some repeated text for benchmarking."
+        )
+    builder.build_index(index_path)
+
+
+def _bench_group(
+    index_path: str,
+    recompute: bool,
+    query: str,
+    repeats: int,
+    complexity: int = 32,
+    top_k: int = 10,
+) -> float:
+    # Independent searcher per group; fixed port when recompute
+    searcher = LeannSearcher(index_path=index_path)
+
+    # Warm-up once
+    _ = searcher.search(
+        query,
+        top_k=top_k,
+        complexity=complexity,
+        recompute_embeddings=recompute,
+    )
+
+    def _once() -> float:
+        t0 = time.time()
+        _ = searcher.search(
+            query,
+            top_k=top_k,
+            complexity=complexity,
+            recompute_embeddings=recompute,
+        )
+        return time.time() - t0
+
+    if repeats <= 1:
+        t = _once()
+    else:
+        vals = [_once() for _ in range(repeats)]
+        vals.sort()
+        t = vals[len(vals) // 2]
+
+    searcher.cleanup()
+    return t
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--num-docs", type=int, default=5000)
+    parser.add_argument("--repeats", type=int, default=3)
+    parser.add_argument("--complexity", type=int, default=32)
+    args = parser.parse_args()
+
+    base = Path.cwd() / ".leann" / "indexes" / f"bench_n{args.num_docs}"
+    base.parent.mkdir(parents=True, exist_ok=True)
+    # ---------- Build HNSW variants ----------
+    hnsw_r = str(base / f"hnsw_recompute_n{args.num_docs}.leann")
+    hnsw_nr = str(base / f"hnsw_norecompute_n{args.num_docs}.leann")
+    ensure_index(hnsw_r, "hnsw", args.num_docs, True)
+    ensure_index(hnsw_nr, "hnsw", args.num_docs, False)
+
+    # ---------- Build DiskANN variants ----------
+    diskann_r = str(base / "diskann_r.leann")
+    diskann_nr = str(base / "diskann_nr.leann")
+    ensure_index(diskann_r, "diskann", args.num_docs, True)
+    ensure_index(diskann_nr, "diskann", args.num_docs, False)
+
+    # ---------- Helpers ----------
+    def _size_for(prefix: str) -> int:
+        p = Path(prefix)
+        base_dir = p.parent
+        stem = p.stem
+        total = 0
+        for f in base_dir.iterdir():
+            if f.is_file() and f.name.startswith(stem):
+                total += f.stat().st_size
+        return total
+
+    # ---------- HNSW benchmark ----------
+    t_hnsw_r = _bench_group(
+        hnsw_r, True, "test document number 42", repeats=args.repeats, complexity=args.complexity
+    )
+    t_hnsw_nr = _bench_group(
+        hnsw_nr, False, "test document number 42", repeats=args.repeats, complexity=args.complexity
+    )
+    size_hnsw_r = _size_for(hnsw_r)
+    size_hnsw_nr = _size_for(hnsw_nr)
+
+    print("Benchmark results (HNSW):")
+    print(f"  recompute=True:  search_time={t_hnsw_r:.3f}s, size={size_hnsw_r / 1024 / 1024:.1f}MB")
+    print(
+        f"  recompute=False: search_time={t_hnsw_nr:.3f}s, size={size_hnsw_nr / 1024 / 1024:.1f}MB"
+    )
+    print("  Expectation: no-recompute should be faster but larger on disk.")
+
+    # ---------- DiskANN benchmark ----------
+    t_diskann_r = _bench_group(
+        diskann_r, True, "DiskANN R test doc 123", repeats=args.repeats, complexity=args.complexity
+    )
+    t_diskann_nr = _bench_group(
+        diskann_nr,
+        False,
+        "DiskANN NR test doc 123",
+        repeats=args.repeats,
+        complexity=args.complexity,
+    )
+    size_diskann_r = _size_for(diskann_r)
+    size_diskann_nr = _size_for(diskann_nr)
+
+    print("\nBenchmark results (DiskANN):")
+    print(f"  build(recompute=True, partition): size={size_diskann_r / 1024 / 1024:.1f}MB")
+    print(f"  build(recompute=False):          size={size_diskann_nr / 1024 / 1024:.1f}MB")
+    print(f"  search recompute=True (final rerank): {t_diskann_r:.3f}s")
+    print(f"  search recompute=False (PQ only):     {t_diskann_nr:.3f}s")
+
+
+if __name__ == "__main__":
+    main()

benchmarks/compare_faiss_vs_leann.py

@@ -0,0 +1,326 @@
+#!/usr/bin/env python3
+"""
+Memory comparison between Faiss HNSW and LEANN HNSW backend
+"""
+
+import gc
+import logging
+import os
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+import psutil
+from llama_index.core.node_parser import SentenceSplitter
+
+# Setup logging
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def get_memory_usage():
+    """Get current memory usage in MB"""
+    process = psutil.Process()
+    return process.memory_info().rss / 1024 / 1024
+
+
+def print_memory_stats(stage: str, start_mem: float):
+    """Print memory statistics"""
+    current_mem = get_memory_usage()
+    diff = current_mem - start_mem
+    print(f"[{stage}] Memory: {current_mem:.1f} MB (+{diff:.1f} MB)")
+    return current_mem
+
+
+class MemoryTracker:
+    def __init__(self, name: str):
+        self.name = name
+        self.start_mem = get_memory_usage()
+        self.stages = []
+
+    def checkpoint(self, stage: str):
+        current_mem = print_memory_stats(f"{self.name} - {stage}", self.start_mem)
+        self.stages.append((stage, current_mem))
+        return current_mem
+
+    def summary(self):
+        print(f"\n=== {self.name} Memory Summary ===")
+        for stage, mem in self.stages:
+            print(f"{stage}: {mem:.1f} MB")
+        peak_mem = max(mem for _, mem in self.stages)
+        print(f"Peak Memory: {peak_mem:.1f} MB")
+        print(f"Total Memory Increase: {peak_mem - self.start_mem:.1f} MB")
+        return peak_mem
+
+
+def test_faiss_hnsw():
+    """Test Faiss HNSW Vector Store in subprocess"""
+    print("\n" + "=" * 50)
+    print("TESTING FAISS HNSW VECTOR STORE")
+    print("=" * 50)
+
+    try:
+        result = subprocess.run(
+            [sys.executable, "benchmarks/faiss_only.py"],
+            capture_output=True,
+            text=True,
+            timeout=300,
+        )
+
+        print(result.stdout)
+        if result.stderr:
+            print("Stderr:", result.stderr)
+
+        if result.returncode != 0:
+            return {
+                "peak_memory": float("inf"),
+                "error": f"Process failed with code {result.returncode}",
+            }
+
+        # Parse peak memory from output
+        lines = result.stdout.split("\n")
+        peak_memory = 0.0
+
+        for line in lines:
+            if "Peak Memory:" in line:
+                peak_memory = float(line.split("Peak Memory:")[1].split("MB")[0].strip())
+
+        return {"peak_memory": peak_memory}
+
+    except Exception as e:
+        return {
+            "peak_memory": float("inf"),
+            "error": str(e),
+        }
+
+
+def test_leann_hnsw():
+    """Test LEANN HNSW Search Memory (load existing index)"""
+    print("\n" + "=" * 50)
+    print("TESTING LEANN HNSW SEARCH MEMORY")
+    print("=" * 50)
+
+    tracker = MemoryTracker("LEANN HNSW Search")
+
+    # Import and setup
+    tracker.checkpoint("Initial")
+
+    from leann.api import LeannSearcher
+
+    tracker.checkpoint("After imports")
+
+    from leann.api import LeannBuilder
+    from llama_index.core import SimpleDirectoryReader
+
+    # Load and parse documents
+    documents = SimpleDirectoryReader(
+        "data",
+        recursive=True,
+        encoding="utf-8",
+        required_exts=[".pdf", ".txt", ".md"],
+    ).load_data()
+
+    tracker.checkpoint("After document loading")
+
+    # Parse into chunks
+    node_parser = SentenceSplitter(
+        chunk_size=256, chunk_overlap=20, separator=" ", paragraph_separator="\n\n"
+    )
+
+    all_texts = []
+    for doc in documents:
+        nodes = node_parser.get_nodes_from_documents([doc])
+        for node in nodes:
+            all_texts.append(node.get_content())
+    print(f"Total number of chunks: {len(all_texts)}")
+
+    tracker.checkpoint("After text chunking")
+
+    # Build LEANN index
+    INDEX_DIR = Path("./test_leann_comparison")
+    INDEX_PATH = str(INDEX_DIR / "comparison.leann")
+
+    # Check if index already exists
+    if os.path.exists(INDEX_PATH + ".meta.json"):
+        print("Loading existing LEANN HNSW index...")
+        tracker.checkpoint("After loading existing index")
+    else:
+        print("Building new LEANN HNSW index...")
+        # Clean up previous index
+        import shutil
+
+        if INDEX_DIR.exists():
+            shutil.rmtree(INDEX_DIR)
+
+        builder = LeannBuilder(
+            backend_name="hnsw",
+            embedding_model="facebook/contriever",
+            graph_degree=32,
+            complexity=64,
+            is_compact=True,
+            is_recompute=True,
+            num_threads=1,
+        )
+
+        tracker.checkpoint("After builder setup")
+
+        print("Building LEANN HNSW index...")
+
+        for chunk_text in all_texts:
+            builder.add_text(chunk_text)
+
+        builder.build_index(INDEX_PATH)
+        del builder
+        gc.collect()
+
+        tracker.checkpoint("After index building")
+
+    # Find existing LEANN index
+    index_paths = [
+        "./test_leann_comparison/comparison.leann",
+    ]
+    index_path = None
+    for path in index_paths:
+        if os.path.exists(path + ".meta.json"):
+            index_path = path
+            break
+
+    if not index_path:
+        print("❌ LEANN index not found. Please build it first")
+        return {"peak_memory": float("inf"), "error": "Index not found"}
+
+    # Measure runtime memory overhead
+    print("\nMeasuring runtime memory overhead...")
+    runtime_start_mem = get_memory_usage()
+    print(f"Before load memory: {runtime_start_mem:.1f} MB")
+    tracker.checkpoint("Before load memory")
+
+    # Load searcher
+    searcher = LeannSearcher(index_path)
+    tracker.checkpoint("After searcher loading")
+
+    print("Running search queries...")
+    queries = [
+        "什么是盘古大模型以及盘古开发过程中遇到了什么阴暗面,任务令一般在什么城市颁发",
+        "What is LEANN and how does it work?",
+        "华为诺亚方舟实验室的主要研究内容",
+    ]
+
+    for i, query in enumerate(queries):
+        start_time = time.time()
+        # Use same parameters as Faiss: top_k=20, ef=120 (complexity parameter)
+        _ = searcher.search(query, top_k=20, ef=120)
+        query_time = time.time() - start_time
+        print(f"Query {i + 1} time: {query_time:.3f}s")
+        tracker.checkpoint(f"After query {i + 1}")
+
+    runtime_end_mem = get_memory_usage()
+    runtime_overhead = runtime_end_mem - runtime_start_mem
+
+    peak_memory = tracker.summary()
+    print(f"Runtime Memory Overhead: {runtime_overhead:.1f} MB")
+
+    # Get storage size before cleanup
+    storage_size = 0
+    INDEX_DIR = Path(index_path).parent
+    if INDEX_DIR.exists():
+        total_size = 0
+        for dirpath, _, filenames in os.walk(str(INDEX_DIR)):
+            for filename in filenames:
+                # Only count actual index files, skip text data and backups
+                if filename.endswith((".old", ".tmp", ".bak", ".jsonl", ".json")):
+                    continue
+                # Count .index, .idx, .map files (actual index structures)
+                if filename.endswith((".index", ".idx", ".map")):
+                    filepath = os.path.join(dirpath, filename)
+                    total_size += os.path.getsize(filepath)
+        storage_size = total_size / (1024 * 1024)  # Convert to MB
+
+    # Clean up
+    del searcher
+    gc.collect()
+
+    return {
+        "peak_memory": peak_memory,
+        "storage_size": storage_size,
+    }
+
+
+def main():
+    """Run comparison tests"""
+    print("Storage + Search Memory Comparison: Faiss HNSW vs LEANN HNSW")
+    print("=" * 60)
+
+    # Test Faiss HNSW
+    faiss_results = test_faiss_hnsw()
+
+    # Force garbage collection
+    gc.collect()
+    time.sleep(2)
+
+    # Test LEANN HNSW
+    leann_results = test_leann_hnsw()
+
+    # Final comparison
+    print("\n" + "=" * 60)
+    print("STORAGE + SEARCH MEMORY COMPARISON")
+    print("=" * 60)
+
+    # Get storage sizes
+    faiss_storage_size = 0
+    leann_storage_size = leann_results.get("storage_size", 0)
+
+    # Get Faiss storage size using Python
+    if os.path.exists("./storage_faiss"):
+        total_size = 0
+        for dirpath, _, filenames in os.walk("./storage_faiss"):
+            for filename in filenames:
+                filepath = os.path.join(dirpath, filename)
+                total_size += os.path.getsize(filepath)
+        faiss_storage_size = total_size / (1024 * 1024)  # Convert to MB
+
+    print("Faiss HNSW:")
+    if "error" in faiss_results:
+        print(f"  ❌ Failed: {faiss_results['error']}")
+    else:
+        print(f"  Search Memory: {faiss_results['peak_memory']:.1f} MB")
+        print(f"  Storage Size: {faiss_storage_size:.1f} MB")
+
+    print("\nLEANN HNSW:")
+    if "error" in leann_results:
+        print(f"  ❌ Failed: {leann_results['error']}")
+    else:
+        print(f"  Search Memory: {leann_results['peak_memory']:.1f} MB")
+        print(f"  Storage Size: {leann_storage_size:.1f} MB")
+
+    # Calculate improvements only if both tests succeeded
+    if "error" not in faiss_results and "error" not in leann_results:
+        memory_ratio = faiss_results["peak_memory"] / leann_results["peak_memory"]
+
+        print("\nLEANN vs Faiss Performance:")
+        memory_saving = faiss_results["peak_memory"] - leann_results["peak_memory"]
+        print(f"  Search Memory: {memory_ratio:.1f}x less ({memory_saving:.1f} MB saved)")
+
+        # Storage comparison
+        if leann_storage_size > faiss_storage_size:
+            storage_ratio = leann_storage_size / faiss_storage_size
+            print(f"  Storage Size: {storage_ratio:.1f}x larger (LEANN uses more storage)")
+        elif faiss_storage_size > leann_storage_size:
+            storage_ratio = faiss_storage_size / leann_storage_size
+            print(f"  Storage Size: {storage_ratio:.1f}x smaller (LEANN uses less storage)")
+        else:
+            print("  Storage Size: similar")
+    else:
+        if "error" not in leann_results:
+            print("\n✅ LEANN HNSW completed successfully!")
+            print(f"📊 Search Memory: {leann_results['peak_memory']:.1f} MB")
+            print(f"📊 Storage Size: {leann_storage_size:.1f} MB")
+        if "error" not in faiss_results:
+            print("\n✅ Faiss HNSW completed successfully!")
+            print(f"📊 Search Memory: {faiss_results['peak_memory']:.1f} MB")
+            print(f"📊 Storage Size: {faiss_storage_size:.1f} MB")
+
+
+if __name__ == "__main__":
+    main()

benchmarks/data/README.md

@@ -0,0 +1,44 @@
+---
+license: mit
+---
+
+# LEANN-RAG Evaluation Data
+
+This repository contains the necessary data to run the recall evaluation scripts for the [LEANN-RAG](https://huggingface.co/LEANN-RAG) project.
+
+## Dataset Components
+
+This dataset is structured into three main parts:
+
+1.  **Pre-built LEANN Indices**:
+    *   `dpr/`: A pre-built index for the DPR dataset.
+    *   `rpj_wiki/`: A pre-built index for the RPJ-Wiki dataset.
+    These indices were created using the `leann-core` library and are required by the `LeannSearcher`.
+
+2.  **Ground Truth Data**:
+    *   `ground_truth/`: Contains the ground truth files (`flat_results_nq_k3.json`) for both the DPR and RPJ-Wiki datasets. These files map queries to the original passage IDs from the Natural Questions benchmark, evaluated using the Contriever model.
+
+3.  **Queries**:
+    *   `queries/`: Contains the `nq_open.jsonl` file with the Natural Questions queries used for the evaluation.
+
+## Usage
+
+To use this data, you can download it locally using the `huggingface-hub` library. First, install the library:
+
+```bash
+pip install huggingface-hub
+```
+
+Then, you can download the entire dataset to a local directory (e.g., `data/`) with the following Python script:
+
+```python
+from huggingface_hub import snapshot_download
+
+snapshot_download(
+    repo_id="LEANN-RAG/leann-rag-evaluation-data",
+    repo_type="dataset",
+    local_dir="data"
+)
+```
+
+This will download all the necessary files into a local `data` folder, preserving the repository structure. The evaluation scripts in the main [LEANN-RAG Space](https://huggingface.co/LEANN-RAG) are configured to work with this data structure.

benchmarks/data/prompts_g5/prompt_dump_hotpot_hnsw.txt

@@ -0,0 +1,484 @@
+=== Prompt Dump for HOTPOT + HNSW ===
+Total prompts: 50
+Showing first 20 prompts:
+
+==================================================
+PROMPT #1:
+==================================================
+Le samedi est le 272 jour de l'année 1962. Naissances Janne Andersson, footballeur suédois devenu entraîneur Lauren Brice (morte le ), actrice pornographique américaine Néstor Clausen, footballeur argentin Nicky Walker, footballeur britannique Olivier Trassard, footballeur français reconverti entraîneur Roger Bart, acteur américain Solal, chanteur français Stefan Niederseer, skieur alpin autrichien Décès Owe Jonsson (né le ), athlète suédois spécialiste du sprint Robert Burton (né le ), acteur américain Sunsiaré de Larcône (née le ), auteure française Événements investiture d'Ahmed Benbella à la présidence de l'Assemblée nationale d'Algérie. lancement de Alouette 1 dans l'espace. Le Canada devient le troisième pays à avoir un satellite artificiel autour de la terre Voir aussi 29 septembre et septembre 1962Le samedi est le 136 jour de l'année 1959. Naissances Andrew Litton, chef d'orchestre et pianiste américain Cynthia del Águila, femme politique guatémaltèque Gerard Donovan, écrivain irlandais Greg Johnston, rameur néo-zélandais Ioánnis Andrianós, homme politique grec Jean-Michel Veillon, musicien français Mare Winningham, actrice américaine Martin Pado, homme politique slovaque Mitch Webster, joueur américain de baseball Ofelia Cano, actrice mexicaine Roberto Piazza, acteur, chanteur, écrivain et styliste argentin Vincent de Brus, réalisateur, scénariste et perchman français Yannick Letty, écrivain français Décès Edward Winslow Gifford (né le ), ethnologue américain Elisha Scott (né le ), joueur de football irlandais Friedrich Grimm (né le ), personnalité politique allemande Gyula Kiss (né le ), joueur international hongrois de football William Hammond Wright (né le ), astronome américain Événements Fin du championnat d'Israël de football 1958-1959 Début du tour d'Italie de 1959Derrick Williams, basketteur américain. 1992 : Yasser al-Shahrani, footballeur saoudien. (38 sélections en équipe nationale). Elena Cecchini, cycliste sur piste et sur route italienne. Championne du monde de cyclisme sur route du contre la montre par équipes 2018. Jón Daði Böðvarsson, footballeur islandais. (26 sélections en équipe nationale). 1993 : Pierre Lees-Melou, footballeur français. Danijel Petković, footballeur monténégrin. (17 sélections en équipe nationale). Norman Powell, basketteur américain. 1994 : Matt Murray, hockeyeur sur glace canadien. Alexandra Raisman, gymnaste artistique américaine. Championne olympique du concours général par équipes et du sol, médaillée de bronze de la poutre aux Jeux de Londres 2012 puis championne olympique du concours général par équipes et médaillée d'argent du concours général individuel ainsi qu'au sol aux Jeux de Rio 2016. Championne du monde de gymnastique artistique du concours général par équipes 2011 et 2015. Hannah Blundell footballeuse anglaise 1995 : José Gayà, footballeur espagnol. 1996 : Dorian Godon, cycliste sur route français. David Pastrňák, hockeyeur sur glace tchèque. 1998 : Javi Puado, footballeur espagnol. 1999 : Ibrahima Konaté, footballeur français. Terem Moffi, footballeur nigérian. 2000 : Krzysztof Kubica, footballeur polonais. Claire Liu, joueuse de tennis américaine. Arthur Theate, footballeur belge. (4 sélections en équipe nationale). 2001 : Douaa Foudali, escrimeuse marocaine. Sturla Ottesen, footballeur norvégien. 2002 : Manfred Ugalde, footballeur costaricien. (1 sélection en équipe nationale). 2004 : Pedro Acosta, pilote de moto espagnol. 2005 : Bella Sims, nageuse américaine. Médaillée d'argent du relais 4 × 200 m nage libre aux Jeux olympiques de Tokyo en 2021. Décès de 1901 à 1950
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: 2014 S/S is the debut album of a South Korean boy group that was formed by who?
+A: YG Entertainment
+
+Q: What government position was held by the woman who portrayed Corliss Archer in the film Kiss and Tell?
+A: Chief of Protocol
+
+Q: What science fantasy young adult series, told in first person, has a set of companion books narrating the stories of enslaved worlds and alien species?
+A: Animorphs
+
+Q: Are the Laleli Mosque and Esma Sultan Mansion located in the same neighborhood?
+A: no
+
+Q: The director of the romantic comedy "Big Stone Gap" is based in what New York city?
+A: Greenwich Village, New York City
+
+Q: Were Scott Derrickson and Ed Wood of the same nationality?
+A: 
+==================================================
+
+==================================================
+PROMPT #2:
+==================================================
+Whiting); Mildred's irritating younger brother Raymond (Tommy Bernard, Kenny Godkin); and Corliss's rival, Betty Cameron (Delores Crane). Meet Corliss Archer was written by F. Hugh Herbert, who first introduced the character and her friends in a Good Housekeeping magazine story "A Private Affair," the first of a series of stories. Kiss and Tell was a 1943 play that was adapted for a 1945 film starring Shirley Temple. The 1949 sequel, A Kiss for Corliss, was re-released in 1954. Television Herbert's Meet Corliss Archer property was adapted to television in 1951 and 1952 with live broadcasts produced and aired by CBS. In 1954 and 1955 Ziv Television Programs produced a syndicated telefilm series starring Ann Baker and Mary Brian. One of the show's unique features was the occasional cut to a comic-book-style drawing, with announcer's commentary, that illustrated the current story situation and was used several times during each episode. The program was produced by Ziv Productions. Several episodes of the Ziv version are available on DVD, and some are also available from the Internet Archive. Robin Morgan portrayed Corliss in a live telecast of Kiss and Tell on The Alcoa Hour (August 5, 1956), with Warren Berlinger as Dexter. Comics Radio listeners had to use their imaginations to visualize Corliss, her friends and her town. But those imaginations got a boost in 1948 when the Meet Corliss Archer comic book, published by Fox Feature Syndicate, came out in three issues from March to July 1948. Al Feldstein (Albert B. Feldstein), later the editor of Mad, wasEdward G. Robinson's Big Town, The Eddie Bracken Show, Favorite Story, Four Star Playhouse, The Gallant Heart, One Man's Family, Sears Radio Theater, and Stars over Hollywood. She also played several characters including Joanne Allen and Mrs. Hodges on the radio show Adventures in Odyssey produced by Focus on the Family broadcasting. She co-starred with Jimmy Lydon in the CBS situation comedy Young Love (1949–50), and she had recurring roles on The Adventures of Ozzie and Harriet (as teenager Emmy Lou), The Red Skelton Show, and People Are Funny. She recorded with jazz vocalist Mel Torme and his vocal group the Mel-Tones. Her eight-year run starring as teenager Corliss Archer on CBS's Meet Corliss Archer left a lasting impression, though Shirley Temple starred in the film adaptations, Kiss and Tell and A Kiss for Corliss. The radio program was the CBS answer to NBC's popular A Date with Judy. Despite the long run of Meet Corliss Archer, fewer than 24 episodes are known to exist. Waldo later turned down the offer to portray Corliss in a television adaptation (the role was recast with Ann Baker taking the role for TV). In 1948, the Meet Corliss Archer comic book, using Waldo's likeness, published by Fox Feature Syndicate, appeared for a run of three issues from March to July 1948, using the original scripts. On April 1, 1948, Waldo married playwright Robert E. Lee, the writing partner of Jerome Lawrence. The couple had two children, and remained married until his death in 1994. Waldo made a rare on-screenKiss and Tell is a 1945 American comedy film starring then 17-year-old Shirley Temple as Corliss Archer. In the film, two teenage girls cause their respective parents much concern when they start to become interested in boys. The parents' bickering about which girl is the worse influence causes more problems than it solves. The movie was based on the Broadway play Kiss and Tell, which was based on the Corliss Archer short stories. The stories, play and movie were all written by F. Hugh Herbert. A sequel film, A Kiss for Corliss, was released in 1949 and also starred Temple, but was not written by Herbert. Plot When their booth at a USO bazaar fails to attract customers, teenager Corliss Archer suggests to her best friend, Mildred Pringle, that they sell kisses. The idea becomes a success among the soldiers visiting the bazaar, and business is booming, until the girls' mothers find out about it. Despite the fact that it was Corliss' idea, Mrs. Archer blames Mildred for the girls' behavior. Mrs. Archer's assumptions greatly upset Mr. Pringle when he hears about them at the Pringle dinner table. He decides the family, including Mildred and their son Raymond, will refrain from socializing with the Archers in the future. That same night, Corliss pretends to be older than she is and starts flirting shamelessly with young Private Jimmy Earhart, who has been invited into the Archer home for dinner. Corliss is actually dating the boy next door, Dexter Franklin. Late that night, Lenny Archer returns home on
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Were Scott Derrickson and Ed Wood of the same nationality?
+A: yes
+
+Q: 2014 S/S is the debut album of a South Korean boy group that was formed by who?
+A: YG Entertainment
+
+Q: What science fantasy young adult series, told in first person, has a set of companion books narrating the stories of enslaved worlds and alien species?
+A: Animorphs
+
+Q: Are the Laleli Mosque and Esma Sultan Mansion located in the same neighborhood?
+A: no
+
+Q: The director of the romantic comedy "Big Stone Gap" is based in what New York city?
+A: Greenwich Village, New York City
+
+Q: What government position was held by the woman who portrayed Corliss Archer in the film Kiss and Tell?
+A: 
+==================================================
+
+==================================================
+PROMPT #3:
+==================================================
+this from happening, and is joined by one of the descendants of the 'altered' humans from the second book. Unknown to Lalelelang, she will discover shocking things about not only the telepathic humans, but a couple of alien species from the first two books as well as herself. Significant parts of the action of the novels take place between the various non-human species that make up both The Weave and the Amplitur's allies, giving an outside perspective of "normal" human interactions and events as the author's apparent commentary on current events. Reception Kirkus Reviews criticized the first book as having no plot or action. Publishers Weekly praises the second book in the series as being an improvement over the first and having more depth. References Science fiction book series Science fiction novel trilogies Novels by Alan Dean FosterTV science fiction series: Star Trek, Stargate, Firefly, Battlestar Galactica and Farscape (the void of the series title being both an allusion to this, as well as 'Sliding Void' being used by spacers as slang for sub-warp transit). The first novel, Void all the Way Down, focuses on the inclusion of Calder, an exiled prince from a medieval-level failed colony world, into the ship's crew as they embark on running supplies to an illegal mining operation. The second novel, Anomalous Thrust, finds the crew stranded on a world boycotted by civilisation for its human colonists' enslavement of the local sentient species. Here, they find they can only escape by taking part in a solar sail race to recover valuable alien garbage jettisoned by a moon-sized vessel making a periodic tour of multiple galaxies - a trope inspired by Rendezvous with Rama. Far-called series The Far-called sequence is a fantasy series from Stephen Hunt set on the world of Pellas. The first book in the series, In Dark Service was published by Gollancz, the science fiction and fantasy imprint of Hachette and Orion Publishing, on 15 May 2014. The second novel in the series, Foul Tide's Turning, was released on 21 May 2015. The third novel in the series, The Stealers' War, was released on 16 March 2016. Hunt mentioned at Comic-Con that his books set in Pellas draw on the events of the U.S. Civil War, just as Game of Thrones draw on the feuding of 15th-century England's Wars of the Roses (an idea that cametale of forbidden love and Machiavellian politics. With detailed history, architecture, customs, inventions, and landscape, Jae creates a fascinating new world to explore." The School Library Journal called it "An engaging and amusing read", but said "the concepts and characters are not fully developed." Study Breaks listed the first book as one of "5 YA Books By Hispanic Authors To Read During National Hispanic Heritage Month", and said "it starts big and stays strong throughout." Meanwhile, Kirkus Reviews called the first book "deeply flawed", and complained of "pacing issues, plot holes, and flat secondary characters", but said the second book was "an improvement on the first outing" and that the third book was "dramatically improved over previous entries". The books' alien vocabulary has been both praised and criticized. The School Library Journal indicated that the language was part of what made the books' world "beautifully conceived...with unique details", but Kirkus Reviews complained about "the inaccessible and forced jargon". References Young adult novel series Science fiction novel trilogies LGBT speculative fiction novels Debut science fiction novels Book series introduced in 2016 Novels set on fictional planets LGBT-related young adult novels 2010s LGBT novels Works published under a pseudonym American science fiction novels American LGBT novels American young adult novels 2016 American novels 2017 American novels 2018 American novels 2018 debut novels Skyhorse Publishing books 2016 LGBT-related literary works
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Were Scott Derrickson and Ed Wood of the same nationality?
+A: yes
+
+Q: What government position was held by the woman who portrayed Corliss Archer in the film Kiss and Tell?
+A: Chief of Protocol
+
+Q: 2014 S/S is the debut album of a South Korean boy group that was formed by who?
+A: YG Entertainment
+
+Q: Are the Laleli Mosque and Esma Sultan Mansion located in the same neighborhood?
+A: no
+
+Q: The director of the romantic comedy "Big Stone Gap" is based in what New York city?
+A: Greenwich Village, New York City
+
+Q: What science fantasy young adult series, told in first person, has a set of companion books narrating the stories of enslaved worlds and alien species?
+A: 
+==================================================
+
+==================================================
+PROMPT #4:
+==================================================
+Laleli Mosque () is a mosque in the Abdi Çavuş quarter of the walled city of Nicosia, currently located in North Nicosia. It is located on Ali Ruhi Street. Its name, meaning "the Mosque with Tulips", is thought to be derived from the tulip motifs adorning its original minaret. Originally a small medieval chapel, it was enlarged and converted to a mosque in the 19th century. History It is understood from the mosque's architecture that the building was originally a medieval chapel. After the Ottoman conquest, it was initially converted to a small mosque without a minaret (mescit), and was known as "Laleli Cami-i Minareli Mescidi" in the 18th century. In the late 20th century, a marble inscription dated to 1742-43 was found in a building at the southwestern corner of the mosque's courtyard. Written in the talik style by a calligrapher named "Ârif", the inscription revealed the existence of a teacher's school ("Laleli Cami-i Minareli Mescit Muallimhâne Mektebi") at the site at the time. In 1827, it was enlarged by the addition of a minaret, minbar, mihrab, a new ceiling and a narthex (son cemaat yeri) and thus converted into a full mosque. The construction was financed by Ali Ruhi Efendi, the governor of Cyprus at the time, and the mosque was inscribed amongst the property of the newly founded Ali Ruhi Efendi Foundation on 24 December 1827 for maintenance. The responsibility for the maintenance changed hands over the course of the 19th century, as it was recorded amongst the property of Seyit Mehmet AğaLaleli (meaning 'with tulips' in Turkish) is a neighbourhood of Fatih, Istanbul, Turkey, lying between Beyazıt and Aksaray. It is known for its large textile wholesaling business and is home to the Literature and Science Faculties of Istanbul University, designed by Sedad Hakkı Eldem and Emin Onat in the 1940s. It is served by a stop on the T1 tram line which runs along Ordu Caddesi. The most prominent historic monument in Laleli is the Laleli Mosque, a work of architect Mehmed Tahir Ağa that was originally constructed in the 1760s. It was built for Sultan Mustafa III whose tomb it contains. An attractive sebil or water dispensary stands on the street side of the complex surrounding the mosque. The mosque stands above a large basement that is now filled with clothes shops. Across the road from the mosque is the Koca Ragıp Paşa complex, also designed by Mehmed Tahir Ağa in 1762. It was undergoing restoration for much of the 2010s. Lurking in the back streets is the much older Bodrum Mosque (AKA Mesih Paşa Cami), which started life as a 10th-century Byzantine church attached to the Myreiaion Palace. Beside it is an underground cistern, probably of similar date. Both stand on the site of a lost Rotunda dating back to the fifth century which is believed to have been the second largest such circular Roman temple after the Pantheon in Rome itself. Also in Laleli is the Big Stone Han (Büyük Taş Hanı in Turkish) which was probably part of the Laleli Mosque complexThe Esma Sultan Mansion (), a historical yalı located on the Bosphorus in the Ortaköy neighborhood of Istanbul, Turkey and named after its original owner Princess Esma Sultan, is used today as a cultural center after being redeveloped. History The three-storey brick manor was designed by the renowned architect Sarkis Balyan and finished in 1875 next to Ortaköy Mosque. It was presented to the Princess Esma Sultan, the daughter of Ottoman Sultan Abdulaziz, as a wedding gift in 1889. The mansion remained in the possession of the Ottoman dynasty until 1915. Subsequently, the building was used first as a tobacco warehouse and then as a coal depot from 1920 until 1975 when it was destroyed by a fire. Redevelopment The ruin, consisting only of the outer walls of the building, was purchased in the early 1990s by The Marmara Collection. Following a renovation with additions designed by architects Haluk Sezgin and Philippe Robert, the mansion was opened in 2001 as a multipurpose event venue. Within the brick exterior, which was left as the original, a steel and glass structure is incorporated. According to the GAD Architecture Group, a redesign by the architect Gökhan Avcıoğlu was completed in 2005. The building includes a bar, a restaurant, and an event hall at several levels. The building is situated in a garden of . The ground floor is wide, long and high. The first floor is merged with the second floor, having dimensions of wide, long and high. Esma Sultana Mansion, run by The Marmara Hotel chain as a
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Were Scott Derrickson and Ed Wood of the same nationality?
+A: yes
+
+Q: What government position was held by the woman who portrayed Corliss Archer in the film Kiss and Tell?
+A: Chief of Protocol
+
+Q: What science fantasy young adult series, told in first person, has a set of companion books narrating the stories of enslaved worlds and alien species?
+A: Animorphs
+
+Q: 2014 S/S is the debut album of a South Korean boy group that was formed by who?
+A: YG Entertainment
+
+Q: The director of the romantic comedy "Big Stone Gap" is based in what New York city?
+A: Greenwich Village, New York City
+
+Q: Are the Laleli Mosque and Esma Sultan Mansion located in the same neighborhood?
+A: 
+==================================================
+
+==================================================
+PROMPT #5:
+==================================================
+of Virginia at Wise (2001), and The University of New Haven, Connecticut (2005, 2016). She is host of the Library of Virginia Literary Awards (11 years), was host of the Poets and Writers Gala in New York City (2016), The Audio Publishers Association Audies Gala (2014), and the Barnes & Noble Writers for Writers Awards. She is permanent host of the Erma Bombeck/Arizona Women's Board Annual Authors Luncheon that is committed to the prevention of kidney disease through awareness, education and research. Novels Big Stone Gap: a novel. New York: Random House, 2000. Big Cherry Holler: a Big Stone Gap novel. New York: Random House, 2001 Milk Glass Moon : a Big Stone Gap novel. New York: Random House, 2002. Lucia, Lucia: a novel. New York: Random House, 2004. Home to Big Stone Gap; a novel. New York: Random House, 2006. The Queen of the Big Time : a novel. New York: Random House, 2004. Rococo: a novel. New York: Random House, 2005. Very Valentine: a novel. New York: Random House, 2009. Viola in reel life. New York: Harper Teen, 2009. Brava, Valentine: a novel. Harper Collins Publishers, 2010 Viola In the Spotlight New York: Harper Teen, 2011. The Shoemaker's Wife: a novel.New York: Harper Collins Publishers, 2012. The Supreme Macaroni Company : a novel. New York : Harper Collins Publishers, 2013. All The Stars in the Heavens : a novel. New York: Harper, Harper Collins Publishers, 2015. Kiss Carlo. New York: Harper Collins Publishers, 2017 Tony's Wife: a novel. New York: Harper Collins Publishers, 2018of Adriana Trigiani. During the 1990s, Trigiani wrote and directed an award-winning documentary Queen of The Big Time (1996), the story of her father's hometown of Roseto, Pennsylvania, shown in film festivals in London and Hong Kong, co-produced Green Chimneys, and later contributed to PBS documentary The Italian Americans. In 2014, Trigiani directed the major motion picture Big Stone Gap (film), a romantic comedy film adaptation of her namesake bestselling novel, produced by Donna Gigliotti for Altar Identity Studios, a subsidiary of Media Society. Big Stone Gap (film) is a story of family secrets and self-discovery in an Appalachian coal-mining town of the late 1970s. The award-winning ensemble cast includes Ashley Judd, Whoopi Goldberg, Jane Krakowski, Jenna Elfman, and Patrick Wilson. Released on October 9, 2015 by Picturehouse (company), Trigiani's narrative directorial debut arrived nearly 30 years after the sale of her first screenplay, Three to Get Married, produced by Kate Benton in 1986. Opening the Virginia Film Festival, Big Stone Gap (film) was ranked among the top 250 grossing women directed films of 2014. Media appearances Trigiani and her work have regularly been featured on NBC's Today Show. She was profiled on CBS Sunday Morning, appeared on The View, Good Day NY with Rosanna Scotto and Lori Stokes, and is heard regularly on NPR around the country. Trigiani has lectured at New York University and the New School for Social Research, has been a commencement speaker and received honorary degrees from Emory & Henry College (2018), Saint Mary's College, Notre Dame, Indiana (2002), The UniversityBig Stone Gap is a 2014 American drama romantic comedy film written and directed by Adriana Trigiani and produced by Donna Gigliotti for Altar Identity Studios, a subsidiary of Media Society. Based on Trigiani's 2000 best-selling novel of the same name, the story is set in the actual Virginia town of Big Stone Gap circa 1970s. The film had its world premiere at the Virginia Film Festival on November 6, 2014. The film was released on October 9, 2015, by Picturehouse. The film was released in Blu-Ray by Universal Pictures Home Entertainment on February 2, 2016. Plot In 1978, 40-year-old independent woman Ave Maria Mulligan owns her dead father's pharmacy in her hometown of Big Stone Gap, Virginia. Ave's mother, an immigrant from Italy, is the town seamstress. Ave is heavily involved in her community, home delivering medications to the country folk, volunteering on the coal mining town's Emergency Response Team, and directing the town's annual production of "Trail of the Lonesome Pine", based on the novel by John Fox Jr. Ave delivers medications to Nan McChesney and is pleased to see Nan's son Jack, a local coal miner and a former schoolmate. Jack tries to make small talk, but Ave leaves when Nan tells her that 40 is not too old to still have children. Spec Broadwater, the town's lawyer and Ave's friend, brings news that her momma has died. At the graveside Fleeta Mullins, Ave's outspoken friend and pharmacy worker, chastises Ave for placing a rose on her father's grave, remarking how badly he
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Were Scott Derrickson and Ed Wood of the same nationality?
+A: yes
+
+Q: What government position was held by the woman who portrayed Corliss Archer in the film Kiss and Tell?
+A: Chief of Protocol
+
+Q: What science fantasy young adult series, told in first person, has a set of companion books narrating the stories of enslaved worlds and alien species?
+A: Animorphs
+
+Q: Are the Laleli Mosque and Esma Sultan Mansion located in the same neighborhood?
+A: no
+
+Q: 2014 S/S is the debut album of a South Korean boy group that was formed by who?
+A: YG Entertainment
+
+Q: The director of the romantic comedy "Big Stone Gap" is based in what New York city?
+A: 
+==================================================
+
+==================================================
+PROMPT #6:
+==================================================
+SS501 is a South Korean boy band consisting of five members: Kim Hyun-joong, Heo Young-saeng, Kim Kyu-jong, Park Jung-min and Kim Hyung-jun. They debuted in 2005 with the EP Warning, which included their first number one song, "Never Again". In 2007, SS501 entered the Japanese market with the maxi single Kokoro, which debuted at number five on the Oricon chart. The following year, in 2008, they became the first South Korean artists to win the award for Best New Artist at the Japan Gold Disc Awards. SS501 has released one studio album and seven EPs in Korean, and two studio albums and three EPs in Japanese. The group has been inactive since 2010, however the sub-group Double S 301, composed of members Heo Young-saeng, Kim Kyu-jong and Kim Hyung-jun, continues to release music. Etymology The name of the band is a combination of letters and numbers that have special meaning. The first “S” stands for "superstar", the second “S” is an abbreviation for "singer", and the combination of 5, 0, and 1 symbolizes "five members united as one forever". History Pre-debut Before debuting with SS501, most of the members had some experience in the entertainment world. Kim Hyun-joong was initially about to debut in a five-member group which included Han Yeon of B2Y (now disbanded) who was to be the leader and Kim would be the youngest member. During 2001 and 2002, while working as a waiter in a family restaurant located at Jamsil-dong, Kim was introduced to a new set up management company CEO. WhileS (에스) is a South Korean project group consisting of three members: Kangta, Lee Ji-hoon and Shin Hye-sung. The group debuted in 2003, under the SM Entertainment label. After 11 years, they released and promoted another mini-album in 2014. History In 2003, S released their first album Fr.In.Cl, which stands for Friends in Classic. In 2014, after more than a decade in hiatus, the group released their second mini-album Autumn Breeze on October 27. As member Kangta wrote and produced the songs, it was said that his "unique music style and the trio’s harmonious voices will create beautiful ballads". On October 18, the group performed their title track "Without You (하고 싶은 거 다)" for the first time at SM Entertainment's agency-wide concert SM Town in Shanghai. The music video for the song was released on October 24, starring Kwon Yuri of Girls' Generation. The group continued to promote the song on various South Korean music programs, such as on Immortal Songs 2 on November 3. Band members Kangta Shin Hye-sung Lee Ji-hoon Discography Studio albums EP Concert tours SMTown 2014: SM Town Live World Tour IV Awards and nominations References South Korean boy bands South Korean contemporary R&B musical groups South Korean musical trios Musical groups established in 2003 Musical groups disestablished in 2014 Kangta2014 S/S is the debut album of South Korean group Winner. It was released on August 12, 2014, by the group's record label, YG Entertainment. The members were credited for writing the lyrics and composing the majority of the album's songs. Composition The members produced the majority of the music for this album themselves, with the help of other producers such as Choice 37, B.I., Airplay, and others. The album was highlighted for incorporating elements generally absent from K-pop releases, including hints of acoustic and alternative rock. Track listing Reception The title track "Empty" became a hit in South Korea, topping the Gaon Chart and Billboard's K-Pop Hot 100 chart. Internationally, 2014 S/S took the top spot in Billboard's World Album Chart. Charts Sales References External links 2014 albums YG Entertainment albums Winner (band) albums Korean-language albums
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Were Scott Derrickson and Ed Wood of the same nationality?
+A: yes
+
+Q: What government position was held by the woman who portrayed Corliss Archer in the film Kiss and Tell?
+A: Chief of Protocol
+
+Q: What science fantasy young adult series, told in first person, has a set of companion books narrating the stories of enslaved worlds and alien species?
+A: Animorphs
+
+Q: Are the Laleli Mosque and Esma Sultan Mansion located in the same neighborhood?
+A: no
+
+Q: The director of the romantic comedy "Big Stone Gap" is based in what New York city?
+A: Greenwich Village, New York City
+
+Q: 2014 S/S is the debut album of a South Korean boy group that was formed by who?
+A: 
+==================================================
+
+==================================================
+PROMPT #7:
+==================================================
+Donald G. Malcolm (March 26, 1919 - June 18, 2007) was an American organizational theorist, professor and dean at Cal State L.A.'s College of Business and Economics and management consultant, known as co-developer of the Performance, Evaluation, and Review Technique (PERT). Biography Malcolm was born and raised in Indianapolis. In 1940 he obtained his BSc in Public Service Engineering from Purdue University, and after the war his MSc in industrial engineering. In World War II Malcolm served on an aircraft carrier in the Pacific as radioman. After his graduation Malcolm started his academic career as research team leader of the Operations Research Team at the Johns Hopkins University. In 1950 he was lecturer in Mechanical Engineering, and by 1952 assistant professor of industrial engineering at the University of California, Berkeley. In the 1950s he was also management consultant for Booz, Allen and Hamilton in Chicago, Illinois, and in those days participated with the US Navy Special Projects Office, especially with Willard Fazar, in the development of PERT. In 1954-1955 he was president of the American Institute of Industrial Engineers, now Institute of Industrial and Systems Engineers. In 1962 he settled as management consultant with his own consultancy firm Management Technology Inc. From 1962 to 1967 he also directed the National Safety Standards division of the US Department of Transportation. From 1970 he was one of the Senior Vice Presidents of Computer Applications, Inc., and from 1972 to 1981 he served as dean of the College of Business and Economics at the California State University, Los Angeles.William James Reddin also known as Bill Reddin (May 10, 1930 – June 20, 1999) was a British-born management behavioralist, theorist, writer, and consultant. His published works examined and explained how managers in profit and non-profit organizations behaved under certain situations and conditions. The focus of his work was to understand to what extent managers were effective in their role and successful in managing situations to have the right impact on the organization's objectives. Through extensive research Reddin concluded that there is no ideal management style. He put forward that there was only one realistic and unambiguous definition of managerial effectiveness, the extent to which a manager or leader achieves the output requirements of the position. This is the manager's or leader's only job: to be effective. Reddin was often quoted as saying both in his writings, to his clients and to his students, that there is no ideal style of managing; and there is no one way to make an organization more effective. He wrote in his 1988 book, The Output Oriented Manager, "… no list exists in the world showing characteristics of effective managers, or, of effective organizations, which apply generally. He went on to write that his works' intent were to serve as a substitute for prescriptive management-guru advice prevalent in modern business, to enable the manager and leader in diagnosing what is the true situation and what are the true needs. They served managers and leaders to make sound decisions on how best to arrive at their planned objectives. This concept ofThis is an annotated list of important business writers. It is in alphabetical order based on last name. For quick navigation, click on one of the letters: A David Aaker (born 1938) - marketing, brand strategy Wil van der Aalst James Abegglen (1926–2007) - management and business in Japan Bodo Abel Russell L. Ackoff (1919–2009) - operations research, organizational theory John Adair (born 1934) - leadership Karol Adamiecki (1866–1933) - management Ichak Adizes Niclas Adler (born 1971) - Swedish organizational theorist Charles Constance César Joseph Matthieu d'Agoult Yoji Akao Ali Akdemir Mark Albion (born 1951) - values-based business Howard E. Aldrich (born 1940s) - American sociologist and organizational theorist Leon P. Alford (1877–1942) - scientific management Tim Ambler (born 1938) - marketing effectiveness Igor Ansoff (1918–2002) - strategic management Ingeman Arbnor Chris Argyris (1923–2013) - learning systems, learning organization Horace Lucian Arnold (1837–1915) Neal Ashkanasy B Stephen R. Barley (born 1953) - technology, organizational change, organizational culture Chester Barnard (1886–1961) - management Gary S. Becker Charles Bedaux (1886–1944) - scientific management Warren Bennis (1925–2014) - leadership studies Per Olof Berg (born 1946) - Swedish organizational theorist Manfred Berliner Björn Bjerke Patrick Blackett, Baron Blackett (1897–1974) - operations research Ken Blanchard Charles Bosanquet Matthew Boulton (1728–1809) - labor productivity Marvin Bower Richard Boyatzis (born 1946) - emotional intelligence, behavior change, and competence Leland Lawrence Briggs (1893–1975) - American accounting scholar John Seely Brown Wilfred Brown, Baron Brown Nils Brunsson (born 1946) - institutionalized hypLine truncated
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Were Scott Derrickson and Ed Wood of the same nationality?
+A: yes
+
+Q: What government position was held by the woman who portrayed Corliss Archer in the film Kiss and Tell?
+A: Chief of Protocol
+
+Q: What science fantasy young adult series, told in first person, has a set of companion books narrating the stories of enslaved worlds and alien species?
+A: Animorphs
+
+Q: Are the Laleli Mosque and Esma Sultan Mansion located in the same neighborhood?
+A: no
+
+Q: The director of the romantic comedy "Big Stone Gap" is based in what New York city?
+A: Greenwich Village, New York City
+
+Q: Who was known by his stage name Aladin and helped organizations improve their performance as a consultant?
+A: 
+==================================================
+
+==================================================
+PROMPT #8:
+==================================================
+Civic Center). The Lewiston Maineiacs, the only American team to have played in the Quebec Major Junior Hockey League, played their first season in 2003–2004 and dissolved the team after the 2010–2011 season. The Colisée is also the home to the state Class A and Class B high school hockey championships each year. The city as a whole is known for its strong passion for the game of hockey, likely related to its French American heritage. Two Lewiston schools, Lewiston High School and St. Dominic Regional High School (now in Auburn), combine for over half of the state class A high school hockey championships in the state's history. During the 2013–2014 American Hockey League season, the Portland Pirates played their first 12 home games at the Colisée while the Cumberland County Civic Center is being renovated. The junior Maine Nordiques of the North American Hockey League have played their home games at the Colisée since 2019. Ali vs. Liston rematch In May 1965, Lewiston became the venue for a heavyweight title rematch between Muhammad Ali and Sonny Liston; Ali had defeated Liston in a controversial fight in Miami Beach, Florida in February 1964, and the World Boxing Council was demanding an immediate rematch, which was against World Boxing Association rules (the WBA eventually stripped Ali of their title). The rematch was originally planned to be held in Boston, but was halted by Massachusetts boxing authorities due to licensing issues. Promoters were eventually able to frame a lucrative deal moving the fight to the Colisée in Lewiston.As the venue held less than 3,700 spectators, only 2,434 fans were present, setting an all-time record for the lowest attendance for a heavyweight championship match. The fight was the scene of the famous photograph of Ali standing over Liston taunting him with his glove. Ali won by first round knockout. Lewiston Twins (1891–1930) Lewiston was home to minor league baseball. Beginning in 1891, Lewiston was home to the Lewiston Twins and other teams, who played in various seasons through 1930. Lewiston teams played as members of the New England League (1891–1896, 1901), Maine State League (1907), Atlantic Association (1908) and New England League (1914–1915, 1919, 1926–1930). Baseball Hall of Fame member Jesse Burkett managed the Lewiston Twins in 1928 and 1929. Between 1901 and 1919, Lewiston teams played home games at A.A.A. Park. Beginning in 1926, the Lewiston Twins played home games at Lewiston Athletic Park. Maine Nordiques (1973–1977) The Maine Nordiques were a professional hockey team that operated in the former North American Hockey League from 1973 to 1977. They were based at the Central Maine Youth Center in Lewiston. The Nordiques served as a farm club for the Quebec Nordiques of the World Hockey Association. Lewiston Maineiacs The Lewiston Maineiacs were a major junior hockey team that played in the Quebec Major Junior Hockey League (QMJHL). The Maineiacs moved to Lewiston in 2003 from Sherbrooke, Quebec, and were the only team in the QMJHL in the United States. They played their home games at the Androscoggin Bank Colisée. In 2006–2007, the Maineiacs wonThe Lewiston Maineiacs were a junior ice hockey team of the Quebec Major Junior Hockey League based in Lewiston, Maine, United States. The team played its home games at the Androscoggin Bank Colisée. They were the second QMJHL team in the United States, and the only one to play a full season. They won the President's Cup in 2007, becoming the only American based franchise to accomplish the feat. On May 31, 2011, the QMJHL decided to fold the team following eight years of financial losses. History Their franchise was granted for the 1969–70 season. They played in Trois-Rivières, Quebec, from 1969 to 1992. They were known as the Ducs from 1969 to 1973, and as the Draveurs from 1973 to 1992. They moved to Sherbrooke, Quebec, in 1992 to become the Faucons (1992–1998) and the Castors (1998–2003). The Castors moved to Lewiston, Maine, in 2003. The MAINEiacs clinched the Jean Rougeau Trophy for first overall in the regular season for 2006–07. During the same season, Lewiston increased its annual attendance totals to 94,903 tickets sold. In the playoffs that year, Lewiston went 16–1 to cruise to their first President's Cup. 2007 Memorial Cup After winning the 2007 President's Cup, the MAINEiacs won eligibility to play in the 2007 Memorial Cup in Vancouver, British Columbia. Following an opening win over the Medicine Hat Tigers, the MAINEiacs suffered a huge loss with the injury of their captain, Marc-André Cliche, and never really found their scoring touch. They crashed out of the tournament in fourth place after losing
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Were Scott Derrickson and Ed Wood of the same nationality?
+A: yes
+
+Q: What government position was held by the woman who portrayed Corliss Archer in the film Kiss and Tell?
+A: Chief of Protocol
+
+Q: What science fantasy young adult series, told in first person, has a set of companion books narrating the stories of enslaved worlds and alien species?
+A: Animorphs
+
+Q: Are the Laleli Mosque and Esma Sultan Mansion located in the same neighborhood?
+A: no
+
+Q: The director of the romantic comedy "Big Stone Gap" is based in what New York city?
+A: Greenwich Village, New York City
+
+Q: The arena where the Lewiston Maineiacs played their home games can seat how many people?
+A: 
+==================================================
+
+==================================================
+PROMPT #9:
+==================================================
+recent film was the 2006 vampire horror film The Thirst, where she played a young girl with hemophilia named Sara. She graduated from East Ascension High School in Gonzales in 2005, and started her first year of college in 2006. She also plays guitar. She studied music at Tulane University in New Orleans, Louisiana, and also occasionally performs in regional theatre. From October 6 to October 16, 2011, Morton starred alongside McArdle in the new musical Greenwood at the New York Musical Theatre Festival. Morton then returned to Gonzales and taught drama at Ascension Christian High School. She now resides in Gonzales and works as a veterinarian technician in nearby Prairieville, Louisiana. Soundtracks Annie (1999 film soundtrack) Filmography References External links Alicia Morton on Instagram 1987 births Living people American child actresses Actresses from Louisiana American musical theatre actresses People from Gonzales, Louisiana 21st-century American womenAlicia Morton (born April 29, 1987) is an American former actress, singer, dancer, drama teacher, and veterinarian technician. She starred as Annie Bennett Warbucks in the 1999 Wonderful World of Disney production of Annie, which was based on the Broadway musical of the same name. Biography Morton was born and raised in Gonzales, Louisiana, and started singing when she was 18 months old. When she was 12, she played the titular role in Annie at her school in nearby Baton Rouge, Louisiana. She was 7 when she landed the part of young Cosette in Broadway's Les Misérables in 1996. Co-star Ricky Martin would rub makeup on her hands for good luck. In 1998, Morton beat 3,000 girls to win the part of Annie in the Disney television film Annie, which premiered on The Wonderful World of Disney on November 7, 1999, following a private screening at the New Amsterdam Theatre on November 1, 1999. In Annie, Morton co-starred with Lalaine, who would go on to star in the Disney Channel original series Lizzie McGuire from 2001 to 2004, and in the 2003 Disney Channel original movie You Wish!. In this role, Morton struggled with some of the dramatic moments. "The emotional scenes were hard for me," she says, until an acting coach suggested that she draw on her father's death. Her best friend, Alexis Kalehoff, is the daughter of Broadway's original Annie, Andrea McArdle. McArdle also had a cameo role in Morton's Annie, playing the Star-to-Be who has a solo in the song “N.Y.C.”. Her mostAnnie Morton (born October 8, 1970) is an American model born in Pennsylvania. She has appeared on the covers of British Vogue, ID, Marie Claire, and other magazines. She has been photographed by Helmut Newton; Peter Lindbergh; Annie Leibovitz; Richard Avedon; Juergen Teller; Paul Jasmin, Mary Ellen Mark, Stephen Shames, and Terry Richardson, and modeled for Donna Karan, Givenchy, Guerlain, Chanel, Harper's Bazaar, Sports Illustrated and Victoria's Secret. A long time vegetarian, an advocate for organic lifestyle choices and natural healthcare. She co-founded Tsi-La Organics, a "Green Luxury" company that creates and sells vegan, organic perfume and skin care products. Modeling She has appeared on many magazine covers and has been featured in several professional photobooks, Peter Lindbergh Selected Work, the cover of Juergen Teller By Juergen Teller, Helmut Newton Pages From The Glossies, and Donna Karan's book A Journey Of A Woman 20 Years. In 1997 she posed for the Pirelli Calendar with Richard Avedon. Morton has also appeared in several music videos. Death in Vegas' "Aisha" directed by Terry Richardson and The Rolling Stones' "Love So Strong" directed by David Fincher. Other business ventures Morton Co-founded Tsi-La Organics with her sister-in-law Natalie Morton in 2006. Tsi-La Organics, an award-winning "Green Luxury" company that creates and sells vegan, organic perfume and skin care products. Natalie and Annie created their "Green Luxury" Tsi~La product line as a supposedly healthy alternative to a predominantly synthetic market. Morton and her husband Michael Morpurgo of Dandelion own 3 School of Rock music schools (in Philadelphia, Pa, Doylestown, Pa and
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Were Scott Derrickson and Ed Wood of the same nationality?
+A: yes
+
+Q: What government position was held by the woman who portrayed Corliss Archer in the film Kiss and Tell?
+A: Chief of Protocol
+
+Q: What science fantasy young adult series, told in first person, has a set of companion books narrating the stories of enslaved worlds and alien species?
+A: Animorphs
+
+Q: Are the Laleli Mosque and Esma Sultan Mansion located in the same neighborhood?
+A: no
+
+Q: The director of the romantic comedy "Big Stone Gap" is based in what New York city?
+A: Greenwich Village, New York City
+
+Q: Who is older, Annie Morton or Terry Richardson?
+A: 
+==================================================
+
+==================================================
+PROMPT #10:
+==================================================
+the paintings came from Bolivia. The United States District Court for the Eastern District of Virginia concluded that, either way, Claimant still needed to provide documentation authorizing import of the paintings because both Peru and Bolivia are State Parties to the UNESCO Convention and both have import restriction agreements with the United States. Because Claimant could not provide such documentation, the paintings were subject to forfeiture. United States v. Twenty-Nine Pre-Columbian & Colonial Artifacts From Peru In 2010, United States Customs and Border Protection (CBP) seized Pre-Columbian and Colonial textiles, metals, lithics, and perishable remains from Peru at Miami International Airport. Claimant moved to dismiss for lack of subject matter jurisdiction, failure to state a claim, and denial of due process of law. First, the United States District Court for the Southern District of Florida found that the court did have subject matter jurisdiction. The court rejected Claimant's argument that the United States Court of International Trade had exclusive jurisdiction on the action. Second, the court found that the United States sufficiently stated a claim by showing the imported objects were protected material under the CCPIA. Third, the court rejected the claimant's argument that he was denied due process during CBP seizure proceedings because the CCPIA authorizes 90 days of artifact detention, and claimant did not protest the seizure despite notice and an opportunity to protest. Ancient Coin Collectors Guild cases The Ancient Coin Collectors Guild purchased twenty-two coins from a numismatic dealer in London in April 2009. The collection included twelve Chinese coins and sevenneed to be two law standards - the Pennsylvania law would apply to Piper and Scottish law would apply to Hartzell. A trial involving two sets of laws would be confusing to the jury. A lack of familiarity with Scottish law would also be confusing. Another powerful reason why Pennsylvania is a bad venue is that Scotland has a very strong interest in this litigation. There is "a local interest in having localized controversies decided at home." "Scotland has a very strong interest in this litigation. The accident occurred in its airspace. All of the decedents were Scottish. Apart from Piper and Hartzell, all potential plaintiffs and defendants are either Scottish or English. As we stated in Gilbert, there is 'a local interest in having localized controversies decided at home.' 330 U.S., at 509. Respondent argues that American citizens have an interest in ensuring that American manufacturers are deterred from producing defective products, and that additional deterrence might be obtained if Piper and Hartzell were tried in the United States, where they could be sued on the basis of both negligence and strict liability. However, the incremental deterrence that would be gained if this trial were held in an American court is likely to be insignificant. The American interest in this accident is simply not sufficient to justify the enormous commitment of judicial time and resources that would inevitably be required if the case were to be tried here." Disposition Reversed Rule If the remedy provided by the alternative forum is so clearly inadequate or unsatisfactoryforeign replacements. The United States Chamber of Commerce maintains a list of years when the random selection process (lottery) was implemented. The lottery process was challenged in Tenrec v. USCIS, a class action lawsuit in Oregon, but the case was decided against the plaintiffs. On June 28, 2021, the lottery process was challenged again in LIU et al. v. MAYORKAS et al., a lawsuit filed in United States District Court for the District of Columbia by 500+ FY 2022 H-1B applicants who didn't get selected in March 2021. Tax status of an individual with H-1B status The taxation of income for an individual with H-1B status depends on whether they are categorized as either nonresident aliens or resident aliens for tax purposes. A nonresident alien for tax purposes is only taxed on income that is effectively connected with a trade or business in the United States and United States-source income that is fixed, determinable, annual, or periodical. A resident alien for tax purposes is taxed on all income, including income from outside the United States. The classification is determined based on the substantial presence test. If the substantial presence test indicates that the individual is a resident, then income taxation is like any other U.S. person and may be filed using Form 1040 and the necessary schedules. Otherwise, the individual must file as a non-resident alien using Form 1040NR or Form 1040NR-EZ; the individual may claim a benefit from tax treaties that exist between the United States and the individual's country of citizenship. An individual in
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Were Scott Derrickson and Ed Wood of the same nationality?
+A: yes
+
+Q: What government position was held by the woman who portrayed Corliss Archer in the film Kiss and Tell?
+A: Chief of Protocol
+
+Q: What science fantasy young adult series, told in first person, has a set of companion books narrating the stories of enslaved worlds and alien species?
+A: Animorphs
+
+Q: Are the Laleli Mosque and Esma Sultan Mansion located in the same neighborhood?
+A: no
+
+Q: The director of the romantic comedy "Big Stone Gap" is based in what New York city?
+A: Greenwich Village, New York City
+
+Q: Are Local H and For Against both from the United States?
+A: 
+==================================================
+
+==================================================
+PROMPT #11:
+==================================================
+game, after the singing of "The Star-Spangled Banner", it is a tradition to sing the alma mater, "Crimson and the Blue". While singing the alma mater, it is tradition for the students in attendance to put an arm over the shoulders of their neighbors and slowly sway side to side, lifting their arms over their heads as the last line of the song is sung. The song is concluded by the Rock Chalk Chant. Post-game, the band will play a rendition of the Kansas state song "Home on the Range", which the crowd will stand up for similar to the national anthem. After singing The Star-Spangled Banner, while the opposing team is being introduced, members of the student section take out a copy of the student-run newspaper, The University Daily Kansan, and wave the paper in front of their faces, pretending to be reading it in an effort to show lack of interest in the opposing team. After the opponents are introduced, a short video is shown, detailing the history and the accomplishments of Kansas basketball. As the Jayhawks are introduced, the students rip up their newspapers and throw the confetti pieces of paper in the air as celebration. Whatever confetti remains is typically thrown in the air after the first basket made by the Jayhawks. If an opposing player fouls out of the game, the fans will "wave the wheat", waving their arms back and forth, as a sarcastic good-bye to the disqualified player, to the tune of "You Didn't Have Your Wheaties", from aI'm a Jayhawk is the fight song of the University of Kansas. History The original lyrics and music were written by George "Dumpy" Bowles in 1911. The lyrics to the song are generally not sung, instead, fans do a specific clap as the KU band plays the melody. The university bands do not play the intro, but only play the chorus of the song. The song was updated in 1958 to reflect the teams of the Big 8 Conference. The lyrics were rewritten again in 2010 to account for the departure of Nebraska and Colorado from the Big 12 Conference. The school has elected to not rewrite the lyrics following the departure of Texas A&M and Missouri from the league. Lyrics External links Birth of the Songs References University of Kansas American college songs College fight songs in the United States Big 12 Conference fight songs 1912 songs"Kansas Song (We’re From Kansas)" is a fight song of the University of Kansas. History The song was written by Carson J. Robison and Jack Riley. It was copyrighted 1922 by Abdallah Temple A.A.O.N.M.S. according to the sheet music. The university bands usually play only one verse of the song. References External links Lyrics and MP3s University of Kansas American college songs College fight songs in the United States Big 12 Conference fight songs 1922 songs Songs written by Carson Robison Songs about Kansas
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Were Scott Derrickson and Ed Wood of the same nationality?
+A: yes
+
+Q: What government position was held by the woman who portrayed Corliss Archer in the film Kiss and Tell?
+A: Chief of Protocol
+
+Q: What science fantasy young adult series, told in first person, has a set of companion books narrating the stories of enslaved worlds and alien species?
+A: Animorphs
+
+Q: Are the Laleli Mosque and Esma Sultan Mansion located in the same neighborhood?
+A: no
+
+Q: The director of the romantic comedy "Big Stone Gap" is based in what New York city?
+A: Greenwich Village, New York City
+
+Q: What is the name of the fight song of the university whose main campus is in Lawrence, Kansas and whose branch campuses are in the Kansas City metropolitan area?
+A: 
+==================================================
+
+==================================================
+PROMPT #12:
+==================================================
+David Diamond is an American screenwriter. He frequently collaborates with David Weissman. Early life Diamond was born on April 19, 1965, in Philadelphia, Pennsylvania. Diamond and Weissman met in high school, at Akiba Hebrew Academy (now Jack M. Barrack Hebrew Academy). They graduated in 1983. Career The duo sold their first spec script, The Whiz Kid, to 20th Century Fox in 1994. Their first produced project came in 2000, The Family Man starring Nicolas Cage and Tea Leoni. Cage's production company, Saturn Films, helped produce the film. The Family Man opened at #3 at the North American box office making $15.1 million in its opening weekend, behind What Women Want and Cast Away, which opened at the top spot. Diamond and Weissman next wrote the sci-fi comedy Evolution, helmed by Ivan Reitman. Evolution was based on a story by Don Jakoby, who turned it into a screenplay along with Diamond and Weissman. The film was originally written as a serious horror science fiction film, until director Reitman re-wrote much of the script. A short-lived animated series, Alienators: Evolution Continues, loosely based on the film, was broadcast months after the film was released. They partnered with Wedding Crashers producer Andrew Panay on Old Dogs and When In Rome. From 2018 to 2021, Diamond taught screenwriting at the Shalhevet High School in Los Angeles until moving back to Philadelphia. In 2019, the duo published a screenwriting manual, Bulletproof: Writing Scripts That Don't Get Shot Down. The book draws from their extensive film experience. References External links Living peopleRenard as Frank Mathieu Goldfeld as Victor Pablo-Noé Étienne as Lucas Critical response Evolution received generally positive reviews from critics. Review aggregator website Rotten Tomatoes reports an approval rating of 82% based on 62 reviews, with an average rating of 7.26/10. The site's critics' consensus reads: "Creepy, provocative, and aesthetically absorbing, Evolution marks a satisfying step forward for director/co-writer Lucile Hadzihalilovic." At Metacritic, which assigns a normalized rating out of 100 to reviews from mainstream critics, the film received an average score of 77 based on 19 reviews, indicating "generally favorable reviews". Accolades References External links 2015 films 2015 horror thriller films 2010s French-language films French horror thriller films Spanish horror thriller films Belgian horror thriller films Films directed by Lucile Hadžihalilović Films scored by Zacarías M. de la Riva 2010s French filmsDavid Weissman is an American screenwriter and director, most known for his comedies. He frequently collaborates with David Diamond. Diamond and Weissman met in high school, at Akiba Hebrew Academy (now Jack M. Barrack Hebrew Academy). They graduated in 1983. His film credits include Dream a Little Dream 2 (his only produced project not co-written with Diamond), The Family Man, Old Dogs, When in Rome, Evolution and the television film Minutemen. Career The duo sold their first spec script, The Whiz Kid, to 20th Century Fox in 1994. Their first produced project came in 2000, The Family Man starring Nicolas Cage and Tea Leoni. Cage's production company, Saturn Films, helped produce the film. The Family Man opened at #3 at the North American box office making $15.1 million in its opening weekend, behind What Women Want and Cast Away, which opened at the top spot. Diamond and Weissman next wrote the sci-fi comedy Evolution, helmed by Ivan Reitman. Evolution was based on a story by Don Jakoby, who turned it into a screenplay along with Diamond and Weissman. The film was originally written as a serious horror science fiction film, until director Reitman re-wrote much of the script. A short-lived animated series, Alienators: Evolution Continues, loosely based on the film, was broadcast months after the film was released. In 2005, they partnered with Wedding Crashers producer Andrew Panay on Old Dogs and When In Rome. In 2019, the duo published a screenwriting manual, Bulletproof: Writing Scripts That Don't Get Shot Down. The book draws from their
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Were Scott Derrickson and Ed Wood of the same nationality?
+A: yes
+
+Q: What government position was held by the woman who portrayed Corliss Archer in the film Kiss and Tell?
+A: Chief of Protocol
+
+Q: What science fantasy young adult series, told in first person, has a set of companion books narrating the stories of enslaved worlds and alien species?
+A: Animorphs
+
+Q: Are the Laleli Mosque and Esma Sultan Mansion located in the same neighborhood?
+A: no
+
+Q: The director of the romantic comedy "Big Stone Gap" is based in what New York city?
+A: Greenwich Village, New York City
+
+Q: What screenwriter with credits for "Evolution" co-wrote a film starring Nicolas Cage and Téa Leoni?
+A: 
+==================================================
+
+==================================================
+PROMPT #13:
+==================================================
+DJ-je egy klasszikus rockzenét játszó adónak a K-DST-nek a Grand Theft Auto: San Andreas nevű videójátékban. Rose feltűnt egy háttérszerepben a Guns N’ Roses akkori felállásával az 1988-as Clint Eastwood-filmben a Holtbiztos tippben. 2009-ben szóba került, hogy Rose lesz a producere annak a filmnek, ami az Almost Home című regényből készül. Diszkográfia Hollywood Rose Guns N’ Roses Vendégszereplések Magánélete Viszályok Mióta a Guns N’ Roses-t megalapították, 1985-ben, Rose sok zenésszel, ismert emberrel, és zenekarral viszálykodott már. Ilyen például Vince Neil, a Mötley Crüe frontembere, Kurt Cobain, Scott Weiland, korábbi barátja és zenekartársa Slash, James Hetfield, a divattervező Tommy Hilfiger, Jon Bon Jovi, az Eagles of Death Metal és Dexter Holland. Kapcsolatai A ’80-as évek közepén és a '90-es évek elején Rose viharos kapcsolatba került Erin Everlyvel, Don Everly (Everly Brothers) énekes lányával. Rose a „Sweet Child o’ Mine” című dalt Erinnek írta, a lány a videóban is megjelent. Axl és Erin 1990 április 28-án Las Vegas-ban házasodtak össze, bár kényszer alatt, miután Rose megjelent Erin házánál, azt állítva, hogy fegyver van az autójában és öngyilkos lesz, ha Erin nem megy hozzá feleségül. Rose azt akarta, hogy Dan McCafferty (a Nazareth énekese) énekeljen az esküvőn. Egy 2004-es interjúban McCafferty úgy nyilatkozott, hogy "Röhögőgörcsöt kaptam, amikor Axl arra kért, hogy énekeljem el nekik a „Love Hurts” című dalt az esküvőn, mivel tudtam, hogy a dal maga hosszabb, mint a házasságuk lesz. A Guns N’ Roses menedzsmenttől 18 ember hívott fel, hogy tegyem meg, de végül elfoglaltságra hivatkozva visszautasítottam. Valószínűleg tényleg elfoglalt lehettem.” A pár kevesebb, mint egy hónapigle nom de Tori Amos (1994) Guns N' Roses Use Your Illusion I (1991) Use Your Illusion II (1991) The Spaghetti Incident? (1993) Live Era: '87-'93 (1999) Greatest Hits (2004) Velvet Revolver Contraband (2004) Libertad (2007) (EP) (2007) The Cult Beyond Good And Evil (2001) Slash's Snakepit It's Five O'Clock Somewhere (1995) Neurotic Outsiders Neurotic Outsiders (1996) Ainsi que d'autres participations avec des groupes tels que Hawk, Jeff Paris, Wired Up, Y Kant Tori Read et Johnny Crash, et un album solo en 2003 intitulé "Hollywood Zen". Bibliographie Mick Wall, Guns N’ Roses: The Most Dangerous Band In The World, Sidgwick & Jackson, 1991. Danny Sugerman, Appetite for Destruction : the days of Guns N'Roses, 1991. Robert John, Guns N'Roses : the photographic history, 1993. Photos et vidéos inédites de Guns N'Roses sur le site du photographe Robert John Slash, Slash, autobiographie en collaboration avec Anthony Bozza, Harper Entertainment, 2007. Nick Kent, L'Envers du rock, éditions Austral, 1996. Christian Lamet, Guns N' Roses 1985-1995, éditions La Sirène, 1995. Notes et références Liens externes Guns N' Roses Naissance en novembre 1960 Batteur de hard rock Batteur américain de rock Naissance à Mission Viejo Membre de Guns N' Roses Membre de Velvet Revolveron oral promises, within the two-year statute of limitations that began with the game's October 2007 release. In March 2013, Rose was sued by an audience member who got hit by a microphone Rose threw into the crowd at the end of a show. Rose threatened legal action in 2014 over the release of Rapidfire (his pre-Hollywood Rose band) recordings, keeping them from digital storefronts for a period of time. In August 2016, former Guns N' Roses keyboardist Chris Pitman sued Rose for $125,000 in unpaid wages. The two parties settled in November 2016. Discography with Guns N' Roses Appetite for Destruction (1987) G N' R Lies (1988) Use Your Illusion I (1991) Use Your Illusion II (1991) "The Spaghetti Incident?" (1993) Chinese Democracy (2008) with Hollywood Rose The Roots of Guns N' Roses (2004) with Rapidfire Ready to Rumble EP (2014) Guest appearances The Decline of Western Civilization Part II: The Metal Years – Original Motion Picture Soundtrack by various artists (1988; "Under My Wheels" ft. Alice Cooper, Slash and Izzy Stradlin) The End of the Innocence by Don Henley (1989; "I Will Not Go Quietly") Fire and Gasoline by Steve Jones (1989; "I Did U No Wrong") Pawnshop Guitars by Gilby Clarke (1994; "Dead Flowers") Anxious Disease by The Outpatience (1996; "Anxious Disease" ft. Slash) Angel Down by Sebastian Bach (2007; "Back in the Saddle," "(Love Is) a Bitchslap," "Stuck Inside") New Looney Tunes (2018, "Rock the Rock") Filmography References External links 1962 births 20th-century American male singers 20th-century American singers 21st-century American male
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Were Scott Derrickson and Ed Wood of the same nationality?
+A: yes
+
+Q: What government position was held by the woman who portrayed Corliss Archer in the film Kiss and Tell?
+A: Chief of Protocol
+
+Q: What science fantasy young adult series, told in first person, has a set of companion books narrating the stories of enslaved worlds and alien species?
+A: Animorphs
+
+Q: Are the Laleli Mosque and Esma Sultan Mansion located in the same neighborhood?
+A: no
+
+Q: The director of the romantic comedy "Big Stone Gap" is based in what New York city?
+A: Greenwich Village, New York City
+
+Q: What year did Guns N Roses perform a promo for a movie starring Arnold Schwarzenegger as a former New York Police detective?
+A: 
+==================================================
+
+==================================================
+PROMPT #14:
+==================================================
+Random House Tower, de asemenea cunoscut ca Park Imperial Apartments, este un zgârie-nori de 52 de etaje din New York City, sediu al Random House și complex rezidențial de lux. Note Clădiri și structuri în New Yorkof up to . The residential floors are numbered 48-70 for marketing purposes. Among the first tenants were rapper P. Diddy and New York Yankees pitcher Randy Johnson. The building has two fluid tuned mass dampers, which are designed to damp building sway. Located on the 50th floor mechanical room, they have concrete walls, and measure wide by long and tall. One tank runs west-east and the other runs north-south. History When the lease of its longtime headquarters at 201 East 50th Street was ending, Random House decided to expand its headquarters and move to a newly built tower. They originally planned in 1998 to build a tower at 45th and Broadway across from its parent company Bertelsmann's headquarters at 1540 Broadway with a neon-lighted skyway across 45th Street connecting them, but after long negotiations the owner of the property withdrew from the plans. Looking for an alternative, the company spoke with various developers, who were already planning apartment buildings, and in 1999 decided for the project of Stephen M. Ross at Broadway and 55th Street, which was already under construction. Subsequently, they assigned architects to develop a new design. While the building has functioned as Random House's headquarters ever since, the 2013 newly formed Penguin Random House parent company initially worked from SoHo at 345 and 375 Hudson Street. In 2016, two years before the lease of the Random House Tower would end, the company extended the contract till 2033 and also moved to the tower of its subsidiary. There, Penguin Random House occupies andThe Random House Tower, also known as the Park Imperial Apartments, is a 52-story mixed-use tower in Manhattan, New York City. It is owned by real estate companies SL Green Realty and Ivanhoé Cambridge, with the office portion leased as the headquarters to book publisher Random House from the beginning and its parent company Penguin Random House (formed in 2013) since 2016, giving the building its name; a luxury apartment complex follows above the offices. The PRH entrance is on Broadway and goes up to 27 floors, while the apartment complex entrance is on West 56th Street. Rising to 684 ft (208 m), it is the 77th tallest building in New York City. Description The complex is on a trapezoidal block between 55th Street and 56th Street and follows the angle of Broadway. It has jagged setbacks to improve the views of Central Park. The Random House Tower is divided into two parts: the office floors on the 2nd through 26th stories, and the residential floors between the 27th and 51st stories. The first floor contains of retail space. Separate architects designed each of the sections. Skidmore, Owings & Merrill designed the office portion, which has a steel frame. Ismael Leyva Architects and Adam D. Tihany designed the residential portion, which has a concrete frame. The two sections do not entirely line up, and trusses were built on the 26th and 27th floor to transfer the load. The residential section of the tower has 130 apartments. The apartments have ten-foot ceilings, and there are five penthouses
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Were Scott Derrickson and Ed Wood of the same nationality?
+A: yes
+
+Q: What government position was held by the woman who portrayed Corliss Archer in the film Kiss and Tell?
+A: Chief of Protocol
+
+Q: What science fantasy young adult series, told in first person, has a set of companion books narrating the stories of enslaved worlds and alien species?
+A: Animorphs
+
+Q: Are the Laleli Mosque and Esma Sultan Mansion located in the same neighborhood?
+A: no
+
+Q: The director of the romantic comedy "Big Stone Gap" is based in what New York city?
+A: Greenwich Village, New York City
+
+Q: Are Random House Tower and 888 7th Avenue both used for real estate?
+A: 
+==================================================
+
+==================================================
+PROMPT #15:
+==================================================
+of Beckham through suspension and Sol Campbell and Steven Gerrard through injury, in a 2–1 win against Poland. In January 2006, Eriksson was recorded saying he would be willing to leave England to manage Aston Villa if England won the World Cup, after being duped into believing that a wealthy Arab would buy the club, and wanted him as manager. The wealthy "Arab" was in fact the "Fake Sheikh" Mazher Mahmood, an undercover News of the World reporter. On 23 January, The FA announced that Eriksson would leave his job after the 2006 World Cup, and it was thought that the News of the World allegations played a part in this decision. This was later denied by both parties, with Eriksson explaining that there was a prior arrangement to terminate his contract immediately after the World Cup. In March 2006, The Guardian reported that South Africa became the first potential employer publicly to register its interest in Eriksson. That job, however, went to Carlos Alberto Parreira. The tournament England finished top of Group B at the finals, beating Paraguay and Trinidad and Tobago, before a 2–2 draw with Sweden, although the English press considered their performances far from satisfactory. In the second round, a David Beckham trademark free kick was enough to see Eriksson's England past Ecuador in a lacklustre 1–0 encounter affected by very high temperatures. Eriksson, however, once again fell to nemesis Luiz Felipe Scolari's Portugal. They defeated England 3–1 on penalties with the score 0–0 after extra time, with Beckham lost to injuryprevious season as Ferguson's assistant at Manchester United. In late July 2003, the club went on a tour of the Far East as part of pre-season training, but also to cash in on Beckham's huge marketing appeal in Asia, where he enjoyed tremendous following. Comparing his reception upon arriving at Kunming Airport in south China to Beatlemania, Marca ran the headline, "Beckham-mania in China". After the opening game in Beijing the tour featured games in Hong Kong, Tokyo and Bangkok. Real's brand recognition in that part of the globe was already well established as the club made financially successful trips to Asia during previous off-seasons. The presence of a global marketing icon such as Beckham, however, made this particular tour a financial smash for los Merengues. Shortly after his transfer to Real, Beckham also ended his relationship with agent Tony Stephens of SFX Europe, who had guided him through his career until that point, including helping to engineer Beckham's move from Manchester to Madrid. Beckham signed on with Simon Fuller and his company 19 Entertainment, which already managed the career of Victoria. Beckham also appointed close friend Terry Byrne to be his personal manager. In late August 2003, Real Madrid won the Spanish Super Cup over two legs versus RCD Mallorca, with Beckham scoring the final goal in a 3–0 return leg win at home, thus setting the stage for the start of the league season. Playing in a star-laden team which included three former FIFA World Player of the Year recipients, Zinedine Zidane, Ronaldo andJim Blyth played scintillating attacking football that swept many teams aside, often by large margins of victory, although they failed to win any silverware or qualify for the UEFA Cup. He later played for Fulham, AFC Bournemouth and finally joined Cambridge United in July 1986. Managerial career Beck's career was ended by injury in 1989, and he was appointed assistant manager at Cambridge. Following the resignation of Chris Turner he was appointed manager January 1990, with Cambridge mid table in the Fourth Division. In his first season United won the Fourth Division playoffs, and reached the quarter-finals of the FA Cup – a rare achievement for a Fourth Division side. The following season (1990–91) they won the Third Division title after an excellent finish to the season saw them climb from 10th to top in less than three months, and they repeated the feat of reaching the FA Cup quarter-finals. During the Summer, Beck turned down the opportunity to manage Leicester City, saying he wanted to take the club all the way to the First Division. In fact he very nearly achieved this, as in 1991–92 they finished 5th in the Second Division, the highest position ever achieved by the club, but lost to Leicester City in the play-off semi-finals, ending their hopes of a third successive promotion (something which has still yet to be achieved in the English Football League) and denying them the chance of becoming Premier League founder members. Beck's long ball playing style, despite its success, was heavily criticised in the media,
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Were Scott Derrickson and Ed Wood of the same nationality?
+A: yes
+
+Q: What government position was held by the woman who portrayed Corliss Archer in the film Kiss and Tell?
+A: Chief of Protocol
+
+Q: What science fantasy young adult series, told in first person, has a set of companion books narrating the stories of enslaved worlds and alien species?
+A: Animorphs
+
+Q: Are the Laleli Mosque and Esma Sultan Mansion located in the same neighborhood?
+A: no
+
+Q: The director of the romantic comedy "Big Stone Gap" is based in what New York city?
+A: Greenwich Village, New York City
+
+Q: The football manager who recruited David Beckham managed Manchester United during what timeframe?
+A: 
+==================================================
+
+==================================================
+PROMPT #16:
+==================================================
+Lake Refuge Luedtke State Wildlife Management Area Perch Creek State Wildlife Management Area State Wildlife Management Area Truman Wildlife Area Lakes Amber Lake Big Twin Lake Bright Lake Budd Lake Buffalo Lake (Cedar Township) Buffalo Lake (Rutland Township) Canright Lake Cedar Lake Clam Lake Clayton Lake Clear Lake Creek Lake Eagle Lake East Chain Lake Fish Lake (Cedar Township) Fish Lake (Lake Belt Township) Fox Lake Hall Lake High Lake Iowa Lake (part) Kiester Lake Lake Charlotte Lake George Lake Imogene Lake Seymour Little Hat Lake Little Twin Lake Martin Lake Mud Lake Murphy Lake North Lake North Silver Lake Okamanpeedan Lake (part) Perch Lake Pierce Lake Rose Lake Round Lake (Fox Lake Township) Round Lake (Waverly Township) Sager Lakes Sisseton Lake South Silver Lake Susan Lake Temperance Lake Tuttle Lake Watkins Lake Wilmert Lake Demographics 2000 census As of the 2000 census, there were 21,802 people, 9,067 households, and 6,047 families in the county. The population density was 30.6/sqmi (11.8/km2). There were 9,800 housing units at an average density of 13.8/sqmi (5.31/km2). The racial makeup of the county was 97.22% White, 0.25% Black or African American, 0.10% Native American, 0.42% Asian, 0.02% Pacific Islander, 1.26% from other races, and 0.73% from two or more races. 1.93% of the population were Hispanic or Latino of any race. 51.6% were of German, 11.9% Norwegian, 5.9% Swedish and 5.0% English ancestry. There were 9,067 households, out of which 29.80% had children under the age of 18 living with them, 56.60% were married couples living together, 7.20% had athe table for some people during a period when the local economy was in a downward slide. In 1969, the Texas Utilities Generating Company located a new power plant near Fairfield called Big Brown Power Plant. A dam was built to create Fairfield Lake to provide stored water for a cooling system for the plant. Fairfield Lake State Park was established around the lake and opened to the public in 1976. Big Brown was shut down in February 2018. Geography According to the U.S. Census Bureau, the county has a total area of , of which are land and (1.6%) are covered by water. Major highways Interstate 45 U.S. Highway 79 U.S. Highway 84 U.S. Highway 287 State Highway 14 State Highway 75 State Highway 164 State Highway 179 Adjacent counties Henderson County (north) Anderson County (northeast) Leon County (southeast) Limestone County (southwest) Navarro County (northwest) Demographics Note: the US Census treats Hispanic/Latino as an ethnic category. This table excludes Latinos from the racial categories and assigns them to a separate category. Hispanics/Latinos can be of any race. As of the census of 2010, 19,816 people, 6,588 households, and 4,664 families were residing in the county. The population density was 20 people/sq mi (8/km2). The 8,138 housing units averaged 9/sq mi (4/km2). The racial makeup of the county was 73.1% White, 16.1% African American, 0.7% Native American, 0.3% Asian, 8.1% from other races, and 1.6% from two or more races. About 13.6% of the population were Hispanic or Latino of any race. Of the 7,259 households,Brown State Fishing Lake (sometimes also known as Brown State Fishing Lake And Wildlife Area) is a protected area in Brown County, Kansas in the United States. The lake is 62 acres (0.25 km2) in area and up to 13 feet (4 m) deep. The area was formerly known as Brown County State Park, and is 8 miles (13 km) east of Hiawatha, Kansas. References Protected areas of Brown County, Kansas Kansas state fishing lakes Bodies of water of Brown County, Kansas Reservoirs in Kansas
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Were Scott Derrickson and Ed Wood of the same nationality?
+A: yes
+
+Q: What government position was held by the woman who portrayed Corliss Archer in the film Kiss and Tell?
+A: Chief of Protocol
+
+Q: What science fantasy young adult series, told in first person, has a set of companion books narrating the stories of enslaved worlds and alien species?
+A: Animorphs
+
+Q: Are the Laleli Mosque and Esma Sultan Mansion located in the same neighborhood?
+A: no
+
+Q: The director of the romantic comedy "Big Stone Gap" is based in what New York city?
+A: Greenwich Village, New York City
+
+Q: Brown State Fishing Lake is in a country that has a population of how many inhabitants ?
+A: 
+==================================================
+
+==================================================
+PROMPT #17:
+==================================================
+The Vermont Catamounts are the varsity intercollegiate athletic programs of the University of Vermont, based in Burlington, Vermont, United States. The school sponsors 18 athletic programs (8 men's, 10 women's), most of which compete in the NCAA Division I America East Conference (AEC), of which the school has been a member since 1979. The men's and women's ice hockey programs compete in Hockey East. The men's and women's alpine and nordic skiing teams compete in the Eastern Intercollegiate Ski Association (EISA). The school's athletic director is Jeff Schulman. The Catamounts have won six national championships, all in skiing. The program's mascot is Rally and colors are green and gold. Teams Vermont is one of only four NCAA Division I schools that do not sponsor volleyball or baseball. The other three are Boston University, Detroit Mercy, and Drexel. Basketball The men's basketball team won the 2003, 2004, 2005, 2010, 2012, 2017, 2019, 2022, and 2023, America East Conference Championships, qualifying for the NCAA Tournament in each of those seasons. In the 2005 NCAA tournament, the team, seeded #13 in the Austin Regional, upset #4 Syracuse 60–57 in overtime. Sports Illustrated named the upset one of the top 10 NCAA Tournament upsets of the 2000s. Ice hockey The men's ice hockey team, coached by Kevin Sneddon, appeared in the 2009 Men's Frozen Four. It has also qualified for the NCAA Tournament in 1996, 1997, 2010, and 2014. UVM's men's hockey team has produced 18 NHL players in its history. UVM alumni currently in the NHL include Viktor StålbergThe Vermont Catamounts football program was the intercollegiate American football team for the University of Vermont located in Burlington, Vermont. The team competed in the NCAA Division I and were members of the Yankee Conference. The school's first football team was fielded in 1886. The football program was discontinued at the conclusion of the 1974 season. Vermont fields a team at the club football level, in a conference that also uses the Yankee Conference name. History The first intercollegiate game in the state of Vermont happened on November 6, 1886, between Dartmouth and Vermont in Burlington. Dartmouth won 91 to 0. Vermont reached a level of success after coach "Dud" Drake in the 1907 and 1908 seasons. The 1907 team fought Dartmouth to a scoreless tie, and the 1908 team gave Cornell a scare. In 2007, The University of Vermont brought football back as a club sport under the Student Government Association by Doug Deluca. The University of Vermont Club football team is currently playing in the NCFA and is led by head coach Jeff Porter, defensive coordinator Jason Paul and offensive coordinator Jack Leclerc. As of recent, the team has sustained some success under head coach Jeff Porter. During the 2019 NCFA season, the Catamounts went undefeated in the North Atlantic conference until losing in their conference championship game to Sacred Heart University. The Catamounts are currently ranked fifth in the NCFA. All-time Yankee Conference records This table reflects the results of Yankee Conference matchups when both Vermont and its opponent were members of theThe Vermont Catamounts men's soccer team represents the University of Vermont in all NCAA Division I men's college soccer competitions. The team competes in the America East Conference. The team currently practices at Virtue Field adjacent to Centennial Field on campus. Roster Year by year results Note: Vermont competed in the Yankee Conference from 1964 to 1979 and has competed in the America East Conference since 1988. {| class="wikitable" style="text-align:center;" |- align="center" See also Vermont Catamounts Vermont Catamounts women's soccer College soccer References External links Soccer clubs in Vermont 1964 establishments in Vermont Association football clubs established in 1964
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Were Scott Derrickson and Ed Wood of the same nationality?
+A: yes
+
+Q: What government position was held by the woman who portrayed Corliss Archer in the film Kiss and Tell?
+A: Chief of Protocol
+
+Q: What science fantasy young adult series, told in first person, has a set of companion books narrating the stories of enslaved worlds and alien species?
+A: Animorphs
+
+Q: Are the Laleli Mosque and Esma Sultan Mansion located in the same neighborhood?
+A: no
+
+Q: The director of the romantic comedy "Big Stone Gap" is based in what New York city?
+A: Greenwich Village, New York City
+
+Q: The Vermont Catamounts men's soccer team currently competes in a conference that was formerly known as what from 1988 to 1996?
+A: 
+==================================================
+
+==================================================
+PROMPT #18:
+==================================================
+Opéra orchestra. In 1828, aged 17, Ambroise joined him in Paris, where he was admitted as a student by the Conservatoire de Paris. He studied the piano with Pierre Zimmerman and harmony and counterpoint with Victor Dourlen. He won premiers prix in these subjects in 1829 and 1830. He went on to study the piano with Friedrich Kalkbrenner, and composition with Jean-François Lesueur and Auguste Barbereau. In 1832, at his second attempt, Thomas won France's premier music prize, the Grand Prix de Rome, with his cantata Hermann et Ketty. The prize brought him three years' study at the Villa Medici, the French Academy in Rome. During his time there he became friendly with the painter Ingres, the head of the academy, with whom he shared an admiration for both Mozart and Beethoven; he also met Berlioz, who encouraged him and wrote about him favourably. During his Italian sojourn he wrote chamber music – a piano trio, a string quintet and a string quartet – and a set of six songs, Souvenirs d'Italie. After leaving Rome, Thomas stayed briefly in Germany, before returning to Paris in 1835, when he began writing for the stage. Composing career The first opera Thomas composed was La double échelle (The Double Ladder, 1837), a one-act comedy, praised by Berlioz for its "extreme vivacity and wit". It was produced at the Opéra-Comique, received 247 performances, and in the next few years was given in Brussels, New Orleans, Berlin, Vienna and London. His first full-length opera, Le perruquier de la Régence (The RegencyCharles Louis Ambroise Thomas (; 5 August 1811 – 12 February 1896) was a French composer and teacher, best known for his operas Mignon (1866) and Hamlet (1868). Born into a musical family, Thomas was a student at the Conservatoire de Paris, winning France's top music prize, the Prix de Rome. He pursued a career as a composer of operas, completing his first opera, La double échelle, in 1837. He wrote twenty further operas over the next decades, mostly comic, but he also treated more serious subjects, finding considerable success with audiences in France and abroad. Thomas was appointed as a professor at the Conservatoire in 1856, and in 1871 he succeeded Daniel Auber as director. Between then and his death at his home in Paris twenty-five years later, he modernised the Conservatoire's organisation while imposing a rigidly conservative curriculum, hostile to modern music, and attempting to prevent composers such as César Franck and Gabriel Fauré from influencing the students of the Conservatoire. Thomas' operas were generally neglected during most of the 20th century, but in more recent decades they have experienced something of a revival both in Europe and the US. Life and career Early years Thomas was born in Metz, the youngest of four children of Martin Thomas (1770–1823) and his wife, Jeanne, née Willaume (1780–1866), both music teachers. By the age of ten he was already an experienced pianist and violinist. When he was twelve his father died, and Ambroise's elder brother Charles moved to Paris, where he played the cello in theThis is a list of the complete operas of the French opera composer Ambroise Thomas (1811–1896). All premieres took place in Paris unless otherwise noted. References Forbes, Elizabeth (1992), 'Thomas, Ambroise' in The New Grove Dictionary of Opera, ed. Stanley Sadie (London) Wild, Nicole; Charlton, David (2005). Théâtre de l'Opéra-Comique Paris: répertoire 1762-1972. Sprimont, Belgium: Editions Mardaga. . Lists of operas by composer Lists of compositions by composer
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Were Scott Derrickson and Ed Wood of the same nationality?
+A: yes
+
+Q: What government position was held by the woman who portrayed Corliss Archer in the film Kiss and Tell?
+A: Chief of Protocol
+
+Q: What science fantasy young adult series, told in first person, has a set of companion books narrating the stories of enslaved worlds and alien species?
+A: Animorphs
+
+Q: Are the Laleli Mosque and Esma Sultan Mansion located in the same neighborhood?
+A: no
+
+Q: The director of the romantic comedy "Big Stone Gap" is based in what New York city?
+A: Greenwich Village, New York City
+
+Q: Are Giuseppe Verdi and Ambroise Thomas both Opera composers ?
+A: 
+==================================================
+
+==================================================
+PROMPT #19:
+==================================================
+in the Carter Administration. He was appointed Deputy Assistant Secretary for Health in the U.S. Department of Health, Education and Welfare (now Department of Health and Human Services), and served through four Administrations—Presidents Jimmy Carter, Ronald Reagan, George H. W. Bush, and William J. Clinton—while also serving as Assistant Surgeon General and Director of the U.S. Office of Disease Prevention. During this period, McGinnis chaired the Secretary's Task Force on Smoking and Health, which resulted in Secretary Califano's 1974 decision to elevate the federal leadership profile on tobacco control; led the development of Healthy People, the nation's prevention agenda; developed, with USDA, the first Dietary Guidelines for Americans; and created the U.S. Preventive Services Task Force in 1984 to develop an ongoing systematic and comprehensive assessment of the effectiveness of clinical preventive services, thereby helping to pioneer the broader advance of evidence-based medicine. Healthy People is perhaps the most widely known of McGinnis’ policy contributions. Begun when he was serving as Chair of the Secretary's Task Force on Disease Prevention and Health Promotion for HEW Secretary Joseph Califano and Surgeon General Julius Richmond in 1978, it started as an effort to develop a Surgeon General's report to serve as a U.S. answer to A New Perspective on the Health of Canadians, a groundbreaking report released in 1974 by Canadian Minister of Health Marc Lalonde. McGinnis felt the U.S. report could go beyond simply marshaling the scientific evidence underpinning the power of prevention, to establish quantifiable, evidence-based goals for improving the health of Americans through preventive interventions.to the rank of colonel and was awarded the Bronze Star Medal and the Legion of Merit. At the end of the war, Egeberg treated the Prime Minister of Japan, Hideki Tojo who had shot himself. Tojo was later hanged. Academic positions After the war, Egeberg was Chief of Medicine at Wadsworth General Hospital, head of County-University of Southern California Medical Center (1959) and Dean, USC School of Medicine (1964). After working for Nixon, Egeberg took a professorial chair at Georgetown University. In his later career, he advocated for the advanced training of doctors in geriatric medicine. Nixon administration From 1969 to 1971, Egeberg was the Assistant Secretary for health and scientific affairs in the Department of Health, Education and Welfare. In this role, he was an advocate for expanding public health care and argued for the financing of medical programs (through the Office of Health Care Financing). During this time, Egeberg visited the USSR and developed a professional relationship with Boris Vasilevich Petrovsky, the Soviet minister of health. In the spring of 1970, while requesting $100 million to train more doctors and nurses, Egeberg alleged the Health, Education and Welfare Secretary, Robert H. Finch, and the department's Under Secretary, John C. Veneman, were not considering his advice on health matters and that Nixon's advisers were ignoring him. He said, "The White House just doesn't appreciate, doesn't know, what is going on in the health field." also, "I just can't get through to Ehrlichman," (referring to John D. Ehrlichman, the president's chief aide for domestic matters.)Roger Olaf Egeberg, M.D. (13 November 1902 – 13 September 1997 Washington, D.C.) was an American medical educator, administrator and advocate of public health. He was General Douglas MacArthur's personal physician during World War II in the Pacific theater. His other roles included Assistant Secretary for Health and Scientific Affairs in the Department of Health, Education, and Welfare (now the United States Department of Health and Human Services) during the Nixon administration and Dean of the University of Southern California's medical school. Early life and family Egeberg was born in Chicago, Illinois to Hans Olaf Egeberg (1875-1932) and Ulrikka Rostrup Egeberg (1875-1932) (née Nielsen), a Norwegian immigrant family. Egeberg married a Canadian sculptor, Margaret Chahoon (25 August 1904 - 31 March 2011), in 1929. Together, they had a son and three daughters. Education and career Egeberg received a bachelor's degree in 1925 from Cornell University where he was a member of the Quill and Dagger society. He earned his medical degree in 1929 at Northwestern University school of medicine. He became an internist and joined the faculty of Western Reserve University School of Medicine. Military service During World War II, Egeberg was a member of the 4th General Hospital which was sent to Australia. From there he transferred to Milne Bay, Papua New Guinea (PNG) where he organised field stations and was a malaria control officer. Through determined attempts to control illnesses such as malaria and sexually transmitted diseases in PNG, Egeberg was noticed by MacArthur who made him his personal physician and aiLine truncated
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Were Scott Derrickson and Ed Wood of the same nationality?
+A: yes
+
+Q: What government position was held by the woman who portrayed Corliss Archer in the film Kiss and Tell?
+A: Chief of Protocol
+
+Q: What science fantasy young adult series, told in first person, has a set of companion books narrating the stories of enslaved worlds and alien species?
+A: Animorphs
+
+Q: Are the Laleli Mosque and Esma Sultan Mansion located in the same neighborhood?
+A: no
+
+Q: The director of the romantic comedy "Big Stone Gap" is based in what New York city?
+A: Greenwich Village, New York City
+
+Q: Roger O. Egeberg was Assistant Secretary for Health and Scientific Affairs during the administration of a president that served during what years?
+A: 
+==================================================
+
+==================================================
+PROMPT #20:
+==================================================
+Robert Cesar Childers (sprich tschillders; * 1838 in Nizza; † 28. Juli 1876 in London) gilt als einer der herausragenden Kenner des Pali und des Buddhismus in seiner Zeit. Er war der Vater des Schriftstellers und Anhängers der irischen Unabhängigkeitsbewegung Robert Erskine Childers und der Großvater des irischen Präsidenten Erskine Hamilton Childers. Childers studierte in Oxford und ging 1860 nach Ceylon. Während eines mehrjährigen Aufenthalts in Ceylon als englischer Zivilbeamter machte er sich mit Hilfe eines Singhalesen mit dem Pâli bekannt. 1864 kehrte er aus gesundheitlichen Gründen nach England zurück, wo er 1872 Unterbibliothekar an der Bibliothek des India Office (Indian Office Library) und später Professor des Pâli und der buddhistischen Literatur am University College in London wurde. Im Journal der Asiatic Society gab er mehrere Pâlitexte mit Übersetzungen sowie Untersuchungen über das Singhalesische, die einheimische Sprache von Ceylon, heraus, die er als Tochter des Sanskrits nachzuweisen versuchte. Sein Hauptwerk ist das preisgekrönte »Dictionary of the Pali language« (London 1875), das eine neue Epoche im Studium der Pâliliteratur und des Buddhismus begründete. An der Herausgabe einer im Manuskript fertigen Pâligrammatik wurde er durch den Tod gehindert. Childers starb am 28. Juli 1876 in London. Quellen Brockhaus Enzyklopädie. 14. A., Bd. 4. 1908. Indologe Buddhologe Brite Engländer Geboren 1838 Gestorben 1876 MannRobert Caesar Childers (12 February 1838 – 25 July 1876) was a British Orientalist scholar, compiler of the first Pāli-English dictionary. Childers was the husband of Anna Barton of Ireland. He was the father of Irish nationalist Erskine Childers and grandfather to the fourth President of Ireland, Erskine Hamilton Childers. Early life and marriage He was born in Nice, Piedmont-Sardinia, the son of Canon Charles Childers, chaplain to the English colony in Nice. His grandmother was Selina Eardley. In 1857, at the age of nineteen, he began the study of Hebrew at Wadham College, Oxford, graduating in 1861. After some travels in Ireland, he soon married Anna Barton of Glendalough House. The Bartons of Wicklow, Ireland, were a very wealthy and respected family in Irish farming and politics. Pali dictionary He then moved to Ceylon for an official position in the civil service. During this period he studied Sinhalese culture, particularly the Pali language. In 1869 he published the first Pali text in Britain, and began to work on a Pali dictionary, which was published 1872–75. In 1872 he was appointed sub-librarian at the India Office, and in the following year he became the first professor of Pali and Buddhist literature at University College, London. In 1876 Childers' dictionary was awarded the Volney Prize by the Institute of France. He died at the age of 38 in London the same year, apparently having succumbed to tuberculosis which had been troubling him for many years, and which grew worse after Childers contracted a cold in the earlyRobert Erskine Childers DSC (25 June 1870 – 24 November 1922), usually known as Erskine Childers (), was an English-born Irish writer, politician, and militant. His works included the influential novel The Riddle of the Sands. Starting as an ardent Unionist, he later became a supporter of Irish Republicanism and smuggled guns into Ireland in his sailing yacht Asgard. He was executed by the authorities of the nascent Irish Free State during the Irish Civil War. He was the son of British Orientalist scholar Robert Caesar Childers; the cousin of Hugh Childers and Robert Barton; and the father of the fourth President of Ireland, Erskine Hamilton Childers. Early life Childers was born in Mayfair, London, in 1870. He was the second son of Robert Caesar Childers, a translator and oriental scholar from an ecclesiastical family, and Anna Mary Henrietta Barton, from an Anglo-Irish landowning family of Glendalough House, Annamoe, County Wicklow, with interests in France such as the winery that bears their name. When Erskine was six, his father died from tuberculosis and, although seemingly healthy, Anna was confined to an isolation hospital, where she died six years later. The five children were sent to the Bartons, the family of their mother's uncle, at Glendalough, County Wicklow. They were treated kindly there and Erskine grew up knowing and loving Ireland, albeit at that stage from the comfortable viewpoint of the "Protestant Ascendancy". According to his biographer Michael Hopkinson, it was the personal tension caused by his innate belief in English superiority, in conflict with this new
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Were Scott Derrickson and Ed Wood of the same nationality?
+A: yes
+
+Q: What government position was held by the woman who portrayed Corliss Archer in the film Kiss and Tell?
+A: Chief of Protocol
+
+Q: What science fantasy young adult series, told in first person, has a set of companion books narrating the stories of enslaved worlds and alien species?
+A: Animorphs
+
+Q: Are the Laleli Mosque and Esma Sultan Mansion located in the same neighborhood?
+A: no
+
+Q: The director of the romantic comedy "Big Stone Gap" is based in what New York city?
+A: Greenwich Village, New York City
+
+Q: Which writer was from England, Henry Roth or Robert Erskine Childers?
+A: 
+==================================================
+

benchmarks/data/prompts_g5/prompt_dump_nq_hnsw.txt

@@ -0,0 +1,484 @@
+=== Prompt Dump for NQ + HNSW ===
+Total prompts: 50
+Showing first 20 prompts:
+
+==================================================
+PROMPT #1:
+==================================================
+First Time<span style="font-size=50%>...</span>" exploded in popularity following the November 1971 release of Play Misty For Me. This persuaded Atlantic Records to issue the track as a single - trimmed by a minute - in February 1972: the track became a smash hit single in the United States, reaching No. 1 for six weeks on both the Billboard Hot 100 and easy listening charts in the spring of 1972, with a No. 4 R&B chart peak. Reaching No. 14 on the UK Singles Chart, Flack's "The First Time<span style="font-size=50%>...</span>" was No. 1 for three weeks on the singles chart in Canada's RPM magazine. "The First Time Ever I Saw Your Face" was played as the wake-up music on flight day 9 to the astronauts aboard Apollo 17 on their last day in Lunar orbit (Friday, December 15, 1972) before returning to Earth, thus ending the last human explorations of the Moon. The use of the song was most likely a reference to the "face" of the Moon below the spacecraft. Chart history Weekly charts Roberta Flack Year-end charts All-time charts Celine Dion Leona Lewis Matt Cardle See also List of Billboard Hot 100 number-one singles of 1972 List of number-one adult contemporary singles of 1972 (U.S.) List of number-one singles in Australia during the 1970s List of number-one singles of 1972 (Canada) References External links Superseventies.com - with quotes from Roberta Flack and information on the song's background 1957 songs 1972 singles 2000 singles Songs written by Ewan MacColl Roberta Flack songs Celine Dion songs The Kingstonthe Apollo 11 mission to the Moon. The song had been recorded three weeks earlier. The Sprague Electric Company delivered a diameter silicon disc to NASA, containing 73 "messages of good will from the leaders of the world's nations to be flown and left on the Moon," that had been inscribed microscopically. The medallion was placed on a package adhered to one of the sleeves of Buzz Aldrin's spacesuit, and was almost forgotten until Aldrin was climbing back into the lunar module before for departure on July 21. Neil Armstrong reminded Aldrin of "that package" and Aldrin tossed it on to the lunar surface. Born: David Tao (Tao Xuzhong), Taiwanese R&B singer and songwriter; in Hong Kong July 12, 1969 (Saturday) Nearly four months after its cancellation by the American NBC TV network, Star Trek was introduced to British television viewers. Its run on BBC-1 began with the pilot episode "Where No Man Has Gone Before" (unlike the original US run, which had opted for "The Man Trap"). Filling in as a summer replacement for Doctor Who between the Doctor's sixth and seventh seasons, the programme was shown in the Doctor Who 5:15 p.m. time slot between afternoon sports and the 10-minute BBC news and weather update. July 13, 1969 (Sunday) The Soviet Union launched the unmanned lunar spacecraft Luna 15, three days before the scheduled liftoff of the American Apollo 11 manned mission to the Moon, with the objective of performing a sample-return mission and bringing back the first lunar soil (a "Moon rock") aheadto 19 December 1972. The 24 people who have flown to the Moon are the only people who have traveled beyond low Earth orbit. Ten of them are alive. Jim Lovell, John Young, and Eugene Cernan are the only three people to have flown to the Moon twice. Young and Cernan each set foot on it during their respective second lunar missions, while Lovell is the only person to have flown to the Moon twice without landing. During Cernan's first lunar mission on Apollo 10, he tied the present record set by Bill Anders on Apollo 8 as the youngest person to fly to the Moon. Each was 35 years and 65 days old on his launch date and 35 years and 68 days old when he entered lunar orbit. The oldest person to fly to the Moon was Alan Shepard, who walked on its surface during the Apollo 14 mission. Shepard was 47 years and 74 days old on his launch date and 47 years and 78 days old when he entered lunar orbit. Jim Lovell and Fred Haise were scheduled to walk on the Moon during the Apollo 13 mission, but the lunar landing was aborted following a major malfunction en route to the Moon. Haise was again scheduled to walk on the Moon as commander of Apollo 19, but Apollo 18 and Apollo 19 were canceled on September 2, 1970. Because of Apollo 13's free-return trajectory, Lovell, Swigert and Haise flew higher above the Moon's 180° meridian (opposite Earth) than anyone else has
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: where did they film hot tub time machine?
+A: Fernie Alpine Resort
+
+Q: who has the right of way in international waters?
+A: Neither vessel
+
+Q: who does annie work for attack on titan?
+A: Marley
+
+Q: when was the immigration reform and control act passed?
+A: November 6, 1986
+
+Q: when was puerto rico added to the usa?
+A: 1950
+
+Q: when was the last time anyone was on the moon?
+A:
+==================================================
+
+==================================================
+PROMPT #2:
+==================================================
+My Brother", the first use of the phrase exactly as it is rendered in the song title. In the 1940s, the words, adapted as "He ain't heavy, Father, he's my brother", were taken as a slogan for Boys Town children's home by founder Father Edward Flanagan. According to the Boys Town website, the phrase as used by Boys Town was said to Fr. Flanagan in 1918 by one of the residents while carrying another up a set of stairs. The boy being carried is said to have had polio and worn leg braces. The Hollies version The Hollies' recorded the song in June 1969 at the EMI Studios (now Abbey Road Studios), with Allan Clarke on lead vocals. Elton John, who was working as a session musician at the time, played the piano on the song, as well their next single, "I Can't Tell the Bottom from the Top". The song was released on 26 September 1969 and reached No. 3 in the UK, and No. 7 in the US. The song was re-released in August 1988 in the UK following its use in a television advertisement for Miller Lite beer. It reached the No. 1 spot in the UK chart for two weeks in September 1988. Weekly charts Year-end charts Neil Diamond version The Neil Diamond version entered at No. 68 on the Hot 100 on November 7, 1970 (UNI Records, 55264, length 4:09). The flip side was "Free Life". The song appears on Diamond's album Tap Root Manuscript, which was released in November 1970."He Ain't Heavy, He's My Brother" is a 1969 song by the band the Hollies. He Ain't Heavy, He's My Brother may also refer to: He Ain't Heavy, He's My Brother (album), a 1969 Hollies album, also known as Hollies Sing Hollies and including the song of the same name He Ain't Heavy, He's My Brother (film), a Hong Kong film, also known as He Ain't Heavy, He's My Father or 新難兄難弟 "He Ain't Heavy, He's My Brother", an episode of Rizzoli & Isles "He Ain't Heavy, He's My Brother", an episode of On Our Own (1994 TV series) The caption of a logo adopted by Boys Town in 1943 See also He Ain't Heavy (disambiguation) He's No Heavy, He's My Brother (disambiguation) He's Heavy, He's My Brother (disambiguation) He Ain't Heavy, He's My Father (disambiguation) "He Ain't a Hottie, He's My Brother", an episode of Hannah Montana "He Ain't Heavy, He's My Bother", an episode of Heathcliff (1984 TV series) "He Ain't Heavy, He's Dee's Brother", an episode of Moesha "He Ain't Heavy, He's Willie's Brother", an episode of ALF "Lee Ain't Heavy, He's My Brother", an episode of Three's Company"He Ain't Heavy, He's My Brother" is a ballad written by Bobby Scott and Bob Russell. Originally recorded by Kelly Gordon in 1969, the song became a worldwide hit for the Hollies later that year and also a hit for Neil Diamond in 1970. It has been recorded by many artists in subsequent years. The Hollies' version was re-released in 1988 and again was a major hit in the UK. Scott and Russell were introduced to each other by Johnny Mercer, at a California nightclub. Although Russell was dying of lymphoma and the pair met only three times, they managed to collaborate on the song. Title James Wells, Moderator of the United Free Church of Scotland, tells the story of a little girl carrying a big baby boy in his 1884 book The Parables of Jesus. Seeing her struggling, someone asked if she wasn't tired. With surprise she replied: "No, he's not heavy; he's my brother." In a 1918 publication by Ralph Waldo Trine titled The Higher Powers of Mind and Spirit, Trine relates the following anecdote: "Do you know that incident in connection with the little Scottish girl? She was trudging along, carrying as best she could a boy younger, but it seemed almost as big as she herself, when one remarked to her how heavy he must be for her to carry, when instantly came the reply: 'He's na heavy. He's mi brither.'" The first editor of Kiwanis magazine, Roe Fulkerson, published a column in September 1924 carrying the title "He Ain't Heavy, He's
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: where did they film hot tub time machine?
+A: Fernie Alpine Resort
+
+Q: who has the right of way in international waters?
+A: Neither vessel
+
+Q: who does annie work for attack on titan?
+A: Marley
+
+Q: when was the immigration reform and control act passed?
+A: November 6, 1986
+
+Q: when was puerto rico added to the usa?
+A: 1950
+
+Q: who wrote he ain't heavy he's my brother lyrics?
+A:
+==================================================
+
+==================================================
+PROMPT #3:
+==================================================
+the Bastards" received immense critical acclaim, with many calling it one of the best television episodes of all time. Critics cited the size and scope of the battle in the North and Daenerys's scene with her dragons at the beginning of the episode. It has a 98% rating on the review aggregator website Rotten Tomatoes from 65 reviews, with an average score of 9.08 out of 10. According to the site consensus, "'Battle of the Bastards' delivered one of the greatest battle sequences in the show's history, and some savagely satisfying vengeance as well." IGN's Matt Fowler gave the episode a 10 out of 10, writing in his review, "At this point, it seems like the pattern when it comes to the ninth episode of a given season of Game of Thrones is tragic death, amazing battle, tragic death, amazing battle, and so on. And this being Season 6, it was time to land on a phenomenal clash of swords and shields — and 'Battle of the Bastards' certainly delivered." He continued, "Game of Thrones did not disappoint when it came to this season's great northern battle, as Jon and Sansa's differences were spectacularly highlighted in a savagely strong war chapter that saw House Stark overcome huge odds to reclaim their home. Plus, Daenerys got to soar, as her dragons quickly stopped a violent siege with fury and fire." Jeremy Egner of The New York Times also praised the episode: "As directed by Miguel Sapochnik, who also oversaw last season's terrific 'Hardhome' episode, the lengthy sequence"Battle of the Bastards" is the ninth and penultimate episode of the sixth season of HBO's fantasy television series Game of Thrones and its 59th episode overall. It was written by series co-creators David Benioff and D. B. Weiss, and directed by Miguel Sapochnik. Jon Snow and Ramsay Bolton face off in a battle for control of Winterfell. Jon's force, mostly composed of Wildlings, is nearly defeated by the Bolton army, but the latter is overcome when Sansa Stark and Petyr Baelish arrive with the Knights of the Vale. The battle culminates in Ramsay's defeat and capture, followed by his death at the hands of Sansa. Meanwhile, Daenerys Targaryen defeats the Masters in Meereen and begins making new alliances. "Battle of the Bastards" received immense critical acclaim, with several reviewers calling it a "masterpiece", and being praised as one of the series' best episodes as well as one of the greatest television episodes of all time. Critics described the battle in the North as "terrifying, gripping and exhilarating", Harington's performance received high praise, and Daenerys' reunion with her dragons at the beginning of the episode was deemed "thrilling". The eponymous battle took 25 days to film and required 500 extras, 600 crew members, and 70 horses. In the United States, the episode had a viewership of 7.66 million in its initial broadcast. It notably won the Emmy award for Outstanding Directing and Outstanding Writing; the episode was Kit Harington's choice to support his nomination for Outstanding Supporting Actor. Sapochnik also won the Directors Guild of Americasource material. Some of the season's storyline is derived from content not yet published in Martin's A Song of Ice and Fire series, although a significant amount of material from A Feast for Crows, A Dance with Dragons and the upcoming sixth novel The Winds of Winter, which Martin previously outlined to showrunners David Benioff and D.B. Weiss, was used. The season was largely met with positive reviews. The "Battle of the Bastards" episode received immense critical acclaim, with many calling it one of the best television episodes of all time. U.S. viewership rose compared to the previous season, and by approximately 13 percent over its course, from 7.9 million to 8.9 million by the finale. The season won 12 of the 23 Emmy Awards for which it was nominated. Three days before the premiere of the episode "The Red Woman", HBO ordered the seventh season. Due to necessary weather conditions required for filming, the production of the penultimate season of the show was delayed that year. Filming began only on August 31, 2016, at Titanic Studios in Belfast, and ended in February 2017. Unlike previous seasons, the seventh and eighth seasons largely consisted of original content not found in the source material. This season comprised only seven episodes. The showrunners stated that they were unable to produce 10 episodes in the show's usual 12 to 14 month time frame, as Weiss said "It's crossing out of a television schedule into more of a mid-range movie schedule". The average runtime of an episode in this season
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: where did they film hot tub time machine?
+A: Fernie Alpine Resort
+
+Q: who has the right of way in international waters?
+A: Neither vessel
+
+Q: who does annie work for attack on titan?
+A: Marley
+
+Q: when was the immigration reform and control act passed?
+A: November 6, 1986
+
+Q: when was puerto rico added to the usa?
+A: 1950
+
+Q: how many seasons of the bastard executioner are there?
+A:
+==================================================
+
+==================================================
+PROMPT #4:
+==================================================
+the Eagles-Cowboys game. The Eagles then defeated the Minnesota Vikings 26–14 in the first round of the playoffs on January 4, 2009. They defeated their division rival and the defending Super Bowl champions the New York Giants 23–11 in the NFC Divisional Round to advance to their fifth NFC Championship game in nine years. For the third time in a decade, an all-Pennsylvania Super Bowl seemed possible, as the Pittsburgh Steelers won the AFC Championship against the Baltimore Ravens. However, the Eagles lost 32–25 to the Arizona Cardinals. 2009 In August, the team stirred up controversy by signing ex-Falcons quarterback Michael Vick, who had recently been released from prison for federal dogfighting charges. On the regular season opener against Carolina, McNabb sustained a broken rib and sat out the next two games, with Kevin Kolb taking his place as starter. Afterwards, Philadelphia endured a bad loss to New Orleans, but then beat Kansas City the following week. McNabb returned, and Vick also began playing (as he had been barred from the first two games). He got off to a slow start and was used sparingly for much of the season. After a 13–9 loss in Oakland, the Eagles defeated the Redskins 27–17 on Monday Night Football. During this game, running back Brian Westbrook suffered a severe concussion and was knocked unconscious. He returned to play in the Week 10 game against San Diego, but was felled by a second concussion. Afterwards, Philadelphia went on a five-game winning streak and clinching a playoff berth after defeating thedivision, after going 6–0 against their division in 2004. The Eagles became the first team in NFL history to go winless against their division after going undefeated against their division the previous season. After expectations of a return to the Super Bowl, it appeared the Eagles would have to retool in the 2006 off-season to make another run for the Vince Lombardi Trophy. 2006 In 2006, the Eagles started off hot, beating the Houston Texans in Houston 24–10. And they kept the streak going, jumping to a 24–7 lead against the Giants before losing 30–24 in overtime. They won their next two games easily, beating the 49ers in San Francisco and then returning home to limit Brett Favre's Packers to three field goals. The Birds' 3–1 record set the stage for Terrell Owens' return to Philadelphia. Owens' return to Philadelphia was being promoted by Fox as the "game of the year". The game had more at stake, because the winner would take the lead in the NFC East. Owens received boos and jokes about his accidental overdose from a sell-out crowd. Lito Sheppard's game-winning interception for a touchdown sealed the game for the Eagles, 38–24. McNabb's season was already considered one of the finest in Eagles history and his career. After the win against Owens' Cowboys, they faced the upstart New Orleans Saints, with critics claiming the winner would be the NFC favorite. The Eagles ended up losing 27–24 on a last-second field goal. The next week at Tampa, the Eagles would once again lose onto the Chicago Bears, the team that Ryan had helped lead to a Super Bowl XX victory as defensive coordinator. The game became known as the "Fog Bowl", due to the inclement weather during the game. The Eagles lost, 20–12. The following two years would see playoff appearances as well, but the team never made it past the first round. This failure was very frustrating to Eagles fans, as the team was widely regarded as among the most talented in the NFL. On offense, the Eagles were led by quarterback Cunningham, one of the most exciting players of his generation; tight end Keith Jackson; and running back Keith Byars. The defense is commonly acknowledged as among the greatest in league history, and as the best to never win a championship. The two 1989 matches with Dallas were known as the Bounty Bowls. Both were won easily by the Eagles (the Cowboys finished 1–15 that year), and were marked by Ryan insulting new Cowboys coach Jimmy Johnson by placing a bounty on their kicker, and by Eagles fans throwing snowballs at him at Veterans Stadium. On November 12, 1990, during a Monday Night Football game at the Vet, the Eagles defeated the Washington Redskins by a score of 28–14, as the defense scored three of the team's four touchdowns. This game, more lopsided than its score would indicate, was subsequently labeled the Body Bag Game in reference to the number of injuries Eagles players inflicted on the Redskins and the physically tough play of the Eagles in
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: where did they film hot tub time machine?
+A: Fernie Alpine Resort
+
+Q: who has the right of way in international waters?
+A: Neither vessel
+
+Q: who does annie work for attack on titan?
+A: Marley
+
+Q: when was the immigration reform and control act passed?
+A: November 6, 1986
+
+Q: when was puerto rico added to the usa?
+A: 1950
+
+Q: when did the eagles win last super bowl?
+A:
+==================================================
+
+==================================================
+PROMPT #5:
+==================================================
+style=""| AAC regular season |- !colspan=12 style=""|American Athletic Conference women's tournament |- !colspan=12 style=""|NCAA women's tournament Rankings WNBA draft Breanna Stewart, Moriah Jefferson and Morgan Tuck were the first, second and third pick of the 2016 WNBA draft. Players from the same college team have never gone 1,2,3 in any draft in any major sport in history. Awards and honors Breanna Stewart won the NCAA Bridgeport Regional Most Outstanding Player earned the Most Outstanding Player of the Final Four at the 2016 NCAA Division I women's basketball tournament for a record fourth consecutive time won the Associated Press Women's College Basketball Player of the Year for the third consecutive year won her second Wade Trophy, presented to the Division I player of the year was awarded the Naismith College Player of the Year for the third consecutive year won the USBWA Women's National Player of the Year for the third consecutive year won the 2016 American Athletic Conference Women's Basketball Player of the Year won the John R. Wooden Award was the recipient of the 2016 Senior CLASS Award was the recipient of the James E. Sullivan Award Geno Auriemma was awarded the Associated Press College Basketball Coach of the Year won the AAC coach of the year Moriah Jefferson won the Nancy Lieberman Award for the second consecutive year won the WBCA Defensive Player of the Year award won the Dawn Staley Award won the AAC Defender of the Year Morgan Tuck won the AAC Sportsmanship Award was named to the all AAC first teamawarded to the Canadian Interuniversity Sport winner in women's basketball Adelaide Lightning won the Australian Women's National Basketball League Championship The WNBA is founded, with eight initial teams. Sheryl Swoopes is the first player signed. The American Basketball League (ABL) formed, a professional basketball league for women in the United States. It lasted two full seasons, and suspended operations in the third. 1997 Tennessee (29-10) won the NCAA Division I women's basketball tournament North Dakota (28-4) won the NCAA Women's Division II Basketball Championship NYU (29-1) won the NCAA Women's Division III Basketball Championship Manitoba Bisons won the Bronze Baby, awarded to the Canadian Interuniversity Sport winner in women's basketball Sydney Flames won the Australian Women's National Basketball League Championship First WNBA draft, with Tina Thompson as the first player selected. The first game is held on 21 June 1997, between the New York Liberty and the Los Angeles Sparks. The Liberty won 67–57. The Houston Comets win the first WNBA Championship. Trent Tucker Rule adopted by WNBA. 1998 Tennessee (39-0) won the NCAA Division I women's basketball tournament North Dakota (31-1) won the NCAA Women's Division II Basketball Championship Washington (MO) (28-2) won the NCAA Women's Division III Basketball Championship Victoria Vikes won the Bronze Baby, awarded to the Canadian Interuniversity Sport winner in women's basketball Adelaide Lightning won the Australian Women's National Basketball League Championship FIBA World Championship for Women Gold—USA Silver—Russia Bronze—Australia The Houston Comets won the WNBA Championship. 1999 Purdue (34-1) won the NCAA Division I women's basketball tournament North Dakota (31-1)to win that award three times) Breanna Stewart was named the Most Outstanding Player of the Final Four (making her the first person to be most outstanding player of the Final Four four times) Lubbock Christian University (35–0) won the NCAA Division II Women's Basketball Championship Thomas More (33–0) won the NCAA Women's Division III Basketball Championship The Los Angeles Sparks won the WNBA Championship. 2017 The Connecticut Huskies women's basketball team obtained the longest winning streak in college basketball (both men's and women's), 111 straight wins, which started with a win against Creighton on December 23, 2014, and continued for 111 games until March 31, 2017, when they were beaten 66–64 on a last second shot in overtime by Mississippi State in the 2017 NCAA Final Four. This streak included an undefeated season in 2015–16. McGill University (25–9) won the Bronze Baby, awarded to the women's basketball champion of the newly renamed U Sports. Sydney Uni Flames won the Australian Women's National Basketball League Championship South Carolina (33–4) won the NCAA Division I women's basketball tournament Ashland (37–0) won the NCAA Women's Division II Basketball Championship Amherst (33–0) won the NCAA Women's Division III Basketball Championship The Minnesota Lynx won the WNBA Championship. 2018 Carleton won the Bronze Baby, awarded to the U Sports women's basketball champion. Townsville Fire won the Australian Women's National Basketball League Championship Notre Dame (35-3) won the NCAA Women's Division I Basketball Championship Central Missouri (30–3) won the NCAA Women's Division II Basketball Championship Amherst (33–0) Line truncated
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: where did they film hot tub time machine?
+A: Fernie Alpine Resort
+
+Q: who has the right of way in international waters?
+A: Neither vessel
+
+Q: who does annie work for attack on titan?
+A: Marley
+
+Q: when was the immigration reform and control act passed?
+A: November 6, 1986
+
+Q: when was puerto rico added to the usa?
+A: 1950
+
+Q: who won last year's ncaa women's basketball?
+A:
+==================================================
+
+==================================================
+PROMPT #6:
+==================================================
+Isle of WightAs an island, the Isle of Wight maintains a culture close to, but distinct from, that of the south of England. A high proportion of the population are now 'overners' rather than locally born, and so with a few notable exceptions it has more often formed the backdrop for cultural events of wider rather than island-specific significance. The Island has inspired many creative works. Local people often seek to defend their real or perceived culture, and local politics is often dictated by a desire to preserve the traditions and habits of the Island. The first creative flowering occurred during the reign of Queen Victoria, under whose patronage the island became a fashionable destination for the gentry. Literature and other media Alfred, Lord Tennyson was made Baron Tennyson, of Aldworth in the County of Sussex and of Freshwater on the Isle of Wight by Queen Victoria in 1884. The poet Algernon Charles Swinburne grew up at Bonchurch, and said in a letter that he had climbed Culver Cliff at 17. He is buried at Bonchurch. The author Maxwell Gray (Mary Gleed Tuttiett) was born in Newport, and a number of her novels, including the best-known, The Silence of Dean Maitland, are set on the island. The isle has been the setting for several novels, including Julian Barnes's utopian novel England, England, and detective thrillers such as The Fallen by Robert Rennick. It also features in John Wyndham's novel The Day of the Triffids and Simon Clark's sequel The Night of the Triffids. The Iranian-born poet Mimi Khalvatithe first county councils were created in 1888. However, a "Home Rule" campaign led to a separate Isle of Wight County Council being established in 1890, and the island has remained independent of mainland local government ever since. The island was, however, part of the "Assize county" of Hampshire. Like inhabitants of many islands, Islanders are fiercely jealous of their real (or perceived) independence, and confusion over the Island's separate status is a perennial source of friction. It was planned to merge the county back into Hampshire as a district in the 1974 local government reform, but a last minute change led to it retaining its county council. However, since there was no provision made in the Local Government Act 1972 for unitary authorities, the Island had to retain a two-tier structure, with a county council and two boroughs, Medina and South Wight. The borough councils were merged with the county council on 1 April 1995 to form a single unitary authority, the Isle of Wight Council, as recommended by the Local Government Commission for England. The only significant present-day administrative link with Hampshire is the police service, which is joint between Hampshire and the Isle of Wight. House of Commons representation The Isle of Wight forms a single constituency of the House of Commons, with an electorate of 113,021 (as of 2019). This is by far the largest electorate in the country. The 2018 proposals from the Boundary Commission for England will split the constituency into two. The constituency was traditionally a battleground between the
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: where did they film hot tub time machine?
+A: Fernie Alpine Resort
+
+Q: who has the right of way in international waters?
+A: Neither vessel
+
+Q: who does annie work for attack on titan?
+A: Marley
+
+Q: when was the immigration reform and control act passed?
+A: November 6, 1986
+
+Q: when was puerto rico added to the usa?
+A: 1950
+
+Q: when did the isle of wight become an island?
+A:
+==================================================
+
+==================================================
+PROMPT #7:
+==================================================
+"Love Yourself" is a 2015 single by Justin Bieber. Love Yourself may also refer to: Music by BTS Love Yourself: Wonder, the music video theme for the group's song "Euphoria" Love Yourself: Her, the group's fifth extended play Love Yourself: Tear, the group's third Korean studio album and sixth overall Love Yourself: Answer, the group's second compilation album BTS World Tour: Love Yourself, the group's third worldwide concert tour Music by other artists Love Yourself (album), by Prudence Liew, 2000 "Love Yourself" (Billy Porter song), 2019 "Love Yourself" (Mary J. Blige song), 2017 "Love Yourself (Kimi ga Kirai na Kimi ga Suki)", by Kat-Tun, 2010 "Love Yourself", a 1996 song by Blueboy "Love Yourself", a 1997 song by Karen Mok from To Be "Love Yourself", a 2019 song by Sufjan Stevens See also Love Myself (disambiguation)third radio single. When talking to Ryan Seacrest, Bieber said "Love Yourself" is "definitely about someone in my past, someone who I don't want to put on blast," he described the song as "cool because so many people can resonate with that because how many women do we bring back that mom doesn't really necessarily like?." Composition and lyrics "Love Yourself" was written by Ed Sheeran, Benny Blanco and Justin Bieber, with production being done by Benny Blanco, who was also responsible for the song's instrumentation and programming. It features lyrical vocals by Bieber and background vocals by Sheeran. According to the sheet music published at Musicnotes.com by Sony/ATV Music Publishing, the song is composed in the key of E major with a moderate tempo of 100 beats per minute and a time signature of . Bieber's vocals range from the note of B3 to B4. It is an acoustic pop, folk and blue-eyed soul song, featuring just vocals and an electric guitar, and a brief flurry of trumpets, which was considered "the most subdued and least electronic of Bieber's new tracks." Lyrically, "Love Yourself" is a kiss-off to a narcissistic ex-lover who did the protagonist wrong, with Bieber singing in a snappy tone while criticizing "a girl for loving herself too much." In the pre-chorus, he sings with a husky tone in the lower registers: "My mama don't like you, and she likes everyone," "in a style that molds well to Sheeran's," according to Rolling Stone Brittany Spanos. "And I [never] like to admit that"Love Yourself" is a song recorded by Canadian singer Justin Bieber for his fourth studio album Purpose (2015). The song was released as a promotional single on November 9, 2015, and as the album's third official single on December 7, 2015. It was written by Ed Sheeran, Benjamin Levin and Bieber, and produced by Levin. A pop song, "Love Yourself" features an electric guitar and a brief flurry of trumpets as its main instrumentation. During the song, Bieber uses a husky tone in the lower registers. Lyrically, the song is a kiss-off to a narcissistic ex-lover who did the protagonist wrong. On the US Billboard Hot 100 and the UK Singles Chart, the song became Bieber's third consecutive number-one, wherein the United States it spent 24 non-consecutive weeks in the top ten (later named the best-performing single of 2016) and was also Bieber's first number one on the Adult Contemporary chart, while in the United Kingdom it spent six weeks at the top. "Love Yourself" topped the charts in fifteen countries, including Australia, Brazil, Canada, New Zealand, and Sweden. "Love Yourself" was nominated for two Grammy Awards: Song of the Year and Best Pop Solo Performance. It was the seventh-best-selling song of 2016 in the US. The music video for the song was released along with the Purpose: The Movement project on November 14, 2015. It features dancers Keone & Mari Madrid doing an interpretive dance in a house. Bieber promoted the song with performances on TV shows, as well as awards shows throughout 2015–16, while
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: where did they film hot tub time machine?
+A: Fernie Alpine Resort
+
+Q: who has the right of way in international waters?
+A: Neither vessel
+
+Q: who does annie work for attack on titan?
+A: Marley
+
+Q: when was the immigration reform and control act passed?
+A: November 6, 1986
+
+Q: when was puerto rico added to the usa?
+A: 1950
+
+Q: love yourself by justin bieber is about who?
+A:
+==================================================
+
+==================================================
+PROMPT #8:
+==================================================
+Charles I (19 November 1600 – 30 January 1649) was King of England, Scotland, and Ireland from 27 March 1625 until his execution in 1649. He was born into the House of Stuart as the second son of King James VI of Scotland, but after his father inherited the English throne in 1603, he moved to England, where he spent much of the rest of his life. He became heir apparent to the kingdoms of England, Scotland, and Ireland in 1612 upon the death of his elder brother, Henry Frederick, Prince of Wales. An unsuccessful and unpopular attempt to marry him to the Spanish Habsburg princess Maria Anna culminated in an eight-month visit to Spain in 1623 that demonstrated the futility of the marriage negotiation. Two years later, he married the Bourbon princess Henrietta Maria of France. After his succession in 1625, Charles quarrelled with the English Parliament, which sought to curb his royal prerogative. He believed in the divine right of kings, and was determined to govern according to his own conscience. Many of his subjects opposed his policies, in particular the levying of taxes without parliamentary consent, and perceived his actions as those of a tyrannical absolute monarch. His religious policies, coupled with his marriage to a Roman Catholic, generated antipathy and mistrust from Reformed religious groups such as the English Puritans and Scottish Covenanters, who thought his views too Catholic. He supported high church Anglican ecclesiastics and failed to aid continental Protestant forces successfully during the Thirty Years' War. His attempts to forcethe Queen as the supreme governor of the Church of England. After the Spanish Armada was defeated in 1588, Its two levels of administration were the House of Lords, composed of influential peers of the realm and Lords Spiritual, and the House of Commons, which consisted of representative members of the aristocracy and the middle-class. Puritans in the House of Commons began demanding more rights, but their demands were ignored. James I would later have problems with them. Before 1600, the Crown founded corporations including the East India Company to monopolise trade routes. Under her successor, James I, further companies were created to colonise North America, including the London Company and the Virginia Company in 1606, and the Massachusetts Bay Company in 1628. Many religious dissidents left England to settle the new world. Stuart dynasty While Elizabeth I maintained a protestant Church, under her successor James, who unified the Scottish and English Crowns, religious and political tensions grew as he asserted a divine right of Kings. In 1605, Guy Fawkes attempted to blow up Parliament, but was caught, tortured and executed. A wave of repression against catholics followed. James acceded to Puritan requests by commissioning the "King James Bible" in 1604, an English language translation and interpretation of the Bible completed in 1611. Possibly persuaded by his (secretly Catholic) wife, James exempted Catholics from paying tithes to the Anglican Church, but this caused a great decrease in Anglican Church revenue, so he quickly took those rights away. The assertion of divine right prompted a series ofThe concept of "British history" began to emerge in the 1600s, largely thanks to the attempts of King James II to assert that the Union of the Crowns of 1603 had created a Kingdom of Great Britain, which in fact did not come into existence until a century later. The governance of the Kingdom of England and the Kingdom of Scotland remained separate until 1707, and until then in most ways the Scots were excluded from sharing in the English overseas possessions. This page presents a timeline of events in English and Scottish history from 1600 until 1699. 1603 – Death of Queen Elizabeth I on 24 March 1603 England – James VI of Scotland crowned King of England (as James I of England) 1603 England – Plague 1605 England and Scotland – on 5 November, the Gunpowder plot is uncovered, in which Guy Fawkes and other catholic associates attempted to blow up the king, James VI and I and the Parliament of England. 1618 England – Execution of Sir Walter Raleigh 1625 England and Scotland – Death of James VI and I on 27 March 1639 England and Scotland – At war until 1644 in what become known as the Bishops' Wars 1640 England – The Long Parliament summoned. 1642 England – English Civil War begins (see Timeline of the English Civil War) 1652 England – Tea arrives in Britain 1666 England – The Great Fire of London ravages the city, 2–5 September. 1688 England – The Glorious Revolution replaces James II with William III
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: where did they film hot tub time machine?
+A: Fernie Alpine Resort
+
+Q: who has the right of way in international waters?
+A: Neither vessel
+
+Q: who does annie work for attack on titan?
+A: Marley
+
+Q: when was the immigration reform and control act passed?
+A: November 6, 1986
+
+Q: when was puerto rico added to the usa?
+A: 1950
+
+Q: who was the ruler of england in 1616?
+A:
+==================================================
+
+==================================================
+PROMPT #9:
+==================================================
+Le mod Hot Coffee (« café chaud » en anglais) est un mini-jeu de la version de 2004 du jeu vidéo Grand Theft Auto: San Andreas. Normalement indisponible, le mini-jeu est parvenu à la connaissance du public en 2005 par la création du mod Hot Coffee, donnant accès au mini-jeu pour la version destinée aux ordinateurs personnels utilisant le système d'exploitation Windows. Le mini-jeu présente des relations sexuelles entre le personnage principal, Carl Johnson, et sa petite amie Denise. Le nom du mod provient de l'offre faite par la petite amie au personnage principal de venir prendre un « café » chez elle, un euphémisme pour parler de l'autre offre. Bien que le mini-jeu ait été seulement souligné lors de la lancée du mod le et qu'il ait été complètement retiré depuis, des versions du mini-jeu ont été retrouvées sur les versions du jeu développées pour la PlayStation 2 et la Xbox. À la mi-, le mini-jeu soulève une forte controverse politique et judiciaire, ce qui amène le reclassement du jeu en un et son retrait de certaines étagères. Depuis, une version mise à jour de San Andreas, avec le mini-jeu complètement retiré, est sortie, ce qui a permis de ramener le jeu à son classement initial. Un patch, désactivant le mini-jeu et mettant fin au jeu lorsque l'on tente d'y accéder, a également été créée pour la version originale. Notes et références Mod de jeu vidéo Grand Theft Auto Controverse autour du jeu vidéode San Andreas. A ESRB alterou o rótulo do jogo para "Adults Only" (AO) — somente para adultos —, fazendo com que muitos varejistas o retirassem da venda, enquanto que a OFLC emitiu uma classificação recusada, banindo o jogo na Austrália. A Rockstar fez um retalho de todas as cópias de varejo de San Andreas, lançou uma nova versão que bloqueava o acesso ao "Hot Coffee" no final de 2005, e lançou um patch para os proprietários existentes para impedir o acesso. O código-fonte do "Hot Coffee" permanece em todas as versões de San Andreas, mas exige mais esforço para ser restaurado devido à falta de modelos e animações. "Hot Coffee" levantou preocupações de legisladores relacionados às classificações obrigatórias do jogo, e vários processos civis foram iniciados contra a Rockstar. A empresa-mãe da Rockstar, Take-Two Interactive, já estava sob investigação pela Comissão de Valores Mobiliários dos Estados Unidos (SEC) em relação ao comércio de informações privilegiadas, e as ações da legisladora devido ao "Hot Coffee" desencadearam uma investigação separada pela Federal Trade Commission. Esses eventos levaram a Take-Two a perder mais de 163 milhões de dólares em 2006 e fez com que os acionistas demitissem o fundador e presidente da empresa, Ryan Brant, e o substituíssem por Strauss Zelnick. Jogabilidade Em Grand Theft Auto: San Andreas, o jogador, como o protagonista Carl "CJ" Johnson, progride no jogo de ação-aventura por meio de uma série de missões baseadas em uma história. Entre as missões, o jogador pode se envolver em uma série de atividades paralelas para ganhar"Hot Coffee" is the unofficial name for a minigame in the 2004 action-adventure video game Grand Theft Auto: San Andreas by Rockstar Games. While it was not playable in the official game release, the modding community discovered hidden code that, when enabled, allows protagonist Carl "CJ" Johnson to have animated sexual intercourse with his in-game girlfriend. Rockstar Games president Sam Houser wanted to include more role-playing elements in San Andreas while also pushing the Grand Theft Auto series' controversial reputation. The development team was forced to curtail the nudity and sexual content of Houser's original vision, however, to obtain a "Mature" rating from the Entertainment Software Rating Board (ESRB). Rather than removing the content, the developers made it inaccessible to players. Modders discovered the code on the game's PlayStation 2 release, and when San Andreas was released for Windows, modder Patrick Wildenborg disabled the controls around the code. He released this modified code online under the name "Hot Coffee". The discovery of the "Hot Coffee" minigame resulted in intense legal backlash for Rockstar Games and their parent company, Take-Two Interactive. While both companies remained mostly silent on the matter, Rockstar Games released a statement claiming that modders were responsible for the minigame. The ESRB re-rated the game "Adults Only" after an investigation, while the game was banned entirely in Australia until the explicit content was removed. Rockstar Games and Take-Two received a warning from the Federal Trade Commission (FTC) for failing to disclose the extent of graphic content present in the game, while a class action
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: where did they film hot tub time machine?
+A: Fernie Alpine Resort
+
+Q: who has the right of way in international waters?
+A: Neither vessel
+
+Q: who does annie work for attack on titan?
+A: Marley
+
+Q: when was the immigration reform and control act passed?
+A: November 6, 1986
+
+Q: when was puerto rico added to the usa?
+A: 1950
+
+Q: what is the hot coffee mod in san andreas?
+A:
+==================================================
+
+==================================================
+PROMPT #10:
+==================================================
+rate of 802.11ad is 7 Gbit/s. IEEE 802.11ad is a protocol used for very high data rates (about 8 Gbit/s) and for short range communication (about 1–10 meters). TP-Link announced the world's first 802.11ad router in January 2016. The WiGig standard is not too well known, although it was announced in 2009 and added to the IEEE 802.11 family in December 2012. 802.11af IEEE 802.11af, also referred to as "White-Fi" and "Super Wi-Fi", is an amendment, approved in February 2014, that allows WLAN operation in TV white space spectrum in the VHF and UHF bands between 54 and 790 MHz. It uses cognitive radio technology to transmit on unused TV channels, with the standard taking measures to limit interference for primary users, such as analog TV, digital TV, and wireless microphones. Access points and stations determine their position using a satellite positioning system such as GPS, and use the Internet to query a geolocation database (GDB) provided by a regional regulatory agency to discover what frequency channels are available for use at a given time and position. The physical layer uses OFDM and is based on 802.11ac. The propagation path loss as well as the attenuation by materials such as brick and concrete is lower in the UHF and VHF bands than in the 2.4 GHz and 5 GHz bands, which increases the possible range. The frequency channels are 6 to 8 MHz wide, depending on the regulatory domain. Up to four channels may be bonded in either one or two contiguous blocks. MIMO operation isa higher throughput, due to the additional data stream. Data rates Assuming equal operating parameters to an 802.11g network achieving 54 megabits per second (on a single 20 MHz channel with one antenna), an 802.11n network can achieve 72 megabits per second (on a single 20 MHz channel with one antenna and 400 ns guard interval); 802.11n's speed may go up to 150 megabits per second if there are not other Bluetooth, microwave or Wi-Fi emissions in the neighborhood by using two 20 MHz channels in 40 MHz mode. If more antennas are used, then 802.11n can go up to 288 megabits per second in 20 MHz mode with four antennas, or 600 megabits per second in 40 MHz mode with four antennas and 400 ns guard interval. Because the 2.4 GHz band is seriously congested in most urban areas, 802.11n networks usually have more success in increasing data rate by utilizing more antennas in 20 MHz mode rather than by operating in the 40 MHz mode, as the 40 MHz mode requires a relatively free radio spectrum which is only available in rural areas away from cities. Thus, network engineers installing an 802.11n network should strive to select routers and wireless clients with the most antennas possible (one, two, three or four as specified by the 802.11n standard) and try to make sure that the network's bandwidth will be satisfactory even on the 20 MHz mode. Data rates up to 600 Mbit/s are achieved only with the maximum of four spatial streams using one 40maximum net data rate of 54 Mbit/s, plus error correction code, which yields realistic net achievable throughput in the mid-20 Mbit/s. It has seen widespread worldwide implementation, particularly within the corporate workspace. Since the 2.4 GHz band is heavily used to the point of being crowded, using the relatively unused 5 GHz band gives 802.11a a significant advantage. However, this high carrier frequency also brings a disadvantage: the effective overall range of 802.11a is less than that of 802.11b/g. In theory, 802.11a signals are absorbed more readily by walls and other solid objects in their path due to their smaller wavelength, and, as a result, cannot penetrate as far as those of 802.11b. In practice, 802.11b typically has a higher range at low speeds (802.11b will reduce speed to 5.5 Mbit/s or even 1 Mbit/s at low signal strengths). 802.11a also suffers from interference, but locally there may be fewer signals to interfere with, resulting in less interference and better throughput. 802.11b The 802.11b standard has a maximum raw data rate of 11 Mbit/s (Megabits per second) and uses the same media access method defined in the original standard. 802.11b products appeared on the market in early 2000, since 802.11b is a direct extension of the modulation technique defined in the original standard. The dramatic increase in throughput of 802.11b (compared to the original standard) along with simultaneous substantial price reductions led to the rapid acceptance of 802.11b as the definitive wireless LAN technology. Devices using 802.11b experience interference from other products operating in the 2.4
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: where did they film hot tub time machine?
+A: Fernie Alpine Resort
+
+Q: who has the right of way in international waters?
+A: Neither vessel
+
+Q: who does annie work for attack on titan?
+A: Marley
+
+Q: when was the immigration reform and control act passed?
+A: November 6, 1986
+
+Q: when was puerto rico added to the usa?
+A: 1950
+
+Q: what is the maximum data rate for the 802.11a standard select one?
+A:
+==================================================
+
+==================================================
+PROMPT #11:
+==================================================
+Karnataka es un estado de la República de la India. Su capital es Bangalore. Está ubicado al suroeste del país, limitando al noroeste con Goa, al norte con Maharastra, al noreste con Telangana, al este con Andhra Pradesh, al sureste con Tamil Nadu al sur con Kerala, y al oeste con el Mar Arábigo (océano Índico). Con 61 095 297 habitantes en 2011 es el octavo estado más poblado —por detrás de Uttar Pradesh, Maharastra, Bihar, Bengala Occidental y Madhya Pradesh, Tamil Nadu y Rajastán— y con 191 791 km², el séptimo más extenso, por detrás de Rajastán, Madhya Pradesh, Maharastra, Uttar Pradesh, Jammu y Cachemira y Guyarat. Antes de 1973 se le conocía como Estado de Mysore y estaba formado por el antiguo reino de Mysore. En 1956, se incluyeron también algunas zonas anexas en las que el idioma principal era el canarés. Aparte de la capital, otras ciudades importantes son Mysore, Mangalore, Hubli-Dharwad, Bellary y Belgaum. El idioma oficial es el canarés. Según datos del censo de 2001, Karnataka tenía una población de 55.868.200 habitantes. Se cree que el origen del nombre de Karnataka proviene de la unión de los términos karu y nadu que significan "tierra elevada". Historia La prehistoria de Karnataka se remonta a las culturas de hacha de mano del Paleolítico. Herramientas y otros objetos achelenses han sido encontrados en Isampur, valle del Hunsgi, que podrían datar de hace 1,27 millones de años. Hachas paleolíticas de mano y cuchillas en forma de canto rodado, hechas con cuarzo y cuarcita se hanhas his official residence in Vijayawada Chhattisgarh: Raipur is the administrative and legislative capital, while the high court (judiciary capital) is located in Bilaspur. The proposed future capital is Nava Raipur. Jammu and Kashmir: Srinagar serves as the summer capital of the state while Jammu is the winter capital. Every six months, the entire state machinery shifts from one city to another. Kerala: Thiruvananthapuram is the administrative and legislative capital of the state, while the high court is located in Ernakulam. Himachal Pradesh: Shimla is the primary capital city. Dharamshala, which is also the headquarters of the Central Tibetan Administration, is the second winter capital of the state. Madhya Pradesh: Bhopal is the administrative and legislative capital of the state, while the high court is located in Jabalpur. Punjab and Haryana: Both states share Chandigarh as their capital city. The city itself is administered as a Union territory. Odisha: Bhubaneswar is the administrative and legislative capital of the state, while the high court is located in Cuttack. Rajasthan: Jaipur is the administrative and legislative capital of the state, while the high court is located in Jodhpur. Uttarakhand: Dehradun is the administrative and legislative capital, while the high court is located in Nainital. The proposed future capital is Gairsain. Ladakh: Leh and Kargil serve as joint capitals of the Union Territory. South Korea: Seoul remains as the capital and seat of the government's branches, but many government agencies have moved to Sejong City. Malaysia: Kuala Lumpur is the constitutional capital, home of the King, and seat ofstates and union territories located in North India are Hindi, Urdu, Punjabi, Kashmiri, Dogri, and English. Definitions Different authorities and sources define North India differently. Government of India definitions The Northern Zonal Council is one of the advisory councils, created in 1956 by the States Reorganisation Act to foster interstate co-operation under the Ministry of Home Affairs, which included the states of Chandigarh, Delhi, Haryana, Himachal Pradesh, Jammu and Kashmir, Ladakh, Punjab and Rajasthan. The Ministry of Culture established the North Culture Zone in Patiala, Punjab on 23 March 1985. It differs from the North Zonal Council in its inclusion of Uttarakhand and the omission of Delhi. In contrast, the Geological Survey of India (part of the Ministry of Mines) included Uttar Pradesh and Delhi in its Northern Region, but excluded Rajasthan and Chandigarh, with a regional headquarters in Lucknow. Wider definition Indian press definition The Hindu newspaper puts Bihar, Delhi and Uttar Pradesh related articles on its North pages. Articles in the Indian press have included the states of Bihar, Gujarat, Jharkhand, Madhya Pradesh, and West Bengal in North India as well. Latitude-based definition The Tropic of Cancer, which divides the temperate zone from the tropical zone in the Northern Hemisphere, runs through India, and could theoretically be regarded as a geographical dividing line in the country. Indian states that are entirely above the Tropic of Cancer are Himachal Pradesh, Punjab, Haryana, Delhi, Uttarakhand, Uttar Pradesh and Bihar and most of North East Indian states. However that definition would also include major parts of Rajasthan,
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: where did they film hot tub time machine?
+A: Fernie Alpine Resort
+
+Q: who has the right of way in international waters?
+A: Neither vessel
+
+Q: who does annie work for attack on titan?
+A: Marley
+
+Q: when was the immigration reform and control act passed?
+A: November 6, 1986
+
+Q: when was puerto rico added to the usa?
+A: 1950
+
+Q: which state is located in the centre of india?
+A:
+==================================================
+
+==================================================
+PROMPT #12:
+==================================================
+"Runaway Child, Running Wild" (shown as "Run Away Child, Running Wild" on the label of the original single) is a 1969 hit single for the Gordy (Motown) label, performed by The Temptations and produced by Norman Whitfield. The single was both the second from their landmark Cloud Nine LP, and the second of their "psychedelic soul" tracks penned by Whitfield and former Motown artist Barrett Strong. Featuring all five Temptations - Dennis Edwards, Eddie Kendricks, Paul Williams, Melvin Franklin, and Otis Williams - trading verses and harmony lines, "Runaway Child, Running Wild" paints a tale of a young boy (presumably a preteen) who runs away from home after being punished for playing hooky i.e. absence from school without permission. The boy wanders the dark streets alone, eventually realizing he cannot survive on his own, but cannot find his way home, and ends up lost, frightened by strangers, unfamiliar landmarks, and his own thoughts. "Runaway child runnin' wild," the Temptations tell the boy during the chorus, "you better go back home/where you belong". The Temptations alternately express and depict his fears, with the tension of the record building to a climax over the first five minutes of the record. At this point, the vocals fade out, and the record briefly gives voice to its young protagonist, who cries desperately for his mother before fading into the mix. "Runaway Child" then segues into an extended instrumental passage, during which Earl Van Dyke's Hammond organ, Joe Messina's electric guitar, and Dennis Coffey's distorted wah-wah pedal guitar take center stageAll the Way Home may refer to: All the Way Home (play), a 1960 play by Tad Mosel adapted from the James Agee novel A Death in the Family All the Way Home (1963 film), a 1963 film adapted from Mosel's play and Agee's novel All the Way Home (1981 film), a TV adaptation of the play and novel, starring Sally Field and Ellen Corby All the Way Home (novel), a 2001 novel by Patricia Reilly Giff Music All the Way Home, a 2012 solo album by Cathy Jordan All The Way Home, a 1970 album by Randy & Gary Scruggs "All the Way Home" (Bruce Springsteen song) "All the Way Home" (Tamar Braxton song), 2013 "All the Way Home", a song recorded by Frank Sinatra written by Teddy Randazzo "All the Way Home", a 1959 song by Andy Williams released on Andy Williams Sings Steve Allen "All the Way Home", a 1960 song by Bobby Darin released on For Teenagers Only "All the Way Home", a 1992 song by Spinal Tap released on Break Like the Wind "All the Way Home", a 1994 song by Third Matinee released on Meanwhile "All the Way Home", a 2000 song by Andrew Peterson released on Carried Along "All the Way Home", a 2000 song by Peter Mulvey released on The Trouble with Poets "All the Way Home", a 2004 song by Jerry Gaskill released on Come Somewhere "All the Way Home", a 2014 song by Melissa Etheridge released on This Is M.E. See also"Sorry (I Ran All the Way Home)" is a song written by Artie Zwirn and Harry Giosasi and produced and arranged by LeRoy Holmes. The single was performed by New York-based doo-wop group The Impalas. It reached #2 on the U.S. pop chart, behind both The Happy Organ by Dave "Baby" Cortez and Kansas City by Wilbert Harrison. It also went to #14 on the U.S. R&B chart. Overseas, "Sorry (I Ran All the Way Home)" went to #28 on the UK Singles Chart in 1959. The song was featured on The Impalas' 1959 album, Sorry (I Ran All the Way Home). The song ranked #24 on Billboard's Year-End top 100 singles of 1959. Other versions Guy Darrell and The Midniters, as a single in 1964, but it did not chart. Heinz, on his 1964 album, Tribute to Eddie. The Royal Showband Waterford, as the B-side to their 1964 single "Huckle Buck". Phil Orsi and The Little Kings, as a single in 1966, but it did not chart. References 1959 songs 1959 singles 1964 singles 1966 singles Doo-wop songs
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: where did they film hot tub time machine?
+A: Fernie Alpine Resort
+
+Q: who has the right of way in international waters?
+A: Neither vessel
+
+Q: who does annie work for attack on titan?
+A: Marley
+
+Q: when was the immigration reform and control act passed?
+A: November 6, 1986
+
+Q: when was puerto rico added to the usa?
+A: 1950
+
+Q: who sang i ran all the way home?
+A:
+==================================================
+
+==================================================
+PROMPT #13:
+==================================================
+Wallace is a Scottish & English given name meaning 'Wales' or 'of Wales, and may refer to: Wallace (footballer, born 1989), Brazilian footballer, most recently played for Rapid Bucharest Wallace (footballer, born May 1994), Brazilian footballer Wallace (footballer, born October 1994), Brazilian footballer, plays for Lazio Wallace (footballer, born 1986), Brazilian footballer Wallace W. Andrew (1850–1919), American politician Wallace Barnes (1926–2020), American politician Wallace Beery (1885–1949), American actor Wallace "Wally" Berman (1926–1976), American filmmaker and artist Wallace Carothers (1896–1937), inventor of nylon Wallace Chung (born 1974), Hong Kong singer and actor Wallace E. Conkling (1896–1979), American Episcopal bishop Wallace de Souza (born 1987), Brazilian volleyball player generally known simply as "Wallace" Wallace H. Graham (1910–1996), White House Physician 1945–1953 Wallace Wilson Graham (1815–1898), American lawyer and legislator from Wisconsin Wallace Gustafson (1925–2018), American lawyer and politician Wallace Johnson (baseball) (born 1956), American former professional baseball player and coach Wallace E. Johnson (1901–1988), co-founder of Holiday Inn Wallace F. Johnson (1889–1971), American tennis player Wallace J.S. Johnson (1913–1979), Mayor of Berkeley, California during the 1960s Wallace Reid (1891–1923), American actor during the silent-film era Wallace M. Rogerson, American exercise leader Wallace Arthur Sabin (1869–1937), composer and organist Wallace Clement Sabine (1868–1919), founder of acoustic science Wallace Shawn (born 1943), American actor, comedian and playwright Wallace Stegner (1909–1993), American environmentalist and fiction writer Wallace Stevens (1879–1955), American poet Wallace Thayer (1866–1944), New York assemblyman Wallace Thompson (1896–1952), American lawyer and legislator Wallace "Wally" Wolf (1930–1997), American swimmer, water polo player, and Olympic champion Wallace "Wally" Wingert (born 1961),Wallace, was an Ulster-Scots immigrant from the village Kilrea in County Londonderry, Ireland, who arrived in Philadelphia in 1823. May (née Broadhead) was born in New York City but was raised by an aunt in Muscatine, Iowa, after her parents' death. Wallace's family moved to Ames, Iowa, in 1892 and to Des Moines, Iowa, in 1896. In 1894, the Wallaces established an agricultural newspaper, Wallace's Farmer. It became extremely successful and made the family wealthy and politically influential. Wallace took a strong interest in agriculture and plants from a young age and befriended African-American botanist George Washington Carver, with whom he frequently discussed plants and other subjects. Wallace was particularly interested in corn, Iowa's key crop. In 1904, he devised an experiment that disproved agronomist Perry Greeley Holden's assertion that the most aesthetically pleasing corn would produce the greatest yield. Wallace graduated from West High School in 1906 and enrolled in Iowa State College later that year, majoring in animal husbandry. He joined the Hawkeye Club, a fraternal organization, and spent much of his free time continuing to study corn. He also organized a political club to support Gifford Pinchot, a Progressive Republican who was head of the United States Forest Service. Journalist and farmer Wallace became a full-time writer and editor for Wallace's Farmer after graduating from college in 1910. He was deeply interested in using mathematics and economics in agriculture and learned calculus as part of an effort to understand hog prices. He also wrote an influential article with pioneering statistician George W. SnedecorThe Clan Wallace is a Lowlands Scottish Clan and is officially recognized as such by the Lord Lyon King of Arms. The most famous member of the clan was the Scottish patriot William Wallace of the late 13th and early 14th centuries. History Origins of the clan The Wallace family first came to Scotland with a Breton family in the 11th century. David I of Scotland was eager to extend the benefits of Norman influence and gave grants to the nobles of the south. Among them was Walter fitz Alan, who the Scottish king appointed his Steward in 1136. One of Fitzallan's followers was Richard Wallace from Oswestry who came north to try to improve his fortunes. Oswestry is on the Welsh border. So it is possible that the name Wallace may be a corruption of Le Waleis meaning the "Welshman". One similar theory is the name's being derived from the Old English wylisc (pronounced "wullish"), meaning "foreigner" or "Welshman".. However, while it is possible that the Wallaces in the Clyde area were originally Britons from Wales, who came north with David I of Scotland in the eleventh century, another theory is that they were Britons who had settled in Strathclyde/Strathclyde kingdom in the tenth century, as the “Welsh” term was also used for the Cumbric-speaking Celtic Britons of that area. The SLine truncated
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: where did they film hot tub time machine?
+A: Fernie Alpine Resort
+
+Q: who has the right of way in international waters?
+A: Neither vessel
+
+Q: who does annie work for attack on titan?
+A: Marley
+
+Q: when was the immigration reform and control act passed?
+A: November 6, 1986
+
+Q: when was puerto rico added to the usa?
+A: 1950
+
+Q: where did the last name wallace come from?
+A:
+==================================================
+
+==================================================
+PROMPT #14:
+==================================================
+that episode, Robinette accuses McCoy and the D.A.'s office of participating in a giant conspiracy to make every Muslim appear to be a terrorist. However, he maintains a respectful relationship with the DA's office, even sometimes having drinks with his now-opponents. In 2017, the character appeared in "Uncertainty Principle", an episode of Chicago P.D., which is set in the same fictional universe as Law & Order. In the episode, Robinette travels to Chicago to defend CPD Officer Kevin Atwater (LaRoyce Hawkins) at the request of Sergeant Hank Voight (Jason Beghe), and goes up against Ben Stone's son, Cook County Assistant State's Attorney Peter Stone (Philip Winchester). References Law & Order characters Fictional African-American people Fictional assistant district attorneys Fictional characters from New York City Television characters introduced in 1990 Fictional lawyers Crossover characters in television American male characters in television pt:Paul Robinettetime has already declared himself "unavailable" to hear the case. In the Chicago Justice episode "Uncertainty Principle", Stone receives a call from his son, Peter, who had just faced off against his former colleague Paul Robinette. The opening scene of the Law & Order: Special Victims Unit episode "The Undiscovered Country" (aired in 2018) reveals that Stone has died. His successor, Jack McCoy, gives the eulogy at the funeral. The Special Victims Unit episode "Dear Ben" mentions Stone several times. The plot of the episode revolves around the cold case of a serial rapist whom Stone repeatedly attempted to track down and prosecute, to no avail. The Special Victims Unit detectives eventually bring the rapist to justice. Family Stone's son is Peter Stone (Philip Winchester), formerly a Cook County Assistant State's Attorney and lead character on Chicago Justice. Peter first appears on the Chicago P.D. episode "Justice", which is the backdoor pilot for Chicago Justice. Peter also appears in season 19 of Law & Order: Special Victims Unit, and eventually becomes the titular sex crimes bureau's ADA. In several episodes, Peter describes Ben as a distant father who neglected his family in favor of work. Stone also had a daughter, Pamela (Amy Korb), who is introduced in the SVU episode "Send in the Clowns". Pamela has paranoid schizophrenia, and has been hospitalized for most of her life. According to Peter, their father would visit Pamela once a week, and Peter has continued the visits since their father's death. She is murdered by sex traffickers in theBenjamin "Ben" Stone is a fictional character portrayed by Michael Moriarty in the TV drama Law & Order. He was the Executive Assistant District Attorney for New York County until his resignation at the end of season four. He appeared in 88 episodes. Character overview Stone works in the Manhattan District Attorney's office under Alfred Wentworth (Roy Thinnes, in the pilot episode "Everybody's Favorite Bagman") and Adam Schiff (Steven Hill). He was born in 1944 and raised in an Irish Catholic family. He was raised mostly by his Irish grandmother, his father having been an alcoholic. Stone is divorced and has a son, Peter Stone (Philip Winchester), who was the Deputy Chief of the Special Prosecutions Bureau in Chicago and was the Chief ADA of the Sex Crimes Bureau in New York, and a daughter, Pamela (Amy Korb). Stone's prosecutorial methods are portrayed as being grounded in moral principles stemming from his Catholic faith. He is shown to be anti-abortion and opposed to the death penalty. He also is a strong advocate of social justice, having marched in the civil rights movement. It is implied that Stone, like actor Moriarty, is a Dartmouth College alumnus, with references to "fraternity row" and New Hampshire. Stone is also likely a graduate of New York University Law School, evidenced by the diplomas on his wall in several episodes. Career within the show Stone became Executive Assistant DA in 1985 after convicting con artist and murderer Philip Swann (Željko Ivanek), who years later enters a civil suit against Stone when
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: where did they film hot tub time machine?
+A: Fernie Alpine Resort
+
+Q: who has the right of way in international waters?
+A: Neither vessel
+
+Q: who does annie work for attack on titan?
+A: Marley
+
+Q: when was the immigration reform and control act passed?
+A: November 6, 1986
+
+Q: when was puerto rico added to the usa?
+A: 1950
+
+Q: who was the actor that played ben stone on law and order?
+A:
+==================================================
+
+==================================================
+PROMPT #15:
+==================================================
+of The Lion Guard. Nala is the most significant female character in The Lion King. As the film was inspired by William Shakespeare's tragedy Hamlet, Nala is considered to be The Lion King's equivalent of Hamlet's love interest Ophelia, although differences remain between the two characters. Many early concepts first developed for Nala were eventually abandoned, including a brother and father for the character, as well as her being romantically pursued by Scar. While critical reception towards Nala has been generally mixed, Kelly's vocal performance has been praised. Nala appears in the Broadway musical adaptation of the film, first played by singer Heather Headley. The character also appears in the television series The Lion Guard, with Gabrielle Union replacing Kelly as the voice of Nala. Beyoncé and Shahadi Wright Joseph voice the character in the 2019 CGI live adaptation of the original film directed by Jon Favreau. Development Conception and creation Screenwriter Linda Woolverton conceived Nala "as part of a gradual progression ... which have driven recent public conversation about what young girls should be able to expect from their cinematic role models." In early versions of the screenplay, Nala has a younger brother named Mheetu, who enjoys accompanying her and her best friend Simba on their adventures. Pronounced "me too", the character's name is a reference to this personality trait. Simba was intended to save the cub from a wildebeest stampede, and Nala would eventually become responsible for protecting Mheetu from Simba's tyrannical uncle Scar. Nala also had a fox friend named Bhati. According toNala is a fictional character in Disney's The Lion King film franchise. Introduced in the animated film The Lion King (1994), Nala subsequently appears as a less prominent character in the film's sequels The Lion King II: Simba's Pride (1998) and The Lion King 1½ (2004), and serves as a recurring character in The Lion Guard (2015–2019). In the original film, the adult Nala is voiced by American actress Moira Kelly. Young Nala's speaking voice is provided by actress Niketa Calame, while singers Laura Williams and Sally Dworsky provide the singing voices of young and adult Nala respectively. Nala is introduced as the daughter of an unnamed lion and Sarafina, the best friend of Simba, and ultimately becomes his wife as well as the daughter-in-law of Mufasa and Sarabi and the niece-in-law of Scar by the end of The Lion King. Nala becomes Simba’s wife as well as his Queen. Nala is also Kopa, Kiara and Kion’s mother. Several years after Simba's uncle Scar has killed Simba's father Mufasa and taken the throne, Nala desperately ventures into the jungle to find help. Upon unexpectedly reuniting with an adult Simba, who she had long been tricked by Scar into presuming dead, Nala encourages him to return to Pride Rock, overthrow his uncle and ultimately become king. As Simba's queen, Nala has a son, Kopa, an adventurous cub in The Lion King: Six New Adventures, a daughter, Kiara, whose story is explored in The Lion King: Simba's Pride and another son named Kion, who serves as the protagonist"would grace any red carpet." Kelly's performance as Nala is responsible for introducing the actress to a younger audience. Previously, she had been better known for appearing in more adult-oriented live-action films at that time. In 2011, Kelly revealed that fans of the film rarely recognize her speaking voice as the voice of Nala, although Robert DeSalvo of NextMovie.com claims that the actress "has a ... distinctive voice that The Lion King fans will instantly recognize as the voice of adult Nala." Characterization and themes A straight woman character, Kelly described Nala as a sensible and nurturing yet authoritative character, while Amber Leab of Bitch Flicks described her as strong, independent and intelligent. Occupying the role of the film's female lead, Nala is The Lion King's most important female character, who contributes to the film's "small romance element". Often identified as the film's second most important character, Taylor Orci of The Atlantic felt that "Nala is really the agent of change in The Lion King", dismissing Simba as a "rich, lazy boyfriend." Leab observed that The Lion King's male characters tend to "take the center stage" while "female characters take a backseat to the action." Lenka Křivánková of Masaryk University wrote in her thesis "1990s Hollywood Break-Away Hits: A Feminist Perspective" that she was not particularly surprised by the film's lack of strong roles for women because of its Shakespearean source material, dubbing the film "an old traditional fairy tale with all its traditional features", including patriarchy and monarchism. Writing for the University of Waterloo's Kinema,
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: where did they film hot tub time machine?
+A: Fernie Alpine Resort
+
+Q: who has the right of way in international waters?
+A: Neither vessel
+
+Q: who does annie work for attack on titan?
+A: Marley
+
+Q: when was the immigration reform and control act passed?
+A: November 6, 1986
+
+Q: when was puerto rico added to the usa?
+A: 1950
+
+Q: who does the voice of nala in the lion king?
+A:
+==================================================
+
+==================================================
+PROMPT #16:
+==================================================
+Gloria Abbott Bardwell is a fictional character from The Young and the Restless, an American soap opera on the CBS network. The role was originated by Joan Van Ark on April 6, 2004, who portrayed the role until 2005, with Judith Chapman assuming the role on January 17, 2005. She is the mother of Michael Baldwin (Christian LeBlanc) and Kevin Fisher (Greg Rikaart). She is known for attracting troublesome men, and being quite troublesome herself. Gloria can be naive at times, always taking things to the extreme. Known for her over-the-top personality, she owned and operated Gloworm, the hottest Genoa City club, until it burned down. Casting In March 2004, it was reported that former Knots Landing star Joan Van Ark would join The Young and the Restless as Gloria Fisher, mother of Michael Baldwin and Kevin Fisher. CBS reportedly wanted a "big name" star to tackle the role of Gloria. Van Ark debuted on April 6, 2004. By December, news broke that Van Ark had decided to leave The Young and the Restless because she was unaccustomed to the rigors of daytime television, including the long work days and the fast turnaround on scripts and production. Within weeks, the role was recast with Judith Chapman; Van Ark was last seen on January 7, 2005, and Chapman assumed the role on January 17, 2005. In March 2011, it was announced that Chapman had been taken off her contract and bumped to a recurring status. In October 2014, it was announced that Chapman would appear as Gloriaand Love Is a Many Splendored Thing, The Young and the Restless' has been successful at replacing some of its lead characters with other actors. Most often, major characters are played by the same actor for decades; if they left the show, the characters left with them. But in the case of The Young and the Restless, their replacements were often popular and remade the character in their own image. When William Grey Espy left the show in 1975, the role of Snapper Foster was given to then-unknown actor David Hasselhoff. Peter Bergman has won three Emmy Awards after replacing Lester as Jack Abbott. And Jess Walton, who took over the role of Jill Foster Abbott after original cast member Brenda Dickson was fired in 1987 amid a fallout with producers, has earned two Emmys. In 2004, Joan Van Ark joined the cast as Gloria Fisher, Michael Baldwin's mother, remaining until early 2005. She was replaced by Judith Chapman. The role of Lily Winters was made popular by actress Christel Khalil. The Young and the Restless recast Lily with Davetta Sherwood in 2005 but the role was eventually given back to Khalil in 2006. The 2005 the role of Mackenzie Browning was recast from Ashley Bashioum to Rachel Kimsey. Kimsey was released from her contract in May 2006 and the character returned in 2009 played by Clementine Ford. The recasting of Mac took place shortly after the character of Billy Abbott returned, played by Billy Miller. There have been many recasts of the role of Billy1990–1991, 1994, 2008–2017, 2019-present (currently played by Elizabeth Hendrickson) Rick Forrester (The Bold and the Beautiful) - 1990–2006, 2007–present (currently played by Jacob Young) Michael Baldwin (The Young and the Restless) - 1991–1993, 1997–present (played by Christian LeBlanc) Nate Hastings (The Young and the Restless) - 1992–2002, 2011, 2019–present (currently played by Sean Dominic) Billy Abbott (The Young and the Restless) - 1993–2003, 2006, 2008–present (currently played by Jason Thompson) Sharon Newman (The Young and the Restless) - 1994–present (currently played by Sharon Case) Phyllis Summers (The Young and the Restless) - 1994–1997, 1997-1999, 2000–2013, 2014–present (currently played by Michelle Stafford) Adam Newman (The Young and the Restless) - 1995–1997, 2002, 2008–2016, 2019-present (currently played by Mark Grossman) Lily Winters (The Young and the Restless) - 1995–1998, 2002–present (currently played by Christel Khalil) References External links 1927 births 2005 deaths Daytime Emmy Award winners Soap opera producers American soap opera writers Burials at Westwood Village Memorial Park Cemetery United States Navy corpsmen University of Michigan alumni 20th-century American screenwriters United States Navy personnel of World War II Deaths from dementia in California Deaths from Alzheimer's disease
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: where did they film hot tub time machine?
+A: Fernie Alpine Resort
+
+Q: who has the right of way in international waters?
+A: Neither vessel
+
+Q: who does annie work for attack on titan?
+A: Marley
+
+Q: when was the immigration reform and control act passed?
+A: November 6, 1986
+
+Q: when was puerto rico added to the usa?
+A: 1950
+
+Q: who plays gram on the young and the restless?
+A:
+==================================================
+
+==================================================
+PROMPT #17:
+==================================================
+The oath of office of the president of the United States is the oath or affirmation that the president of the United States takes upon assuming office. The wording of the oath is specified in Article II, Section One, Clause 8, of the United States Constitution, and a new president must take it before exercising or carrying out any official powers or duties. This clause is one of three oath or affirmation clauses in the Constitution, but it is the only one that actually specifies the words that must be spoken. Article I, Section 3 requires Senators, when sitting to try impeachments, to be "on Oath or Affirmation." Article VI, Clause 3, similarly requires the persons specified therein to "be bound by oath or affirmation, to support this Constitution." The presidential oath requires much more than that general oath of allegiance and fidelity. This clause enjoins the new president to swear or affirm: "I will to the best of my ability, preserve, protect and defend the Constitution of the United States." Text Ceremony A newly elected or re-elected president of the United States begins their four-year term of office at noon on the twentieth day of January following the election, and, by tradition, takes the oath of office during an inauguration on that date; prior to 1937 the president's term of office began on March 4. If January 20 falls on a Sunday, the president will be sworn in that day by taking the oath privately, but will then re-take the oath in a public ceremonyHis Majesty Charles III, his heirs, and successors, according to law. Since 1 January 2004, applicants for British citizenship are also required to make a pledge (see Pledges below). United States The United States Oath of Allegiance (officially referred to as the "Oath of Allegiance," 8 C.F.R. Part 337 (2008)) is an oath that must be taken by all immigrants who wish to become United States citizens. I hereby declare, on oath, that I absolutely and entirely renounce and abjure all allegiance and fidelity to any foreign prince, potentate, state, or sovereignty of whom or which I have heretofore been a subject or citizen; that I will support and defend the Constitution and laws of the United States of America against all enemies, foreign and domestic; that I will bear true faith and allegiance to the same; that I will bear arms on behalf of the United States when required by the law; that I will perform noncombatant service in the Armed Forces of the United States when required by the law; that I will perform work of national importance under civilian direction when required by the law; and that I take this obligation freely without any mental reservation or purpose of evasion; so help me God. For people who object to taking an oath (or are not religious), the words "on oath" can be replaced with "and solemnly affirm", and the words "so help me God" can be omitted. Pledges In some countries, a pledge is taken by a new citizen, as opposed to anThe Oath of Allegiance of the United States is the official oath of allegiance that must be taken and subscribed by every immigrant who wishes to become a United States citizen. The oath may be administered by any immigration judge or any authorized officer of the United States Citizenship and Immigration Services (USCIS), including by any eligible federal judge. Text The current Oath of Allegiance of the United States is as follows: Modifications According to U.S. regulations, the phrase "so help me God" is optional and that the words on oath can be substituted with and solemnly affirm. According to U.S. Congress, if the prospective citizen is unable or unwilling to promise to bear arms or perform noncombatant military service because of "religious training and belief", he or she may request to leave out those clauses. The law specifies: Qualifying for this modification may require supporting documentation from the applicant's religious organization. The applicant is not required to belong to a specific religious group, but must have "a sincere and meaningful belief that has a place in the applicant's life that is equivalent to that of a religious belief." Renunciation of title or order of nobility New citizens who hold foreign titles must also renounce those titles by adding the following phrase:I further renounce the title of (give title or titles) which I have heretofore held; or I further renounce the order of nobility (give the order of nobility) to which I have heretofore belonged. Waivers The requirement to take and subscribe to the Oath of
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: where did they film hot tub time machine?
+A: Fernie Alpine Resort
+
+Q: who has the right of way in international waters?
+A: Neither vessel
+
+Q: who does annie work for attack on titan?
+A: Marley
+
+Q: when was the immigration reform and control act passed?
+A: November 6, 1986
+
+Q: when was puerto rico added to the usa?
+A: 1950
+
+Q: what is the oath that new citizens take?
+A:
+==================================================
+
+==================================================
+PROMPT #18:
+==================================================
+leaving him for dead on the banks of a lava flow, where Vader sustains severe burns. Palpatine finds a barely alive Vader and takes him to Coruscant, where his mutilated body is treated and covered in the black suit first depicted in the original trilogy. When Vader asks if Padmé is safe, Palpatine says that he killed her out of anger, devastating him. At the end of the film, Vader supervises the construction of the first Death Star alongside Palpatine and Tarkin (Wayne Pygram). Jake Lloyd played Anakin Skywalker as a child in The Phantom Menace, while Hayden Christensen played him as a young adult in the following two films. James Earl Jones reprised his role as the voice of Vader under his mask. Sequel trilogy Thirty years after the Galactic Civil War, Darth Vader's melted helmet appears in The Force Awakens (2015), in which Vader's grandson Kylo Ren (Adam Driver) – who has followed in his grandfather's footsteps by falling to the dark side and betraying the Jedi – is seen addressing him, though Vader does not appear in the film. At one point, his helmet was considered as the film's MacGuffin. The helmet appears again in The Rise of Skywalker (2019), when Kylo briefly meditates with it, and during the film's first duel between Kylo and Rey (Daisy Ridley). The helmet is last seen on the planet Kijimi, which is later destroyed by a Sith Star Destroyer. The film also reveals that the voice which Ren perceived coming from Vader's helmet in The Forcea sculptor, he was imprisoned at a young age for his creations, which scared most people who saw them. Momin was eventually rescued by a Sith Lady named Shaa, who trained him in the dark side of the Force until he became more powerful than her and killed her. Momin then built a superweapon to destroy a city and perished when the Jedi intervened to stop him, losing control over the energy he wielded and causing his physical body to be destroyed, leaving behind only his mask with his spirit inside. Many years later, the mask was retrieved from the Jedi Archive vault by Darth Sidious, who gave it to Darth Vader as a gift for his journey to Mustafar, which Vader sought to make his personal stronghold. After Momin's spirit killed some of his personnel, Vader examined the mask and learned of Momin's past, before letting him possess the body of a Mustafarian and build a fortress for him. Shortly after Momin finished building of the fortress, Vader was distracted by an invasion of Mustafarians, which Momin took advantage of to open the door to the Dark Side and resurrect himself. However, after challenging Vader to a duel, Momin met a quick demise by being crushed with a giant rock. Despite Momin's death, his dark essence remained in the mask for several more years. At one point, Momin's mask was aboard the Imperial yacht Imperialis, when a group of thieves led by Lando Calrissian attempt to rob the ship. After Momin's spirit possessed the bodiesof the most iconic villains in popular culture, and has been listed among the greatest villains and fictional characters ever. His masked face is one of the most iconic character designs of all time. Originally a slave on Tatooine, Anakin Skywalker is a Jedi prophesied to bring balance to the Force. He is lured to the dark side of the Force by Chancellor Palpatine (Darth Sidious) and becomes a Sith Lord, assuming the name of Darth Vader. After a lightsaber battle with his former mentor Obi-Wan Kenobi on Mustafar, in which he is severely injured, Vader is transformed into a cyborg. Now the Emperor's handpicked right-hand, he serves the Galactic Empire for over two decades as the Commander-in-Chief of its forces, purging the last Jedi and hunting the Rebels. Vader ultimately redeems himself by saving his son, Luke Skywalker, and killing Palpatine, sacrificing his own life in the process. He is also the secret husband of Padmé Amidala, the biological father of Princess Leia, and the grandfather of Kylo Ren (Ben Solo). In the non-canonical Star Wars Legends continuity, he is also the grandfather of Ben Skywalker, his eponym Anakin Solo, Jaina Solo and Darth Caedus (Jacen Solo), and the great-grandfather of Allana Solo. The character has been portrayed by numerous actors: David Prowse physically portrayed Vader while James Earl Jones has voiced him in all of the films and some television shows, and Sebastian Shaw portrayed the unmasked Anakin in Return of the Jedi, as well as the character's spirit in the original release of
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: where did they film hot tub time machine?
+A: Fernie Alpine Resort
+
+Q: who has the right of way in international waters?
+A: Neither vessel
+
+Q: who does annie work for attack on titan?
+A: Marley
+
+Q: when was the immigration reform and control act passed?
+A: November 6, 1986
+
+Q: when was puerto rico added to the usa?
+A: 1950
+
+Q: who is under the mask of darth vader?
+A:
+==================================================
+
+==================================================
+PROMPT #19:
+==================================================
+(4) his wife taken from him, (5) circumcision, (6) Hagar and Ishmael driven away, (7) the binding of Isaac, and (8) buying the land to bury Sarah. Genesis chapter 24 Josephus reported that Rebekah told Abraham's servant, "my father was Bethuel, but he is dead; and Laban is my brother; and, together with my mother, takes care of all our family affairs, and is the guardian of my virginity." In classical rabbinic interpretation The parashah is discussed in these rabbinic sources from the era of the Mishnah and the Talmud: Genesis chapter 23 A Midrash noted that recorded that "the life of Sarah was a hundred and seven and twenty years" rather than "one-hundred-twenty-seven years," and deduced that as the righteous are whole and unblemished by sin, so are their years reported whole in the Bible. Thus the Midrash taught that at the age of 20, Sarah was as at the age of 7 in beauty, and at the age of 100, she was as at the age of 20 in sin (the age below which Providence does not punish for sin). Rabbi Haggai said in Rabbi Isaac's name that all of the Matriarchs were prophets. Rabbi Abba bar Kahana interpreted the words, "The sun rises, and the sun sets," in Ecclesiastes to teach that before God causes the sun of one righteous person to set, God causes the sun of another righteous person to rise. Thus a Midrash taught that before God allowed Sarah's sun to set, God caused Rebekah's sun to rise. Thus firstGabriel gives a prophecy about seventy weeks and the "Anointed One". If one adds the 180 days that Elizabeth was pregnant before Mary's conception plus the 270 days of Mary's pregnancy plus the forty days of "purification" in Luke 2:22, one gets 490 days, or seventy weeks. Many Christians have seen this as a fulfillment of prophecy, but skeptics tend to counter that Luke could be constructing his story to fit what he sees as fulfillments of prophecy, whether real or not. God's promise of a messiah from the house of David is foretold in 2 Samuel 7. It is significant that Luke states that Mary lives in Nazareth, considered a small backwater town. He thus has God's grace bestowed on a young, unmarried woman living in an insignificant town. Luke frequently has favor shown on women, sinners, and various "unimportant" people. Paul never mentions a virgin birth for Jesus, but in Romans speaks of him as the Son of God by virtue of his resurrection, and states he had a more than human nature. Luke here has Jesus' divine nature declared from the first moment of his conception. Mary and Elizabeth (1:39–56) Mary then goes and visits her relative Elizabeth. Upon meeting the pregnant Mary, Elizabeth feels John move in her womb and is "filled with the Holy Spirit" (verse 41). This may be related to . Elizabeth gives praise to Mary: "Blessed are you among women, and blessed is the fruit of your womb!", words which echo Moses' declaration to the people of Israelthan a stillborn child, which at least finds rest (verse 6). Verse 3 If a man begets a hundred children and lives many years, so that the days of his years are many, but his soul is not satisfied with goodness, or indeed he has no burial, I say that a stillborn child is better than he. The Jerusalem Bible (1966) and The Living Bible (1971) both suggest "as many daughters" as preferable translations in place of "lives many years". Retired Baptist minister Peter Pett suggests that, although Psalm 127 celebrates the blessing of many children, the number 100 should be interpreted as "a great many children, more than the norm". Insatiable longing (6:7–9) While people work to earn a living, Qoheleth argues that they are left unsatisfied because of the unanswered important questions: what is the point of wisdom, and what reason is there for the poor to find a role in life to improve one's lot? Verse 9 advises for contentment, because the desire to find answers beyond the obvious is seen as "vanity" (hebel). Verse 7 All the labour of man is for his mouth, and yet the appetite is not filled. "Appetite" is translated from , , which can also mean "soul, creature, living being, desire". The insatiability of desire is portrayed like "Sheol", the realm of death in the Hebrew Bible (cf. Isaiah 5:14; Habakkuk 2:5; Proverbs 27:20; 30:16). An impasse (6:10–12) This part starts a passage that ends in 7:13–14, insisting that 'no one knows what is good for humanity'.
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: where did they film hot tub time machine?
+A: Fernie Alpine Resort
+
+Q: who has the right of way in international waters?
+A: Neither vessel
+
+Q: who does annie work for attack on titan?
+A: Marley
+
+Q: when was the immigration reform and control act passed?
+A: November 6, 1986
+
+Q: when was puerto rico added to the usa?
+A: 1950
+
+Q: who had a baby at 100 in the bible?
+A:
+==================================================
+
+==================================================
+PROMPT #20:
+==================================================
+to the police authority in the applicant's district. Rifle and shotgun ownership permission can be given to "sober and responsible" persons 18 years or older. The applicant for the permission must document a need for the weapon. Three exceptions exist to this age qualification. Persons under the age of 18, but over 16 may apply for rifle or shotgun ownership licence with the consent of parents or guardian. For handguns, the ownership age is 21, but a license can be issued to those 18 or older if certain criteria are met. However, such license is valid for only 1 year and must be renewed to avoid revocation. For inherited weapons, it is up to the local police chief to make a decision based on the individual facts of the case. Obtaining a license There are two ways of obtaining an ownership license in Norway. The most common is through the process of obtaining a hunting license, the other is through a sports shooting license. For hunting To obtain a hunting license, the applicant must complete a 30-hour, 9-session course and pass a written multiple choice exam. The course includes firearm theory, firearm training, wildlife theory, and environmental protection training. Once the exam is passed, the applicant may enroll in the hunter registry and receive a hunting license. The membership must be renewed each year, through license payment. The hunting license is brought to the police station, where the applicant fills out an application for obtaining the proper firearm for his or her hunt. After evaluation, partmore than 90 days, or a fine of not more than $100.00, or both; Michigan Penal Code pg. 386 Sec. 223 (1). A person 18 years of age can legally purchase a muzzle loading pistol, cannon up to 42 lbs, or long gun from a private seller. A person 18 years of age can legally purchase a handgun from a private seller, but first must obtain a purchasing permit which is good for 30 days or have valid cpl license. Purchasing firearms At the age of 18 or up, it is legal to buy a handgun with a purchase license from a private seller, at the age of 21, it is legal to buy a firearm from a Federally licensed (FFL) dealer. No purchase license is required to purchase a long gun, or muzzle loader (a firearm that is more than 26 inches long) in Michigan. According to state law, a long gun may be purchased by anyone aged 18 or over who is not subject to restrictions based on criminal history, mental health history, or other disqualifying factor. A person must be at least 18 years old to purchase a long gun from a federal dealer or a private seller under Michigan law. Only Michigan residents may purchase handguns, sbr, sbs (meaning firearms that are 26 inches or less in length) in Michigan. Residents of any state may purchase long guns in Michigan and Michigan residents may purchase long guns in any state. NON-CPL Holders Only that are buying a handgun from a private seller:became C categorized. Category D2: No Storage restriction. Ownership and purchase conditions Category D requires the owner to be older than 18. Category C requires the owner to be older than 18, have hunting, shooting or ball-trap license of the current year stamped by a doctor. Category B requires the owner to be older than 18, be affiliated with a shooting range, have attended at least 3 shooting sessions with an instructor, and have a medical certificate. The shooter then receives a 5-year authorization to purchase and own of Category B firearms (and therefore Category C firearms since they are affiliated with a shooting range). Category A11 and A12 purchase not allowed since August 1st 2018, for gun ownership before its same requirement as Category B. No civilian may carry any weapons in a public place. A special form allows a civilian to apply for a 1-year carry license, which allows them to carry a handgun and a maximum of 50 rounds if they are "exposed to exceptional risks to their life". In practice, such authorizations are rare. Exceptions exist for children and teenagers with a shooting or ball-trap license and parental approval. A child aged between 9 and 12 can own D categorized weapon that shoot projectiles In a non pyrotechnic way between 2 and 20 joules. A teenager aged 12 to 16 can own C and D categorized weapons. They can also own one-shot, rimfire Category B firearms if they participate in international shooting competitions (only with a shooting licence). A shooter between the
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: where did they film hot tub time machine?
+A: Fernie Alpine Resort
+
+Q: who has the right of way in international waters?
+A: Neither vessel
+
+Q: who does annie work for attack on titan?
+A: Marley
+
+Q: when was the immigration reform and control act passed?
+A: November 6, 1986
+
+Q: when was puerto rico added to the usa?
+A: 1950
+
+Q: what age do you need to be to buy a bb gun?
+A:
+==================================================
+

benchmarks/data/prompts_g5/prompt_dump_trivia_hnsw.txt

@@ -0,0 +1,484 @@
+=== Prompt Dump for TRIVIA + HNSW ===
+Total prompts: 50
+Showing first 20 prompts:
+
+==================================================
+PROMPT #1:
+==================================================
+Jason Lee also portrays David Seville in live action/CGI films starring Alvin and the Chipmunks, which use a combination of live-action acting and computer animation. While Ross Bagdasarian Jr. does not do any voices for the film series, the films are all produced in association with Bagdasarian Productions, which owns the rights to all of the characters. Portrayed by Filmography Films Television See also References Fictional characters introduced in 1958 Alter egos Alvin and the Chipmunks Fictional managers Fictional producers American male characters in televisionRoss Dickran Bagdasarian (born May 6, 1949) is an American actor, animator and producer, known for his work on the Alvin and the Chipmunks franchise. He is the son of the franchise's creator, Ross Bagdasarian. Early life Bagdasarian was born in Fresno, California, the son of Armenian-American parents Armenuhi Bagdasarian (née Kulhanjian) and Ross Bagdasarian (1919–1972). As a child, he worked with his father on The Alvin Show by helping edit and coordinate the soundtracks and falsetto voice-overs of the Chipmunks. Career Bagdasarian graduated from law school. He succeeded his father as president of Bagdasarian Productions in 1972 after the death of the elder Bagdasarian. The company had fallen into obscurity after significant success between 1958 and the late 1960s. Bagdasarian was also admitted to the California bar as an attorney in 1975. Under Bagdasarian's supervision, new Chipmunks records were created shortly after his marriage to Karman, including Chipmunk Punk. In 1981, the Chipmunks returned to television in the cartoon special A Chipmunk Christmas. Two years later, Ruby-Spears Productions' Alvin and the Chipmunks Saturday morning cartoon series debuted on NBC. Based on that series, a feature film, The Chipmunk Adventure was released in 1987. Bagdasarian voices Alvin, Simon, and Dave Seville, and Karman voices Theodore and the Chipettes (Brittany, Jeanette, and Eleanor). Bagdasarian and Karman hold tight creative and financial control over the Chipmunk franchise, reviewing each and every business contract in great detail. In the mid-90s, Bagdasarian bought out his brother's and sister's portions of the Chipmunk rights, to take complete control of the franchise.Alvin and the Chipmunks, originally David Seville and the Chipmunks or simply The Chipmunks, are an American animated virtual band and media franchise first created by Ross Bagdasarian for novelty records in 1958. The group consists of three singing animated anthropomorphic chipmunks named Alvin, Simon, and Theodore who are originally managed by their human adoptive father, David "Dave" Seville. Bagdasarian provided the group's voices by producing sped-up recordings of his own, a technique pioneered on the successful "Witch Doctor". Later in 1958, Bagdasarian released the similarly-engineered "The Chipmunk Song" for which he came up with the chipmunk characters and their human father, attributing the track to them. David Seville and the Chipmunks released several more records over the following decade until Bagdasarian's death in 1972. The franchise was revived in 1979 with the characters' voices provided by his son Ross Bagdasarian Jr. and the latter's wife Janice Karman. Through the successful franchise, the Chipmunks have become one of the most successful children's artists of all time. It has garnered two number-one singles on the Billboard Hot 100 and won five Grammy Awards, having four Top 10 albums on the Billboard 200 and three certified platinum albums. "The Chipmunk Song" became one of the best-selling singles of all time at 5 million physical copies sold. The Chipmunks were first depicted in animated form in The Alvin Show (1961). The characters have since featured in several television series and films, as well as other media. In 2019, The Chipmunks received a star on the Hollywood Walk of Fame.
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Rita Coolidge sang the title song for which Bond film??
+A: Octopussy
+
+Q: Which Lloyd Webber musical premiered in the US on 10th December 1993??
+A: Sunset Boulevard
+
+Q: Who was the next British Prime Minister after Arthur Balfour??
+A: Campbell-Bannerman
+
+Q: Who had a 70s No 1 hit with Kiss You All Over??
+A: Exile
+
+Q: What claimed the life of singer Kathleen Ferrier??
+A: Cancer
+
+Q: Who was the man behind The Chipmunks?
+A: 
+==================================================
+
+==================================================
+PROMPT #2:
+==================================================
+and the drum set. Their film counterparts are Michelle and Eleni. Production history Broadway (2015-2019) Auditions began on January 19, 2015 for children ages nine through fifteen. Some recruiting was done through the School of Rock after-school educational program (which predated the film by several years) and open calls were held in New York at the Winter Garden, in Chicago and in Los Angeles. The production closed on January 20, 2019, after 1,309 performances. West End (2016–2020) On 7 December 2015, following the show's Broadway opening, it was announced by Andrew Lloyd Webber that the show would transfer to London's West End in autumn 2016, with the intention to open at the London Palladium. On 20 May 2016, the musical was confirmed at the Gillian Lynne Theatre instead of the Palladium with previews starting on 24 October 2016, opening night on 14 November 2016, and public booking opening on 25 May 2016. Lloyd Webber revealed that the production was able to open several months earlier than anticipated due to finding the child musician actors easily. Anna Louizos' scenery has been modified to fit the architecture of the Gillian Lynne Theatre from the traditional proscenium arch stage at Winter Garden Theatre. Changes include the removal of the pre-show curtain, the use of a revolving stage and action taking place in the aisles of the stalls. While the show remains to be set in America, the script has been adapted to include some minor references for a British audience. The original London cast includes David Fynn as DeweyThe Sound of Music, Camelot and Fiddler on the Roof played at the theatre in the early 1980s. In 1984, the interior was extensively modified by the introduction of a 'race track' that ran through the audience, for the show Starlight Express with performers on roller skates. The show premièred on 27 March, composed by Andrew Lloyd Webber and directed by Trevor Nunn and ran for 7,406 performances, over 18 years. With the removal of the 'tracks', the interior was extensively restored by architects Jaques Muir and Partners. This included the removal of 3,500 incandescent lamps that had become difficult to maintain and consumed a considerable amount of power. These were replaced by 88,000 low power LEDs specially designed for the theatre, creating the first auditorium completely lit in this way. Another Lloyd Webber production followed, Bombay Dreams premièred on 19 June 2002. It was created by A. R. Rahman with lyrics by Don Black and was directed by Steven Pimlott, closing after 1,500 performances on 13 June 2004. This was followed by the return to the West End of the Bee Gee's musical Saturday Night Fever on 6 July 2004, closing 22 October 2005 to tour. This was followed on 10 April 2006 by the jukebox musical Movin' Out, featuring the music of Billy Joel. This starred James Fox but ran for only two months. The Broadway musical Wicked received its London première at the venue on 27 September 2006 with a cast featuring Idina Menzel as Elphaba, Helen Dallimore as Glinda, Nigel Planer asand also starred comedian Tim Minchin as Judas Iscariot, former Spice Girl Melanie C as Mary Magdalene and BBC Radio 1 DJ Chris Moyles as King Herod. Tickets for most venues went on sale on 18 May 2012. In 2013, Lloyd Webber reunited with Christopher Hampton and Don Black on Stephen Ward the Musical. For his next project, a 2015 musical adaptation of the 2003 film School of Rock, auditions were held for children aged nine to fifteen in cooperation with the School of Rock music education program, which predated the film by several years. In April 2016, the English National Opera staged a revival of Sunset Boulevard at the London Coliseum. The limited run, semi-staged production directed by Lonny Price brought Glenn Close to reprise her star turn as "Norma Desmond", which was her first time performing the role in London; she had originated the role in Los Angeles in December 1993 and then on Broadway in November 1994 (which won her the 1995 Tony Award for Best Actress in a Musical). The 2016 London revival was so well-received that the production transferred to the Palace Theatre on Broadway in February 2017, making Lloyd Webber the first musical-theatre composer since 1953 to have four musicals running simultaneously on Broadway – a feat that his heroes Rodgers and Hammerstein had previously achieved. Lloyd Webber's memoir, Unmasked, was published in 2018. On 9 September 2018, Lloyd Webber, along with Tim Rice and John Legend each won an Emmy for Jesus Christ Superstar Live in Concert. With this
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Who was the man behind The Chipmunks??
+A: David Seville
+
+Q: Rita Coolidge sang the title song for which Bond film??
+A: Octopussy
+
+Q: Who was the next British Prime Minister after Arthur Balfour??
+A: Campbell-Bannerman
+
+Q: Who had a 70s No 1 hit with Kiss You All Over??
+A: Exile
+
+Q: What claimed the life of singer Kathleen Ferrier??
+A: Cancer
+
+Q: Which Lloyd Webber musical premiered in the US on 10th December 1993?
+A: 
+==================================================
+
+==================================================
+PROMPT #3:
+==================================================
+Cabinet Louis Botha, Prime Minister of the Union of South Africa (1910–1919) Behind Churchill are: George Barnes, leader of the National Democratic and Labour Party Sir Robert Borden, Prime Minister of Canada (1911–1920) To their right are: Arthur Balfour, 1st Earl of Balfour, former Prime Minister of the United Kingdom (1902–1905); First Lord of the Admiralty (1915–1916) and Foreign Secretary (1916–1919) (standing adlocutio in a black suit) H. H. Asquith, 1st Earl of Oxford and Asquith, Prime Minister of the United Kingdom (1908–1916) (sitting in front) Sir Eric Geddes, First Lord of the Admiralty (1917–1919) (behind, cleanshaven) Bonar Law, Leader of the Opposition (United Kingdom) (1911–1915), Secretary of State for the Colonies (1915–1916), Chancellor of the Exchequer (1916–1919) (later Prime Minister of the United Kingdom, 1922–1923) (dark moustache) Edward Morris, 1st Baron Morris, Prime Minister of Newfoundland (1909–1917) (white moustache, in the shadows) Herbert Kitchener, 1st Earl Kitchener, Secretary of State for War (1914–1916) (in the shadows) Bailey decided that the painting should include British and Dominion civilian leaders in office at the beginning and the end of the First World War. It includes Prime Ministers of Australia, Canada, Newfoundland, and New Zealand, and the Prime Ministers, Foreign Secretaries, Secretaries of War, and First Lords of the Admiralty of the United Kingdom, together with two leaders of the British Conservative and Labour parties. The Maharaja of Bikaner, a member of the Imperial War Cabinet and the Indian delegate to the Versailles Peace Conference, stands to the left next to Botha, both in military uniform. Kitchener standsArthur James Balfour, 1st Earl of Balfour, (, ; 25 July 184819 March 1930), also known as Lord Balfour, was a British Conservative statesman who served as Prime Minister of the United Kingdom from 1902 to 1905. As foreign secretary in the Lloyd George ministry, he issued the Balfour Declaration of 1917 on behalf of the cabinet, which supported a "home for the Jewish people" in Palestine. Entering Parliament in 1874, Balfour achieved prominence as Chief Secretary for Ireland, in which position he suppressed agrarian unrest whilst taking measures against absentee landlords. He opposed Irish Home Rule, saying there could be no half-way house between Ireland remaining within the United Kingdom or becoming independent. From 1891 he led the Conservative Party in the House of Commons, serving under his uncle, Lord Salisbury, whose government won large majorities in 1895 and 1900. An esteemed debater, he was bored by the mundane tasks of party management. In July 1902, he succeeded his uncle as prime minister. In domestic policy he passed the Land Purchase (Ireland) Act 1903, which bought out most of the Anglo-Irish land owners. The Education Act 1902 had a major long-term impact in modernising the school system in England and Wales and provided financial support for schools operated by the Church of England and by the Catholic Church. Nonconformists were outraged and mobilised their voters, but were unable to reverse it. In foreign and defence policy, he oversaw reform of British defence policy and supported Jackie Fisher's naval innovations. He secured the Entente Cordiale withthe county of Haddington. In October 1922 he, with most of the Conservative leadership, resigned with Lloyd George's government following the Carlton Club meeting, a Conservative back-bench revolt against continuance of the coalition. Bonar Law became prime minister. Like many Coalition leaders, he did not hold office in the Conservative governments of 1922–1924, but as an elder statesman, he was consulted by the King in the choice of Stanley Baldwin as Bonar Law's successor as Conservative leader in May 1923. His advice was strongly in favour of Baldwin, ostensibly due to Baldwin's being an MP but in reality motivated by his personal dislike of Curzon. Later that evening, he met a mutual friend who asked 'Will dear George be chosen?' to which he replied with 'feline Balfourian satisfaction,' 'No, dear George will not.' His hostess replied, 'Oh, I am so sorry to hear that. He will be terribly disappointed.' Balfour retorted, 'Oh, I don't know. After all, even if he has lost the hope of glory he still possesses the means of Grace.' Balfour was not initially included in Baldwin's second government in 1924, but in 1925, he returned to the Cabinet, in place of the late Lord Curzon as Lord President of the Council, until the government ended in 1929. With 28 years of government service, Balfour had one of the longest ministerial careers in modern British politics, second only to Winston Churchill . Last years Lord Balfour had generally good health until 1928 and remained until then a regular tennis player. Four years previously
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Who was the man behind The Chipmunks??
+A: David Seville
+
+Q: Which Lloyd Webber musical premiered in the US on 10th December 1993??
+A: Sunset Boulevard
+
+Q: Rita Coolidge sang the title song for which Bond film??
+A: Octopussy
+
+Q: Who had a 70s No 1 hit with Kiss You All Over??
+A: Exile
+
+Q: What claimed the life of singer Kathleen Ferrier??
+A: Cancer
+
+Q: Who was the next British Prime Minister after Arthur Balfour?
+A: 
+==================================================
+
+==================================================
+PROMPT #4:
+==================================================
+classic '70s pop song." In 1992, Mexican trio Pandora released a cover version titled "Pierdo el Control" on their album Ilegal. In 1979 Ginger Rogers sang this song on The Love Boat in the episode "Critical Success / The Love Lamp Is Lit / Take My Boyfriend, Please / Rent a Family / The Man in Her Life: Parts 1 & 2" In 2001, the film Get Over It featured a dance to this song at the beginning by some of the cast. References 1973 songs 1975 debut singles Songs written by Neil Sedaka Songs with lyrics by Howard Greenfield Neil Sedaka songs Captain & Tennille songs Andy Williams songs Number-one singles in Australia Billboard Hot 100 number-one singles Cashbox number-one singles RPM Top Singles number-one singles Grammy Award for Record of the Year A&M Records singles Juno Award for Best Selling Single singlesMusic Week rated the song four out of five, concluding, "A third huge hit for the boys." Tracklisting CD single "Kiss You All Over" (Radio Edit) - 4:31 "Kiss You All Over" (Club Mix) - 5:53 "Bonita" (Radio Edit) - 3:54 "Bonita" (Club Mix) - 7:08 Charts Release history References 1978 songs 1978 singles 1997 singles 1998 singles Billboard Hot 100 number-one singles Cashbox number-one singles Exile (American band) songs Number-one singles in New Zealand Number-one singles in South Africa Number-one singles in Australia Songs written by Mike Chapman Song recordings produced by Frank Farian Song recordings produced by Mike Chapman Songs written by Nicky Chinn RAK Records singles Curb Records singles Hilltak Records singles Warner Records singles Arista Records singles No Mercy (pop band) songs Songs about kissing Phyllis Hyman songs"Kiss You All Over" is a 1978 song performed by American group Exile, written by Mike Chapman and Nicky Chinn. It was included on the band's third album, Mixed Emotions (1978), and featured lead vocalist Jimmy Stokley and guitarist J.P. Pennington on vocals. On the American Top 40 broadcast of May 26, 1979, Casey Kasem reported that Chapman stated his source of inspiration for "Kiss You All Over" was "It's Ecstasy When You Lay Down Next to Me" by Barry White. The song was a number one single in the United States, but proved to be Exile's only big hit in the pop market (they would later have great success on the country music charts). It held the number one spot on the Billboard Hot 100 for four weeks (starting September 30), and Billboard ranked it as the No. 5 song for 1978. The track also reached number-one in at least three other nations. In the United Kingdom, the song was released on Mickie Most's RAK Records, and peaked at number 6 on the UK Singles Chart. The strings are played with a synthesizer in a backing track. In 2010, Billboard ranked the song tenth on its list of "The 50 Sexiest Songs of All Time". Lead vocalist on the number, Stokley was ousted from the band in 1979, his health declining thereafter until he died at the age of 41 in 1985. After the success of soft rock singles from the albums Mixed Emotions and All There Is, the band moved into country music in
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Who was the man behind The Chipmunks??
+A: David Seville
+
+Q: Which Lloyd Webber musical premiered in the US on 10th December 1993??
+A: Sunset Boulevard
+
+Q: Who was the next British Prime Minister after Arthur Balfour??
+A: Campbell-Bannerman
+
+Q: Rita Coolidge sang the title song for which Bond film??
+A: Octopussy
+
+Q: What claimed the life of singer Kathleen Ferrier??
+A: Cancer
+
+Q: Who had a 70s No 1 hit with Kiss You All Over?
+A: 
+==================================================
+
+==================================================
+PROMPT #5:
+==================================================
+21st century world: "We dislike low-lying voices, for one thing— contraltos now sound freakish and headmistressy, and even the majority of mezzo-sopranos should more accurately be categorised as almost-sopranos". However, she was "a singer of, and for, her time — a time of grief and weariness, national self-respect and a belief in human nobility". In this context "her artistry stands upright, austere, unfussy, fundamental and sincere". Shortly after Ferrier's death an appeal was launched by Barbirolli, Walter, Myra Hess and others, to establish a cancer research fund in Ferrier's name. Donations were received from all over the world. To publicise the fund a special concert was given at the Royal Festival Hall on 7 May 1954, at which Barbirolli and Walter shared the conducting duties without payment. Among the items was a rendition of Purcell's When I am laid in earth, which Ferrier had often sung; on this occasion the vocal part was played by a solo cor anglais. The Kathleen Ferrier Cancer Research Fund helped establish the Kathleen Ferrier Chair of Clinical Oncology at University College Hospital, in 1984. , it was continuing to fund oncology research. As the result of a separate appeal, augmented by the sales proceeds of a memoir edited by Neville Cardus, the Kathleen Ferrier Memorial Scholarship Fund was created to encourage young British and Commonwealth singers of either sex. The Fund, which has operated from 1956 under the auspices of the Royal Philharmonic Society, initially provided an annual award covering the cost of a year's study to a single prizewinner.In the course of her professional life the English contralto Kathleen Ferrier made a large number of recordings. In the summer of 1944 she signed a contract with Columbia, which lasted until February 1946. She then transferred to Decca, and remained with them until her death in October 1953. Apart from her studio recordings, many of her live performances and broadcast recitals were recorded, sometimes privately. Some of these were later issued as commercial recordings; others are held by individuals or in the archives of broadcasting companies. The following list is neither up to date nor entirely accurate, particularly in regard to a CD issue, entitled 'Kathleen Ferrier Remembered', released in June 2017, on SOMM264, comprising 26 tracks, 19 of which have never previously been issued. Most of these 19 are not listed below. They include Lieder by Schubert, Brahms, Wolf and Mahler and songs by Stanford, Parry, Jacobson and Rubbra, all taken from BBC broadcasts between 1947 and 1952. In April 2019, a recording of Ferrier singing in Bach's 'Magnificat' during the 1950 Vienna International Bach Festival was issued for the first time. The CD catalogue number is SOMM Ariadne 5004 and it also features Irmgard Seefried and Friedl Riegler (sopranos), Hugo Meyer-Welfing (tenor) and Otto Edelmann (bass). The Vienna Philharmonic Orchestra and Chorus of the Vienna State Opera are conducted by Volkmar Andreae. The existence of this recording was not known until a vinyl disc was offered for sale on an internet auction site in 2018. In superb recorded sound, this discovery is aKathleen Mary Ferrier, CBE (22 April 19128 October 1953) was an English contralto singer who achieved an international reputation as a stage, concert and recording artist, with a repertoire extending from folksong and popular ballads to the classical works of Bach, Brahms, Mahler and Elgar. Her death from cancer, at the height of her fame, was a shock to the musical world and particularly to the general public, which was kept in ignorance of the nature of her illness until after her death. The daughter of a Lancashire village schoolmaster, Ferrier showed early talent as a pianist, and won numerous amateur piano competitions while working as a telephonist with the General Post Office. She did not take up singing seriously until 1937, when after winning a prestigious singing competition at the Carlisle Festival she began to receive offers of professional engagements as a vocalist. Thereafter she took singing lessons, first with J.E. Hutchinson and later with Roy Henderson. After the outbreak of the Second World War Ferrier was recruited by the Council for the Encouragement of Music and the Arts (CEMA), and in the following years sang at concerts and recitals throughout the UK. In 1942 her career was boosted when she met the conductor Malcolm Sargent, who recommended her to the influential Ibbs and Tillett concert management agency. She became a regular performer at leading London and provincial venues, and made numerous BBC radio broadcasts. In 1946, Ferrier made her stage debut, in the Glyndebourne Festival premiere of Benjamin Britten's opera The Rape of Lucretia.
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Who was the man behind The Chipmunks??
+A: David Seville
+
+Q: Which Lloyd Webber musical premiered in the US on 10th December 1993??
+A: Sunset Boulevard
+
+Q: Who was the next British Prime Minister after Arthur Balfour??
+A: Campbell-Bannerman
+
+Q: Who had a 70s No 1 hit with Kiss You All Over??
+A: Exile
+
+Q: Rita Coolidge sang the title song for which Bond film??
+A: Octopussy
+
+Q: What claimed the life of singer Kathleen Ferrier?
+A: 
+==================================================
+
+==================================================
+PROMPT #6:
+==================================================
+"You Only Live Twice", performed by Nancy Sinatra, is the theme song to the 1967 James Bond film of the same name. The music was by veteran Bond film composer John Barry, with lyrics by Leslie Bricusse. The song is widely recognized for its striking opening bars, featuring a simple 2-bar theme in the high octaves of the violins and lush harmonies from French horns. It is considered by some to be among the best James Bond theme songs, and has become one of Nancy Sinatra's best known hits. Shortly after Barry's production, Sinatra's producer Lee Hazlewood released a more guitar-based single version. The song has been covered by many artists including Coldplay, Soft Cell, Björk and Shirley Bassey. In 1998, Robbie Williams re-recorded portions of the song (including the opening strings) for use in his UK number-one single "Millennium". Background James Bond veteran John Barry returned to the franchise to produce the score. The lyrics were by Leslie Bricusse, who had previously cowritten the lyrics for the theme to Goldfinger. An initial version of the song was performed by Julie Rogers and recorded with a 50 or 60 piece orchestra at CTS Studios. However, this version was not used since Barry decided to re-write and re-record the song: "It was usually the producers that said 'this isn't working, there's a certain something that it needed'. If that energy wasn't there, if that mysterioso kind of thing wasn't there, then it wasn't going to work for the movie." The Rogers song shares only two lines withBassey belting out the fantastic title song." He added that the remastered edition's sound quality was "impeccable". Chart positions Track listing Credits Project manager: Herb Agner Creative director: Michelle Azzopardi Composer, conductor, primary artist: John Barry Primary artist, vocals: Shirley Bassey Liner notes: Jeff Bond Composer, lyricist: Leslie Bricusse Project manager: Wendy Brueder Producer, reissue producer: Frank Collura Remastering: Bob Fisher Guitar, soloist: Vic Flick Art direction, design: Peter Grant Orchestra contractor: Sid Margo Lyricist: Anthony Newley A&R: Gregg Ogorzelec Engineer: John Richards Saxophone, soloist: John Scott Source: Aftermath Following the success of her performance on the title track, Shirley Bassey sang the title songs for two later Bond films, Diamonds Are Forever and Moonraker. John Barry used the Goldfinger theme on his 1965 John Barry Plays Goldfinger album that featured Robert Brownjohn artwork. References Footnotes Citations Bibliography Soundtrack albums from James Bond films Soundtrack 1964 soundtrack albums EMI Records soundtracks John Barry (composer) soundtracksJames Bond (Roger Moore), and the title evidently refers to the key aerial sequences featured in the movie. Prior to Rita Coolidge being assigned the Octopussy theme, Mari Wilson was a contender, a British singer whose retro-image evoked the mid-'60s when the Bond series originated; but Wilson's lack of a US-profile led to a negative decision. In January 1983, the producer of Octopussy: Cubby Broccoli, stated that he hoped to have current hitmaker Laura Branigan sing the movie's theme song, an artist choice which both Barry and Rice have stated would have pleased them. However, on March 29, 1983 Rita Coolidge was revealed as the singer, a seemingly surprising choice in that Coolidge's career peak had occurred some six years previously. Coolidge recalls that Barbara Broccoli, daughter of Cubby Broccoli and herself the assistant director of Octopussy, was a fan of Coolidge and made a point of playing Coolidge records around her father until "one day [he said], "Who is that? That's the voice I want for the movie." Rice still had to complete his contribution as the singer arrived in the studio, with Coolidge stating that "we were waiting for the lyrics as the instrumental track had already been done." The chorus of "All Time High" features a lyric similar to that of Coolidge's #2 hit "(Your Love Has Lifted Me) Higher and Higher" whose lyric "When you wrap your loving arms around me I can stand up and face the world again" is echoed by the "All Time High" lyric "We'll take on the
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Who was the man behind The Chipmunks??
+A: David Seville
+
+Q: Which Lloyd Webber musical premiered in the US on 10th December 1993??
+A: Sunset Boulevard
+
+Q: Who was the next British Prime Minister after Arthur Balfour??
+A: Campbell-Bannerman
+
+Q: Who had a 70s No 1 hit with Kiss You All Over??
+A: Exile
+
+Q: What claimed the life of singer Kathleen Ferrier??
+A: Cancer
+
+Q: Rita Coolidge sang the title song for which Bond film?
+A: 
+==================================================
+
+==================================================
+PROMPT #7:
+==================================================
+which allowed the first legal beer sales since the beginning of Prohibition on January 16, 1920. In 1933 state conventions ratified the Twenty-first Amendment, which repealed Prohibition. The Amendment was fully ratified on December 5, 1933. Federal laws enforcing Prohibition were then repealed. Dry counties Following repeal some states continued prohibition within their own jurisdictions. Almost two-thirds of the states adopted some form of local option which enabled residents in political subdivisions to vote for or against local prohibition. For a time, 38 percent of Americans lived in areas with Prohibition. By 1966, however, all states had repealed their statewide prohibition laws, with Mississippi the last state to do so. Notes Sources Walker, Robert S. and Samuel C. Patterson, Oklahoma Goes Wet: The Repeal of Prohibiton, Eagleton Institute, Rutgers University, (1961). External links Repeal Day is December Fifth See more related images by selecting the "Alcohol" subject at the Persuasive Cartography, The PJ Mode Collection, Cornell University Library Prohibition in the United States Economic history of the United States 1933 in the United States Articles containing video clipsimportation of alcoholic beverages in the United States. The resolution was sent to the states for ratification and became the Eighteenth Amendment to the U.S. Constitution. On January 8, 1918, Mississippi became the first state to ratify the amendment and on January 16, 1919, Nebraska became the 36th state to do so, securing its passage with the required three-fourths of the states. By the end of February 1919, only three states remained as hold-outs to ratification: New Jersey, Connecticut and Rhode Island. The National Prohibition Act, also known as the Volstead Act, was enacted on October 18, 1919. Prohibition in the United States went into effect on January 17, 1920. Nationwide prohibition was repealed in 1933 with the passage of the Twenty-first Amendment on February 20 and its ratification on December 5. List of formerly dry states This table lists the effective dates each state went dry and any dates of repeal that do not coincide with the end of national prohibition in 1933. See also Dry county Alcoholic beverage control state List of alcohol laws of the United States by state Notes Alcohol law in the United States Prohibition in the United StatesAugust 19. PPS functionals were completed August 21. GATV 5006 was then transferred to complex 14 for mating with the Atlas. July 27, 1966 (Wednesday) Following the announcement of his austerity programme, British Prime Minister Harold Wilson survived a vote of censure in the House of Commons, as members of his Labour Party (with an 88-seat majority) supported him. The final result was 246 votes in favor, and 325 against. On the same day, the nation's chief labor union, the Trades Union Congress, voted 20 to 12 in support of a resolution pledging to halt strikes that had been threatened during the six-month freeze against raising wages. For the first time in 58 years, liquor was legally served in Mississippi, the last of the United States to have repealed its prohibition laws. Effective July 1, individual local governments were allowed to hold referendum elections on whether to allow the sale of liquor at state-approved resorts, and Harrison County voters had endorsed the measure. At 6:55 p.m., after police cars escorted a liquor delivery truck into Biloxi. The first drink in the state was poured at the Broadwater Beach Hotel, and Louis Cobb, the first legal bartender in Mississippi, sold a glass of scotch whiskey to hotel manager T.M. Dorsett. Biloxi Mayor Dan Guice then cut the ribbon to open the entrance to the hotel's bar.Died: Brenda Sue Brown, 11, was beaten to death after walking with her sister to summer school in Shelby, North Carolina. Police were unable to charge a suspect with the crime, until
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Who was the man behind The Chipmunks??
+A: David Seville
+
+Q: Which Lloyd Webber musical premiered in the US on 10th December 1993??
+A: Sunset Boulevard
+
+Q: Who was the next British Prime Minister after Arthur Balfour??
+A: Campbell-Bannerman
+
+Q: Who had a 70s No 1 hit with Kiss You All Over??
+A: Exile
+
+Q: What claimed the life of singer Kathleen Ferrier??
+A: Cancer
+
+Q: What was the last US state to reintroduce alcohol after prohibition?
+A: 
+==================================================
+
+==================================================
+PROMPT #8:
+==================================================
+to New York City for work in summer stock theatre shortly before winning a supporting role in MGM's These Glamour Girls (1939) opposite Lana Turner and Lew Ayres. The role of Betty was said to have been written especially with Hunt in mind. Other roles in major studio productions soon followed, including supporting roles as Mary Bennet in MGM's version of Pride and Prejudice (1940) with Laurence Olivier, and as Martha Scott's surrogate child Hope Thompson in Cheers for Miss Bishop (1941). Years at MGM In 1941, Hunt signed a contract with MGM, where she remained for the next six years. While filming Blossoms in the Dust, film director Mervyn LeRoy lauded Hunt for her heartfelt and genuine acting ability. During this period she had starring roles in 21 films, including The Penalty (1941) opposite Lionel Barrymore, Panama Hattie (1942) opposite Ann Sothern and Red Skelton, and the war drama Pilot No. 5 (1943) in which she was cast as the love interest of Franchot Tone, and The Valley of Decision (1945). In 1944 she polled seventh in a list by exhibitors of "Stars of Tomorrow". She previously did a screen test to play Melanie Hamilton in Gone with the Wind (1939) and was told by David O. Selznick she would play the role, but to "keep it a secret for now." Three days later, it was announced that Olivia de Havilland was cast. In 1944, she appeared in None Shall Escape, a film that is now regarded as the first about the Holocaust. She playedMiss America 1941, the 15th Miss America pageant, was held at the Boardwalk Hall in Atlantic City, New Jersey on September 6, 1941. Shortly after the crowning of Miss California, Rosemary LaPlanche, who had been first runner-up in 1940, the pageant committee adopted this rule: "No contestant can compete in Atlantic City for the title of Miss America more than once", thus eliminating future state winners with more than one attempt at the national title. LaPlanche became a film actress, as did her sister, Louise LaPlanche. 1941 was also the first year that the special award, “Miss Congeniality” was created. It went to Mifaunwy Shunatona, a member of the Otoe and Pawnee tribes — she was also the first American Indian contestant in the pageant’s history. Results Awards Preliminary awards Other awards Contestants References Secondary sources External links Miss America official website 1941 1941 in the United States 1941 in New Jersey September 1941 events Events in Atlantic City, New JerseyMiss America 1942, the 16th Miss America pageant, was held at the Warner Theater in Atlantic City, New Jersey on September 12, 1942. Miss Texas, Jo-Carroll Dennison won the title after winning the swimsuit and talent categories. She was the first Miss Texas to win the Miss America title. Dennison became an actress and had roles in films such as Winged Victory. She was married at one time to comedian Phil Silvers. Results Awards Preliminary awards Other awards Contestants References Secondary sources External links Miss America (1942) 1942 1942 in the United States 1942 in New Jersey September 1942 events Events in Atlantic City, New Jersey
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Who was the man behind The Chipmunks??
+A: David Seville
+
+Q: Which Lloyd Webber musical premiered in the US on 10th December 1993??
+A: Sunset Boulevard
+
+Q: Who was the next British Prime Minister after Arthur Balfour??
+A: Campbell-Bannerman
+
+Q: Who had a 70s No 1 hit with Kiss You All Over??
+A: Exile
+
+Q: What claimed the life of singer Kathleen Ferrier??
+A: Cancer
+
+Q: Which actress was voted Miss Greenwich Village in 1942?
+A: 
+==================================================
+
+==================================================
+PROMPT #9:
+==================================================
+De Tokyo Stock Price Index (Japans: 東証株価指数) of TOPIX is een belangrijke aandelenindex van de Tokyo Stock Exchange. Berekening In deze index zijn alle bedrijven opgenomen die op de beurs van Tokio staan genoteerd in de First Section. Dit zijn de grootste en meest liquide aandelen die op de beurs worden verhandeld. Tot medio 2006 werd het gewicht van de individuele bedrijven in de index bepaald op basis van de marktkapitalisatie, hierna wordt ook de free float in de berekening meegenomen. Het effect van deze verandering was significant, daar veel Japanse bedrijven aandelen houden in andere Japanse bedrijven, ook wel bekend als crossholdings, om daarmee de langdurige zakenrelatie te onderstrepen. Deze belangen worden voor lange tijd gehouden en worden niet tot de free float gerekend. De index heeft 4 januari 1968 als startdatum, maar ging op 1 juli 1969 daadwerkelijk van start. Een andere belangrijke beursindex in Japan is de Nikkei 225. In deze index zijn 225 bedrijven opgenomen en dit is een prijsgewogen index. Samenstelling Eind maart 2021 bestond de index uit 2187 aandelen. Door het grote aantal aandelen is het gewicht van de individuele namen zeer klein. De top 10 aandelen hebben een gezamenlijk gewicht in de index van slechts 18,4% en de lijst zag er als volgt uit, met de gewichten tussen de haakjes: De belangrijkste drie sectoren zijn: elektronische apparatuur, informatie technologie en chemie. Deze drie vertegenwoordigen tezamen zo'n 34% van de index, waarvan de sector elektronische apparatuur het grootst is met een gewicht van 17,5%. Koershistorie De hoogste stand van deTOPIX steht für Tōkyō Stock Price Index (jap. , Tōshō kabuka shisū) und ist neben dem Nikkei 225 ein Kursindex der Tokioter Börse. Berechnet wird der TOPIX seit dem 1. Juli 1969. Die Index-Basis liegt bei 100 Punkten per 4. Januar 1968. Er enthält alle japanischen Aktien, welche im amtlichen Handel zugelassen sind. Die Gewichtung der einzelnen Unternehmen im Index erfolgt anhand der Marktkapitalisierung. Gegenwärtig (8. September 2021) setzt sich der Index aus 2.189 Aktien zusammen. Wegen dieser hohen Zahl an vertretenen Unternehmen wird der TOPIX als aussagekräftiger für den Zustand der japanischen Wirtschaft angesehen als der Nikkei 225. Weblinks Beschreibung des TOPIX (engl.) TOPIX in Echtzeit Jährliche Entwicklung des TOPIX seit 1949 (Daten vor 1969 – dem Einführungsjahr des TOPIX – sind rückgerechnet; XLS-Format, 31,5 KB; abgerufen am 12. Oktober 2017) Einzelnachweise Aktienindex Wirtschaft (Japan) Abkürzung, commonly known as TOPIX, along with the Nikkei 225, is an important stock market index for the Tokyo Stock Exchange (TSE) in Japan, tracking all domestic companies of the exchange's Prime market division. It is calculated and published by the TSE. , there were 1,669 companies listed on the First Section of the TSE, and the market value for the index was ¥197.4 trillion. The index transitioned from a system where a company's weighting is based on the total number of shares outstanding to a weighting based on the number of shares available for trading (called the free float). This transition took place in three phases starting in October 2005 and was completed in June 2006. Although the change is a technicality, it had a significant effect on the weighting of many companies in the index, because many companies in Japan hold a significant number of shares of their business partners as a part of intricate business alliances, and such shares are no longer included in calculating the weight of companies in the index. The TOPIX index is traded as a future on the Osaka Exchange under the ticker symbol JTPX. The CQG contract specifications for the TOPIX Index are listed below. TSE currently calculates and distributes TOPIX every second and further plans to launch a new High-Speed Index dissemination service provided at the millisecond level starting from February 28, 2011. History of TOPIX 1969-07-01 TSE to begin calculating and publishing “TOPIX” and “TOPIX Sector Indices” 1969-08-18 TSE to begin calculating and publishing “Tokyo Stock
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Who was the man behind The Chipmunks??
+A: David Seville
+
+Q: Which Lloyd Webber musical premiered in the US on 10th December 1993??
+A: Sunset Boulevard
+
+Q: Who was the next British Prime Minister after Arthur Balfour??
+A: Campbell-Bannerman
+
+Q: Who had a 70s No 1 hit with Kiss You All Over??
+A: Exile
+
+Q: What claimed the life of singer Kathleen Ferrier??
+A: Cancer
+
+Q: What is the Japanese share index called?
+A: 
+==================================================
+
+==================================================
+PROMPT #10:
+==================================================
+Man in the Music: The Creative Life and Work of Michael Jackson is a non-fiction book written by Joseph Vogel, published in June 2011 by the Sterling Publishing. Reception Man in the Music: The Creative Life and Work of Michael Jackson, was described by the Associated Press as "a fascinating read and really a must have for any fan of Jackson." Filmmaker Spike Lee characterized it as having "brilliantly cracked the DNA, the code, the artistry of Michael Joseph Jackson." References Works about Michael Jackson 2011 non-fiction books Sterling Publishing booksMoonwalk is a 1988 autobiography written by American recording artist Michael Jackson. The book was first published by Doubleday on February 1, 1988, five months after the release of Jackson's 1987 Bad album, and named after Jackson's signature dance move, the moonwalk. The book contains a foreword by Jacqueline Onassis. It reached number one on the New York Times Best Seller list. The book was reissued by Doubleday on October 13, 2009, following Jackson's death on June 25, 2009. Production Jacqueline Onassis, who was an editor at Doubleday, secured the book deal and paid Jackson a $300,000 advance. As part of the deal Jackson wanted Onassis to write a foreword, which she initially refused not wanting her name on any books she worked on but agreed to three paragraphs. She also edited the book. The first manuscript of the book was written by Robert Hilburn and was refused by the publishers, Doubleday, because it lacked "juicy details". A second manuscript was written by Stephen Davis, which Jackson drastically edited. Jackson finally decided to write the book himself, with help from Shaye Areheart. Due to the public interest in Jackson, Moonwalk was prepared for publication in secret. Relatives of Doubleday employees were hired as couriers, to deliver portions of the book from the company's head office in Manhattan to the printing plant in Fairfield, Pennsylvania. At the printing plant, the book was given the code name "Neil Armstrong", after the first "moonwalker". Narrative Dedicated to Fred Astaire, the book discusses Jackson's show business friends, girlfriends and hisMichael Jackson: Unauthorized in a 1994 biography of the late pop star Michael Jackson, written by celebrity biographer Christopher Andersen. Development According to Andersen, work started on the book in early 1991 when he received a call from a fellow journalist, who told him that two workers at Jackson's Neverland Ranch allegedly witnessed Jackson fondling a young celebrity. Andersen tried to interview Jackson several times, but was turned down. When Michael was publicly accused of child molestation in 1993, Andersen was told that he was under surveillance from investigators. Reception The book was largely overlooked by the public. Dana Kennedy of Entertainment Weekly felt that, with its "killer material", Anderson "probably could have retired from the celebrity-bio grind for good" had it been released five years before. People magazine found it to be a "sad book", considering its dark revelations about Jackson's behaviour. References 1994 non-fiction books Unauthorized biographies Works about the Michael Jackson sexual abuse allegations Biographies about musicians
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Who was the man behind The Chipmunks??
+A: David Seville
+
+Q: Which Lloyd Webber musical premiered in the US on 10th December 1993??
+A: Sunset Boulevard
+
+Q: Who was the next British Prime Minister after Arthur Balfour??
+A: Campbell-Bannerman
+
+Q: Who had a 70s No 1 hit with Kiss You All Over??
+A: Exile
+
+Q: What claimed the life of singer Kathleen Ferrier??
+A: Cancer
+
+Q: What was the name of Michael Jackson's autobiography written in 1988?
+A: 
+==================================================
+
+==================================================
+PROMPT #11:
+==================================================
+including popular titles by Sérgio Mendes and Herb Alpert were released with this audio process starting in September 1968. Other record labels soon followed suit, and an estimated 10% of all stereophonic albums released during the late 1960s and early 1970s employed the system. Other labels known to have used the system include Warner Bros. Records and Reprise Records. One of the biggest selling albums using the process is The Association's Greatest Hits, released in 1968. This recording has sold more than 2 million copies in the United States. The process was also used on the 1968 Frank Sinatra album Cycles as well as on most of the studio recordings on Wheels of Fire by Cream. Early 1968 copies of Neil Young's self-titled debut album also used the system. Use of Haeco-CSG in promotional recordings for radio The original intention of using Haeco-CSG on commercial LP releases was rather short lived, however, use of the process continued well into the mid-1970s on promotional records sent to radio stations. Many commercial FM Rock stations did not transition from mono to stereo broadcasting until the mid to late 1970s. AM Pop music stations continued to broadcast in mono, as AM stereo broadcasting was not introduced until 1982 and was never widely adopted. Many promotional singles and some commercial singles from the Warner/Reprise/Atlantic label group from this era had "CSG Mono Process" or "CSG Process" printed on the labels. Artists included Frank Sinatra, Gordon Lightfoot, James Taylor, Seals and Crofts. Warner subsidiary labels such as Atlantic issued a serieswas introduced to the public on December 13, 1957, at the Times Auditorium in New York City. 500 copies of this initial demonstration record were pressed. On December 16, 1957, Frey advertised in the trade magazine Billboard that he would send a free copy to anyone in the industry who wrote to him on company letterhead. Frey became known as "Mr. Stereo" during that era. Stereophonic sound was not entirely new to the public. In 1952 sound engineer Emory Cook developed a "Binaural" disk that used two separate grooves and playback needles to produce stereophonic sound; the following year he had a catalog of about 25 disks available for audiophiles. Multi-channel sound was integral to the widescreen motion picture processes Cinerama (1952) and CinemaScope (1953). Stereophonic audio tapes had been commercially available to audiophiles, although expensive, since the early-1950s. After the release of the Audio Fidelity demonstration disks, the other spur to the popularity of stereo disks was the reduction in price of a stereo magnetic cartridge, for playing the disks, from $250 to $29.95 in June 1958. The first four stereo discs available to the general public were released by Audio Fidelity in March, 1958--Johnny Puleo and his Harmonica Gang Volume 1 (AFSD 5830), Railroad - Sounds of a Vanishing Era (AFSD 5843), Lionel - Lionel Hampton and his Orchestra (AFSD 5849) and Marching Along with the Dukes of Dixieland Volume 3 (AFSD 5851). By the end of March the company had four more stereo LPs available. In the summer of 1958, Audio Fidelity recordedin 1957, with his Essex Records office manager George Phillips, he founded Somerset Records and Somerset Stereo Fidelity Records budget albums. His greatest claim to fame was selling large amounts of cheaply priced albums, with Somerset claiming to have manufactured the first stereo budget albums. The name of Somerset high fidelity albums was suggested by Miller International's West Coast distributor, Jimmy Warren, with the name of Stereo Fidelity (stereo albums) thought of by Wally Hill to capitalize on the public's interest in both high fidelity and stereophonic sound. The economy came from Miller starting his own record factory in Swarthmore, Pennsylvania, using public domain music and non union musicians from outside the United States to record cover versions of hit songs of the time. Many original tunes were written by Monty Kelly, Robert Lowden, and Joseph Kuhn with the music published by Miller's own music publisher, Chesdel Music created in 1962. Miller had his own distribution channels of his records in supermarkets and drugstores with the cheap albums being sold in metal racks similar to those holding paperback books or cardboard record holders called "dumps" that could be placed anywhere. Miller's record albums were sold wholesale for 93 cents to salesmen who sold them to merchants who sold them to the public for $1.98. Somerset Records used artist Anthony "Chic" Laganella to create attractive eye catching album covers. Miller used the name 101 Strings for several German orchestras; their first album appearing in September 1957. In 1958 Somerset released 24 101 Strings titles. Miller International's philosophy
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Who was the man behind The Chipmunks??
+A: David Seville
+
+Q: Which Lloyd Webber musical premiered in the US on 10th December 1993??
+A: Sunset Boulevard
+
+Q: Who was the next British Prime Minister after Arthur Balfour??
+A: Campbell-Bannerman
+
+Q: Who had a 70s No 1 hit with Kiss You All Over??
+A: Exile
+
+Q: What claimed the life of singer Kathleen Ferrier??
+A: Cancer
+
+Q: In which decade did stereo records first go on sale?
+A: 
+==================================================
+
+==================================================
+PROMPT #12:
+==================================================
+Flack in 1896) to win gold medals in both the 800 m and 1500 m in the same Olympics. Billy Mills, an unfancied runner, became the only American to win the gold in the men's 10,000 m. Bob Hayes won the 100 metre title in a time of 10.06 seconds, equaling the world record, and set the current record for the fastest relay leg in the 4×100 m. Joe Frazier, future heavyweight champion of the world, won a gold medal in heavyweight boxing while competing with a broken thumb. This was the last Summer Olympics to use a cinder running track for athletic events, and the first to use fiberglass poles for pole vaulting. Zambia declared its independence on the day of the closing ceremony of the 1964 Summer Olympics, thereby becoming the first country ever to have entered an Olympic games as one country, and left it as another. This was celebrated in the ceremony itself by the team using a placard with "Zambia" instead of the "Northern Rhodesia" placard from the opening ceremony. Zambia was the only team to use a placard in the closing ceremony. The start of operations for the first Japanese "bullet train" (the Tōkaidō Shinkansen) between Tokyo Station and Shin-Ōsaka Station was scheduled to coincide with the Olympic games. The first regularly scheduled train ran on 1 October 1964, just nine days before the opening of the games, transporting passengers in about four hours, and connecting the three major metropolitan areas of Tokyo, Nagoya, and Osaka. Ranatunge Karunananda who representedsystems were used: official hand timing, hand started photo-finish times, and the Gustavus Town Kirby timing device, which was designed by Kirby to determine the correct order of finish in horse races. The official report for 1932 Olympics states: "In addition to hand timing, two auxiliary electrical timing devices were used. Both were started by an attachment to the starters gun. One was stopped by hand at the time the runners hit the tape. The other was provided with a motion picture camera which photographed the runner at the tape and the dial of the time indicator simultaneously." Kirby's system was also used at the 1932 US. Olympic Trials, where Ralph Metcalfe's winning time of 10.62 in the 100 meters is considered possibly the first automatically timed world record. FAT was also used in 1936, but very few times have been found. In 1948, Bulova began developing the Phototimer, a unique combination of photo-finish camera and precision electronic timing instrument. The Phototimer was the first automatic timing device to be used in competitive sports. It was used extensively in North America, including at the 1948 US Olympic trials. The Bulova device was activated by the sound of the starting gun firing, rather than by a direct connection, which means that the times were around 0.02 seconds faster than reality. The 1948 Olympics, however, continued to use Omega timing with a device called the 'Magic Eye', developed by British Race Finish Recording Co. Ltd. The automatic times produced in the 1948 Olympics have never been released, butWhile the most notable story coming out of 1968 was socio-political, politics involved with the Olympics was not something unique to this year. However, the year marked the beginning of several emerging elements of contemporary track and field. Automatic timing While timing to the 100th of a second had been experimented with for many years, the 1968 Summer Olympics were the first to use Fully Automatic Timing, in not only athletics, but in canoeing, rowing, cycling, equestrian and swimming competitions. Subsequently, systems to record such times became more common and thus the accuracy of Fully Automatic Timing became mandated for World Record acceptance. While this rule was officially put into place in 1977, many 1968 records still stood as the first Automatically timed record. All weather tracks This technology too had been developing, but Tartan tracks were used as the competition surface for the first time at an Olympics. Since then an all-weather running track was required for all top-level competition. Subsequently, the inconsistency of the running surface became a significantly smaller factor in athletic performance. Altitude With the Olympics happening in Mexico City, at high altitude, the effect of the thin air on athletic performance became a factor on world records. This was already a known phenomenon, and the American team was selected by holding the Olympic Trials at high altitude at Echo Summit, California. In 1955, Lou Jones set the world record in the 400 meters at altitude in Mexico City. Following the 1968 Summer Olympics the: Men's 100 meters record, set by Jim
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Who was the man behind The Chipmunks??
+A: David Seville
+
+Q: Which Lloyd Webber musical premiered in the US on 10th December 1993??
+A: Sunset Boulevard
+
+Q: Who was the next British Prime Minister after Arthur Balfour??
+A: Campbell-Bannerman
+
+Q: Who had a 70s No 1 hit with Kiss You All Over??
+A: Exile
+
+Q: What claimed the life of singer Kathleen Ferrier??
+A: Cancer
+
+Q: In what year's Olympics were electric timing devices and a public-address system used for the first time?
+A: 
+==================================================
+
+==================================================
+PROMPT #13:
+==================================================
+A list of stratovolcanoes follows below. Africa Cameroon Mount Cameroon Democratic Republic of Congo Mount Nyiragongo, Goma; designated as a Decade Volcano It contains an active lava lake inside its crater which overflowed due to cracks in 2002. Mount Mikeno Eritrea Alid Volcano Dubbi Volcano Nabro Volcano Ethiopia Adwa Borawli, Afar Region Dabbahu Volcano Mount Fentale Kenya Mount Kenya, which contains several volcanic plugs on its peak. Mount Longonot Rwanda Mount Bisoke, on the border between Rwanda and the Democratic Republic of the Congo. Mount Gahinga, on the border between Rwanda and Uganda. Mount Karisimbi, on the border between Rwanda and the Democratic Republic of the Congo. Mount Muhabura, on the border between Rwanda and Uganda. Mount Sabyinyo, marks the border between Rwanda, Uganda, and the Democratic Republic of the Congo. Tanzania Ol Doinyo Lengai, the Earth's only active carbonatite lava-producing volcano. Mount Kilimanjaro, a dormant stratovolcano. It is the highest point of Africa. Mount Meru Mid-Atlantic Ridge Mount Pico in Pico Island, Azores, Portugal Teide in Tenerife, Canary Islands, Spain; designated as a Decade Volcano Cumbre Vieja in La Palma, Canary Islands, Spain Mount Fogo in Fogo, Cape Verde Green Mountain, Ascension Island Pico de las Nieves in Gran Canaria, Canary Islands, Spain Americas Caribbean La Grande Soufrière on Basse-Terre Island, Guadeloupe Soufriere Hills on the island Montserrat Its 1995 eruptions resulted in the abandonment of its capital city, Plymouth. Soufrière on the island Saint Vincent Mount Pelée on the island Martinique Its devastating eruption on 8 May 1902 resulted in the complete destruction ofMount Kilimanjaro is a volcano in Tanzania and the highest mountain in Africa. Kilimanjaro may also refer to: Tanzania Kilimanjaro National Park comprises the whole of Mount Kilimanjaro above the tree line and six forest corridors stretching down Kilimanjaro Region, a region in Tanzania Kilimanjaro (ward), a ward in the Moshi Urban district of Kilimanjaro Region, Tanzania Kilimanjaro International Airport in Tanzania a Tanzanian beer, see Beer in Africa#Eastern Africa a Tanzanite jewellery brand owned by F. Hinds Music Killamanjaro, a Jamaican reggae sound system Albums Kilimanjaro, an album by German artist Superpitcher Kilimanjaro (The Rippingtons album), a 1988 album by The Rippingtons Kilimanjaro (The Teardrop Explodes album), an album by The Teardrop Explodes Songs "Kilimanjaro", song by The Del Vikings 1962 "Kilimanjaro", song by Manhattan Brothers 1955 "Kilimanjaro", song by The Teardrop Explodes 1980 "Kilimanjaro", song by Juluka 1984 "Kilimandjaro" (song), a 1966 French-language song by French singer Pascal Danel "Kilimanjaro" (song), a 2010 song by A.R. Rahman from the film Enthiran "Kilimanjaro", a song by KSI from the 2016 extended play Keep Up Film Kilimanjaro (film), a 2013 American film Nigeria Kilimanjaro restaurant, a fast-food chain in Nigeria. See also The Snows of Kilimanjaro (disambiguation)Mount Kilimanjaro () is a dormant volcano located in Kilimanjaro Region of Tanzania. It has three volcanic cones: Kibo, Mawenzi, and Shira. It is the highest mountain in Africa and the highest single free-standing mountain above sea level in the world: above sea level and about above its plateau base. It is the highest volcano in Africa and the Eastern Hemisphere. Kilimanjaro is the fourth most topographically prominent peak on Earth. It is part of Kilimanjaro National Park and is a major hiking and climbing destination. Because of its shrinking glaciers and ice fields, which are projected to disappear between 2025 and 2035, it has been the subject of many scientific studies. Toponymy The origin of the name Kilimanjaro is not known, but a number of theories exist. European explorers had adopted the name by 1860 and reported that Kilimanjaro was the mountain's Kiswahili name. The 1907 edition of The Nuttall Encyclopædia also records the name of the mountain as Kilima-Njaro. Johann Ludwig Krapf wrote in 1860 that Swahilis along the coast called the mountain Kilimanjaro. Although he did not offer any support, he claimed that Kilimanjaro meant either mountain of greatness or mountain of caravans. Under the latter meaning, kilima meant mountain and jaro meant caravans. Jim Thompson claimed in 1885, again without support, that the term Kilima-Njaro "has generally been understood to mean" the mountain (kilima) of greatness (njaro). He also suggested "though not improbably it may mean" the white mountain. Njaro is an ancient Kiswahili word for shining. Similarly, Krapf wrote that a
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Who was the man behind The Chipmunks??
+A: David Seville
+
+Q: Which Lloyd Webber musical premiered in the US on 10th December 1993??
+A: Sunset Boulevard
+
+Q: Who was the next British Prime Minister after Arthur Balfour??
+A: Campbell-Bannerman
+
+Q: Who had a 70s No 1 hit with Kiss You All Over??
+A: Exile
+
+Q: What claimed the life of singer Kathleen Ferrier??
+A: Cancer
+
+Q: Which volcano in Tanzania is the highest mountain in Africa?
+A: 
+==================================================
+
+==================================================
+PROMPT #14:
+==================================================
+of the Libyan Draft Constitutional Charter for the Transitional Stage: The national flag shall have the following shape and dimensions: Its length shall be double its width, its shall be divided into three parallel coloured stripes, the uppermost being red, the centre black and lowest green, the black stripe shall be equal in area to the other two stripes together and shall bear in its centre a white crescent, between the two extremities of which there shall be a five-pointed white star. On 10 March 2011, France was the first country to recognise the council as the official government of Libya, as well as the first to allow the Libyan embassy staff to raise the flag. On 21 March, the flag was flown by the Permanent Mission of Libya to the United Nations and appeared on their official website, and thereafter in late August by the Arab League and by Libya's own telecommunications authority, the Libya Telecom & Technology, on its own website. In the following months many other Libyan embassies replaced the green flag of Gaddafi with the tricolour flag. This original flag of Libya is now the only flag used by the United Nations to represent Libya, according to the following UN statement: "Following the adoption by the General Assembly of resolution 66/1, the Permanent Mission of Libya to the United Nations formally notified the United Nations of a Declaration by the National Transitional Council of 3 August 2011 changing the official name of the Libyan Arab Jamahiriya to 'Libya' as well as athe flag's colours and symbols. According to Omar Faiek Shennib, "red was selected for the blood sacrificed for the freedom of Libya, black to remember the dark days that Libyans lived under the occupation of the Italians and green to represent its primary wealth, agriculture, [Libya once being referred to as the 'agricultural basket' or 'breadbasket' of the Ottoman Empire] and the future prosperity of the country. The star and crescent were placed within the black central strip of the flag as a reference to the Senussi flag and the role of King Idris in leading the country to independence". The flag's colours also echo the colours of the flags of the three regions of Libya: Fezzan (red), Cyrenaica (black), and Tripolitania (green). Under Muammar Gaddafi's dictatorship, Libya had a red-white-black flag from 1969 to 1977, and it was replaced by the all-green flag from 1977 to 2011, during which it was the only flag in the world to have one color and no design. During the Libyan Civil War against the rule of Muammar Gaddafi, the 1951–69 flag – as well as various makeshift versions without the crescent and star symbol, or without the green stripe – came back into use in areas held by the Libyan opposition and by protesters at several Libyan diplomatic missions abroad. The National Transitional Council, formed on 27 February 2011, adopted the flag previously used in the Kingdom of Libya between 1951 and 1969 as the "emblem of the Libyan Republic". The flag was officially defined in article threeThe flag of Libya from 1977 to 2011 was used by the Socialist People's Libyan Arab Jamahiriya from 1977 to 1986 and later the Great Socialist People's Libyan Arab Jamahiriya until 2011. The design is a green field in 1:2 ratio and was considered the only solid colour national flag in the world during its time. In 2011, after the collapse of Gaddafi's government, the 1951–1969 flag from the Kingdom of Libya was re-adopted but the flag introduced by Gaddafi remained in use by Pro-Gaddafists and Gaddafi loyalists. Before 1977, the country was called the Libyan Arab Republic from 1969 to 1977 and used a red-white-black flag similar to most traditional Arab national flags bearing a resemblance to the modern flag of Yemen. in 1977 after the Egyptian-Libyan War, the blank green flag was introduced to replace the red-white-black flag to avoid similarities with Egypt. History of Libya under Muammar Gaddafi Flags introduced in 1977 1977 establishments in Libya 2011 disestablishments in Libya
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Who was the man behind The Chipmunks??
+A: David Seville
+
+Q: Which Lloyd Webber musical premiered in the US on 10th December 1993??
+A: Sunset Boulevard
+
+Q: Who was the next British Prime Minister after Arthur Balfour??
+A: Campbell-Bannerman
+
+Q: Who had a 70s No 1 hit with Kiss You All Over??
+A: Exile
+
+Q: What claimed the life of singer Kathleen Ferrier??
+A: Cancer
+
+Q: The flag of Libya is a plain rectangle of which color?
+A: 
+==================================================
+
+==================================================
+PROMPT #15:
+==================================================
+la Francophonie. Places of worship Niger being a predominantly Muslim country, mosques are the most common places of worship, with the Grande Mosquée being the largest in the city. There are also various Christian churches, most notably Our Lady of Perpetual Help Cathedral and the Cathedral de Maourey. Governance Administration Niamey makes up a special capital district of Niger, which is surrounded by the Region of Tillabéri. The city of Niamey itself is governed as an autonomous first-level administrative block, the Niamey Urban Community (Fr. Communauté Urbaine de Niamey, or CUN). It includes five Urban Communes, divided into 44 "Districts" and 99 "Quartiers", including formerly independent towns. It is a co-equal first division subdivision with the seven Regions of Niger. The Niamey Urban Community includes an administration and Governor appointed by national leaders. Like the rest of Niger, Niamey has seen a decentralisation of governance since 2000. Government Ordinance n°2010–56 and Presidential Decree n°2010-679 of September 2010 mandated an elected City Council for the city of Niamey, subsumed under the CUN. This excludes some outlying areas of the CUN. Forty-five councillors are popularly elected and in turn elect the Mayor of the City of Niamey. In July 2011, the first Mayor under the new system, Oumarou Dogari Moumouni, was installed by the Governor of the CUN Mrs. Aïchatou Boulama Kané and the City Council. The City Council and Mayor have limited roles compared to the CUN Governor. Niamey has a third layer of government in the Commune system. Each Commune elects its own council, and outsidein Niger Niamey NigerNiamey () is the capital and largest city of Niger. Niamey lies on the Niger River, primarily situated on the east bank. Niamey's population was counted as 1,026,848 as of the 2012 census. As of 2017, population projections show the capital district growing at a slower rate than the country as a whole, which has the world's highest fertility rate. The city is located in a pearl millet growing region, while manufacturing industries include bricks, ceramic goods, cement, and weaving. History Niamey was probably founded in the 18th century and originated as a cluster of small villages (Gaweye, Kalley, Maourey, Zongo and Foulani Koira). Niamey was of little importance until the French developed it as a colonial centre in the late 1890s. The town, then with an estimated population of some 1,800, was chosen as the capital of the newly created Military Territory of Niger in 1905, however, the capital was shifted to the more established city of Zinder in 1912. Zinder's proximity to the Nigerian border and distance from French-controlled ports prompted the French to move the capital back to Niamey in 1926, by which time the city had some 3,000 inhabitants. A series of devastating droughts prompted significant population growth during this period, and by 1945 the population was about 8,000. Prior to 1926-27 the Upper Volta-Niger border ran along the Niger river, meaning that Niamey lay directly on the boundary. At the time of independence in 1960 the population had grown to around 30,000. The period from 1970 to 1988 was one in
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Who was the man behind The Chipmunks??
+A: David Seville
+
+Q: Which Lloyd Webber musical premiered in the US on 10th December 1993??
+A: Sunset Boulevard
+
+Q: Who was the next British Prime Minister after Arthur Balfour??
+A: Campbell-Bannerman
+
+Q: Who had a 70s No 1 hit with Kiss You All Over??
+A: Exile
+
+Q: What claimed the life of singer Kathleen Ferrier??
+A: Cancer
+
+Q: Of which African country is Niamey the capital?
+A: 
+==================================================
+
+==================================================
+PROMPT #16:
+==================================================
+James Walter McCord Jr. (January 26, 1924 – June 15, 2017) was an American CIA officer, later head of security for President Richard Nixon's 1972 reelection campaign. He was involved as an electronics expert in the burglaries which precipitated the Watergate scandal. Career McCord was born in Waurika, Oklahoma. He served as a bombardier with the rank of second lieutenant in the Army Air Forces during World War II. He briefly attended Baylor University before receiving a B.B.A. from the University of Texas at Austin in 1949. In 1965, he received an M.S. in international affairs from George Washington University. After beginning his career at the Federal Bureau of Investigation (FBI), McCord worked for the Central Intelligence Agency (CIA), ultimately ascending to the GS-15 directorship of the Agency's Office of Security. For a period of time, he was in charge of physical security at the Agency's Langley headquarters. L. Fletcher Prouty, a former colonel in the United States Air Force, claimed then-Director of Central Intelligence Allen Dulles introduced McCord to him as "my top man.". In 1961, under his direction, a counter-intelligence program was launched against the Fair Play for Cuba Committee. He also held the rank of lieutenant colonel in the United States Air Force Reserve. Watergate scandal Shortly after resigning from the CIA, McCord was interviewed and then hired by Jack Caulfield in January 1972 "for strict, solely defensive security work at the Republican National Committee (RNC) and the Committee to Re-Elect the President (CRP)." Some of the money from this contract came fromadministration as assistant director of the Bureau of the Budget, devoting most of his time to Defense matters. In 1971, President Nixon appointed Schlesinger a member of the Atomic Energy Commission (AEC) and designated him as chairman. Serving in this position for about a year and a half, Schlesinger instituted extensive organizational and management changes in an effort to improve the AEC's regulatory performance. CIA Director Schlesinger was CIA Director from February 2, 1973, to July 2, 1973. He was succeeded by William Colby. Schlesinger was extremely unpopular with CIA staff, as he reduced CIA staff by 7%, and was considered a Nixon loyalist seeking to make the agency more obedient to Nixon. He had a CCTV camera installed near his official portrait at the CIA headquarters in Langley, Va., as it was believed that vandalism of the portrait by disgruntled staff was likely. Secretary of Defense (1973–1975) Schlesinger left the CIA to become Secretary of Defense on July 2, aged 44. As a university professor, researcher at Rand, and government official in three agencies, he had acquired an impressive resume in national security affairs. Nuclear strategy Shortly after assuming office, Schlesinger outlined the basic objectives that would guide his administration: maintain a "strong defense establishment"; "assure the military balance so necessary to deterrence and a more enduring peace"; obtain for members of the military "the respect, dignity and support that are their due"; assume "an . . . obligation to use our citizens' resources wisely"; and "become increasingly competitive with potential adversaries.... [W]e must nota conventional North Vietnamese assault in 1975. The CORDS model and its approach influenced U.S. strategy and thinking on counterinsurgency in the 2000s in Iraq and Afghanistan. CIA HQ: Director Colby returned to Washington in July 1971 and became executive director of CIA. After long-time DCI Richard Helms was dismissed by President Nixon in 1973, James Schlesinger assumed the helm at the Agency. A strong believer in reform of the CIA and the intelligence community more broadly, Schlesinger had written a 1971 Bureau of the Budget report outlining his views on the subject. Colby, who had had a somewhat unorthodox career in the CIA focused on political action and counterinsurgency, agreed with Schlesinger's reformist approach. Schlesinger appointed him head of the clandestine branch in early 1973. When Nixon reshuffled his agency heads and made Schlesinger secretary of defense, Colby emerged as a natural candidate for DCI—apparently on the basis of the recommendation that he was a professional who would not make waves. Colby was known as a media-friendly CIA director. His tenure as DCI, which lasted two and a half tumultuous years, was overshadowed by the Church and Pike congressional investigations into alleged U.S. intelligence malfeasance over the preceding 25 years, including 1975, the so-called Year of Intelligence. Colby's time as DCI was also eventful on the world stage. Shortly after he assumed leadership, the Yom Kippur War broke out, an event that surprised not only the American intelligence agencies but also the Israelis. This intelligence surprise reportedly affected Colby's credibility with the Nixon administration. Colby
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Who was the man behind The Chipmunks??
+A: David Seville
+
+Q: Which Lloyd Webber musical premiered in the US on 10th December 1993??
+A: Sunset Boulevard
+
+Q: Who was the next British Prime Minister after Arthur Balfour??
+A: Campbell-Bannerman
+
+Q: Who had a 70s No 1 hit with Kiss You All Over??
+A: Exile
+
+Q: What claimed the life of singer Kathleen Ferrier??
+A: Cancer
+
+Q: Who was the director of the CIA from 1976-81?
+A: 
+==================================================
+
+==================================================
+PROMPT #17:
+==================================================
+"On the Street Where You Live" is a song with music by Frederick Loewe and lyrics by Alan Jay Lerner from the 1956 Broadway musical My Fair Lady. It is sung in the musical by the character Freddy Eynsford-Hill, who was portrayed by John Michael King in the original production. In the 1964 film version, it was sung by Bill Shirley, dubbing for actor Jeremy Brett. Recorded versions The most popular single of the song was recorded by Vic Damone in 1956 for Columbia Records. It reached No. 4 on the Billboard chart and No. 6 on Cashbox magazine's chart. It was a No. 1 hit in the UK Singles Chart in 1958. Eddie Fisher also had a top 20 Billboard hit with the song in 1956, reaching No. 18. Lawrence Welk and His Orchestra released a version that went to No. 96 in 1956. Andy Williams' recording appeared in the Billboard top 40 in 1964, reaching No. 3 on the adult contemporary chart and No. 28 on the Billboard Hot 100. The song has been recorded by a wide variety of other performers, including Ray Conniff and Bing Crosby, who recorded the song in 1956 for use on his radio show and it was subsequently included in the boxed set The Bing Crosby CBS Radio Recordings (1954–56) issued by Mosaic Records (catalog MD7-245) in 2009, Lawrence Welk (whose band also performed it on his weekly TV series numerous times), Shirley Horn, Doris Day, George Shearing, Frank Chacksfield, Alfie Boe, Bobby Darin, Dean Martin, Mario Lanza,The Times praised it as "Alan Jay Lerner's terrific autobiography". The Street Where I Live was reissued in 1989 by Columbus Books and in 1994 by the Da Capo Press. In 2000, BBC radio broadcast a serialization of the book, read by Henry Goodman, which The Times called "one of the delights of the evening schedule". References Sources Non-fiction books about musical theatre"On the Street Where You Live" is a song from the 1956 Broadway musical My Fair Lady. On the Street Where You Live may also refer to: On the Street Where You Live (TV series), an Irish documentary television series On The Street Where You Live, a 2001 novel by Mary Higgins Clark
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Who was the man behind The Chipmunks??
+A: David Seville
+
+Q: Which Lloyd Webber musical premiered in the US on 10th December 1993??
+A: Sunset Boulevard
+
+Q: Who was the next British Prime Minister after Arthur Balfour??
+A: Campbell-Bannerman
+
+Q: Who had a 70s No 1 hit with Kiss You All Over??
+A: Exile
+
+Q: What claimed the life of singer Kathleen Ferrier??
+A: Cancer
+
+Q: Which musical featured the song The Street Where You Live?
+A: 
+==================================================
+
+==================================================
+PROMPT #18:
+==================================================
+engineers were ordered to end construction work. The Allies were unaware of this and mounted further attacks on the site as part of the United States Army Air Forces experimental Operation Aphrodite, involving radio-controlled B-24 Liberators packed with explosives. Two such attacks were mounted but failed; in the second such attack, on 12 August, Lt Joseph P. Kennedy, Jr. – the elder brother of future US President John F. Kennedy – was killed when the drone aircraft exploded prematurely. By the end of the bombing campaign, over 4,100 tons of bombs had been dropped on Mimoyecques, more than on any other V-weapons site. The Mimoyecques site was never formally abandoned, but German forces left it at the start of September 1944 as the Allies advanced northeast from Normandy towards the Pas de Calais. It was captured on 5 September by the Canadian 3rd Infantry Division. Subsequent investigations and attempted demolition In September 1944, Duncan Sandys ordered the constitution of a Technical Inter-Services Mission under Colonel T.R.B. Sanders. It was given the task of investigating the V-weapons sites at Mimoyecques, Siracourt, Watten, and Wizernes, collectively known to the Allies as the "Heavy Crossbow" sites. Sanders' report was submitted to the War Cabinet on 19 March 1945. Even at this stage the true purpose of the site was unclear. Claims that it had been intended to be used for "electro-magnetic projectors" (railguns), firing huge shells at London, were debunked by Lord Cherwell, Winston Churchill's scientific adviser, who calculated that it would take sixty times the output of Battersearesearched at a facility in Peenemünde along with the V-1 flying bomb. The V-2's first target was Paris on 8 September 1944. The program while advanced proved to be an impediment to the war economy. The large capital investment was not repaid in military effectiveness. The rockets were built at an underground factory at Mittelwerk. Labor to build the A4 rockets came from the Mittelbau-Dora concentration camp. Of the 60,000 people who ended up at the camp 20,000 died, due to the appalling conditions. On 14 April 1944, Speer lost control of Organisation Todt to his Deputy, Franz Xaver Dorsch. He opposed the assassination attempt against Hitler on 20 July 1944. He was not involved in the plot, and played a minor role in the regime's efforts to regain control over Berlin after Hitler survived. After the plot Speer's rivals attacked some of his closest allies and his management system fell out of favor with radicals in the party. He lost yet more authority. Defeat of Nazi Germany Losses of territory and a dramatic expansion of the Allied strategic bombing campaign caused the collapse of the German economy from late 1944. Air attacks on the transport network were particularly effective, as they cut the main centres of production off from essential coal supplies. In January 1945, Speer told Goebbels that armaments production could be sustained for at least a year. However, he concluded that the war was lost after Soviet forces captured the important Silesian industrial region later that month. Nevertheless, Speer believed that Germany shouldof 1944 the Allies continued their gains in the Mediterranean Theatre and massed men and materiel for a European invasion along the French channel coastline. The conspirators began to organize for another attempt to assassinate Hitler and take over both German civil government and its military. The von Stauffenberg bomb attempt and aftermath By the summer of 1944 unrest in the German military and diplomatic ranks was widespread. The Allied landing at Normandy in June and failed German response raised the specter of doom among the upper ranks even of German field marshals. The Schwarze Kapelle responded by organizing a deadly attempt on Hitler's life at his Wolf's Lair compound in East Prussia. Undertaken by an aristocratic member of a hereditarily military family, Colonel Claus von Stauffenberg, the July 20 Plot nearly succeeded. Although surrounded by fatalities from the bomb Hitler escaped with a concussion and various injuries. In the aftermath he was determined to get vengeance upon the plotters. The Gestapo rounded up the members of the Schwarze Kapelle and many, many more it believed were either implicated in or sympathetic to it; according to its records it put 7,000 of them to death. Stauffenberg and three others were summarily shot that night. Most of the conspirators were put on trial in the Volksgerichtshof (People's Court) between August 1944 to February 1945. Many were executed the day after their convictions by hanging from meat hooks at Plötzensee Prison. Architect of the 1943 bomb plot on Hitler's plane Fabian von Schlabrendorff only escaped death because an
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Who was the man behind The Chipmunks??
+A: David Seville
+
+Q: Which Lloyd Webber musical premiered in the US on 10th December 1993??
+A: Sunset Boulevard
+
+Q: Who was the next British Prime Minister after Arthur Balfour??
+A: Campbell-Bannerman
+
+Q: Who had a 70s No 1 hit with Kiss You All Over??
+A: Exile
+
+Q: What claimed the life of singer Kathleen Ferrier??
+A: Cancer
+
+Q: "Who was the target of the failed ""Bomb Plot"" of 1944?"
+A: 
+==================================================
+
+==================================================
+PROMPT #19:
+==================================================
+propelling him into the first rank of international superstars. The album contained the number-one hit "All Night Long", a Caribbean-flavored dance number that was promoted by a colorful music video produced by former Monkee Michael Nesmith. In 1984, he performed "All Night Long" at the ending ceremony of the XXIII Olympic Games in Los Angeles. Several more Top 10 hits followed, the most successful of which was the ballad "Hello" (1984), a sentimental love song that showed how far he had moved from his R&B roots. Richie had three more top ten hits in 1984, "Stuck on You" (No. 3), "Running with the Night" (No. 7) and "Penny Lover" (No. 8), as well as writing and producing "Missing You" for former labelmate and duet partner Diana Ross (No. 10 Pop, No. 1 R&B). In 1985, he wrote and performed "Say You, Say Me" for the film White Nights. The song won an Academy Award and reached No. 1 on the U.S. charts, staying there for four weeks, making it the number-two song of 1986 according to Billboards Year-End Hot 100 chart, behind the charity single "That's What Friends Are For" by Dionne and Friends. He also collaborated with Michael Jackson on the charity single "We Are the World" by USA for Africa, another number-one hit. In 1986, Richie released Dancing on the Ceiling, his last widely popular album, which produced a run of five US and UK hits, "Say You, Say Me" (U.S. No. 1), "Dancing on the Ceiling" (U.S. No. 2), "Love Will Conquer All"top 20 US R&B chart hit in 1972. Their first few recordings were released on Buddah Records, including "Hold Back the Night", which was a hit on the Billboard R&B chart in 1973, before a re-release saw it climb in the UK two years later. Several R&B hits followed during a stay with Philadelphia International subsidiary Golden Fleece (run by Baker-Harris-Young) before they signed to Atlantic Records. Their single "Disco Inferno" (1976), which was included on the Grammy Award-winning Saturday Night Fever: The Original Movie Sound Track in 1977, reached No. 11 on the Billboard Hot 100 chart in May 1978. Other major hits included "Hold Back the Night" (1975) (UK No. 5) and "That's Where the Happy People Go" (1976). In late 1977, the Trammps released the song "The Night the Lights Went Out" to commemorate the electrical blackout that affected New York City on July 13–14, 1977. Their signature song "Disco Inferno" has been covered by Tina Turner and Cyndi Lauper. In addition, Graham Parker covered "Hold Back the Night" on "The Pink Parker EP" in 1977, and reached No. 24 in the UK Singles Chart, and top 60 in the US. In 2021, "Disco Inferno" was certified Silver by the British Phonographic Industry, together with "Can We Come Together" (from the album Where the Happy People Go). Dissolution and aftermath On September 19, 2005, the group's "Disco Inferno" was inducted into the Dance Music Hall of Fame at a ceremony held in New York. The song was part-written by Ron Kersey, a producer-arranger"Hold On to the Nights" is a power ballad written and performed by American rock singer/songwriter/musician Richard Marx. This was the fourth and final single released from his self-titled debut album, and his first to reach number one on the US Billboard Hot 100 chart. The song has been re-released on numerous albums and is included on Marx's live performance DVD A Night Out with Friends (2012). Release "Hold On to the Nights" reached the Billboard Hot 100 number 1 position on July 23, 1988, preventing Def Leppard's "Pour Some Sugar on Me" from reaching the top spot that same week. The song was on the chart for twenty-one weeks, and left the chart at number 91. The song also reached at number three on the Billboard Adult Contemporary chart. Chart performance Charts Personnel Richard Marx – vocals, keyboards, acoustic piano Michael Landau – guitars Patrick O'Hearn – bass Tris Imboden – drums Paulinho da Costa – percussion Other performances Marx appeared as lounge singer/piano player Buddy Daquiri in the "Poison Fire Teats Universe" episode of the TV series Life in Pieces in 2017, in which he played the song on the piano while whistling. References 1987 songs 1988 singles Richard Marx songs Billboard Hot 100 number-one singles Songs written by Richard Marx Pop ballads Rock ballads EMI Records singles Songs about nights
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Who was the man behind The Chipmunks??
+A: David Seville
+
+Q: Which Lloyd Webber musical premiered in the US on 10th December 1993??
+A: Sunset Boulevard
+
+Q: Who was the next British Prime Minister after Arthur Balfour??
+A: Campbell-Bannerman
+
+Q: Who had a 70s No 1 hit with Kiss You All Over??
+A: Exile
+
+Q: What claimed the life of singer Kathleen Ferrier??
+A: Cancer
+
+Q: Who had an 80s No 1 hit with Hold On To The Nights?
+A: 
+==================================================
+
+==================================================
+PROMPT #20:
+==================================================
+Turner Classic Movies in November 2006 features directors Steven Spielberg, Clint Eastwood, and Martin Scorsese, who suggest that the string of classic films Ford directed during 1936 to 1941 was due in part to an intense six-month extramarital affair with Katharine Hepburn, the star of Mary of Scotland (1936), an Elizabethan costume drama. 1939–1941 Stagecoach (1939) was Ford's first western since 3 Bad Men in 1926, and it was his first with sound. Orson Welles claimed that he watched Stagecoach forty times in preparation for making Citizen Kane. It remains one of the most admired and imitated of all Hollywood movies, not least for its climactic stagecoach chase and the hair-raising horse-jumping scene, performed by the stuntman Yakima Canutt. The Dudley Nichols–Ben Hecht screenplay was based on an Ernest Haycox story that Ford had spotted in Collier's magazine and he purchased the screen rights for just $2500. Production chief Walter Wanger urged Ford to hire Gary Cooper and Marlene Dietrich for the lead roles, but eventually accepted Ford's decision to cast Claire Trevor as Dallas and a virtual unknown, his friend John Wayne, as Ringo; Wanger reportedly had little further influence over the production. In making Stagecoach, Ford faced entrenched industry prejudice about the now-hackneyed genre which he had helped to make so popular. Although low-budget western features and serials were still being churned out in large numbers by "Poverty Row" studios, the genre had fallen out of favor with the big studios during the 1930s and they were regarded as B-grade "pulp" movies at best.Stagecoach is a 1986 American made-for-television Western action drama film and remake of the classic 1939 film Stagecoach, directed by Ted Post and starring Kris Kristofferson as the Ringo Kid, the role originally played by John Wayne. Willie Nelson portrays famous gunslinger and dentist Doc Holliday, Johnny Cash portrays Marshal Curly Wilcox and Waylon Jennings plays the gambler Hatfield. The four main stars of the film (Nelson, Kristofferson, Cash and Jennings) were associated as members of the country music supergroup The Highwaymen. The supporting cast features Elizabeth Ashley, Anthony Newley, Tony Franciosa, Mary Crosby, June Carter Cash and Jessi Colter. Plot In 1880, a group of strangers boards the east-bound stagecoach from Tonto, Arizona Territory, to Lordsburg, New Mexico Territory. The travelers seem ordinary, but many have secrets from which they are running. Among them are Dallas, a prostitute, who is being driven out of town; an alcoholic dentist, Doc Holliday; pregnant Lucy Mallory, who is meeting her cavalry officer husband; and whiskey salesman Trevor Peacock. As the stage sets out, U.S. Cavalry Lieutenant Blanchard announces that Geronimo and his Apaches are on the warpath; his small troop will provide an escort to Dry Fork. Cast Willie Nelson as Doc Holliday Kris Kristofferson as Ringo / Ringo Kid / Bill Williams Johnny Cash as Marshal Curly Wilcox Waylon Jennings as Hatfield (Gambler) John Schneider as Buck (Overland Stage Driver) Elizabeth Ashley as Dallas Anthony Newley as Trevor Peacock (Old John's Whiskey Salesman) Tony Franciosa as Henry Gatewood (Tonto Banker) Merritt Butrick as Lieutenant Blanchard Mary CrosbyStagecoach is a 1939 American Western film directed by John Ford and starring Claire Trevor and John Wayne in his breakthrough role. The screenplay by Dudley Nichols is an adaptation of "The Stage to Lordsburg", a 1937 short story by Ernest Haycox. The film follows a group of strangers riding on a stagecoach through dangerous Apache territory. The film has long been recognized as an important work that transcends the Western genre. Philosopher Robert B. Pippin has observed that both the collection of characters and their journey "are archetypal rather than merely individual" and that the film is a "mythic representation of the American aspiration toward a form of politically meaningful equality." In 1995, the film was deemed "culturally, historically, or aesthetically significant" by the United States Library of Congress and selected for preservation in their National Film Registry. Still, Stagecoach has not avoided controversy. Like most Westerns of the era, its depiction of Native Americans as simplistic savages has been criticized. Stagecoach was the first of many Westerns that Ford shot in Monument Valley, on the Arizona–Utah border in the American Southwest. Many of the movies Ford shot there also starred John Wayne. Scenes from Stagecoach, including a sequence introducing John Wayne's character the Ringo Kid, blended shots of Monument Valley with shots filmed on the Iverson Movie Ranch in Chatsworth, California, RKO Encino Movie Ranch, and other locations. Geographic incongruities are visible throughout the film, including the closing scene where Ringo (Wayne) and Dallas (Trevor) depart Lordsburg, in southwestern New Mexico, by way of
+Think hard, but answer shortly and concisely. Only give direct answers to the questions. No additional explanations. Directly answer these questions:
+Q: Who was the man behind The Chipmunks??
+A: David Seville
+
+Q: Which Lloyd Webber musical premiered in the US on 10th December 1993??
+A: Sunset Boulevard
+
+Q: Who was the next British Prime Minister after Arthur Balfour??
+A: Campbell-Bannerman
+
+Q: Who had a 70s No 1 hit with Kiss You All Over??
+A: Exile
+
+Q: What claimed the life of singer Kathleen Ferrier??
+A: Cancer
+
+Q: Who directed the classic 30s western Stagecoach?
+A: 
+==================================================
+

benchmarks/diskann_vs_hnsw_speed_comparison.py

@@ -0,0 +1,286 @@
+#!/usr/bin/env python3
+"""
+DiskANN vs HNSW Search Performance Comparison
+
+This benchmark compares search performance between DiskANN and HNSW backends:
+- DiskANN: With graph partitioning enabled (is_recompute=True)
+- HNSW: With recompute enabled (is_recompute=True)
+- Tests performance across different dataset sizes
+- Measures search latency, recall, and index size
+"""
+
+import gc
+import multiprocessing as mp
+import tempfile
+import time
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+# Prefer 'fork' start method to avoid POSIX semaphore leaks on macOS
+try:
+    mp.set_start_method("fork", force=True)
+except Exception:
+    pass
+
+
+def create_test_texts(n_docs: int) -> list[str]:
+    """Create synthetic test documents for benchmarking."""
+    np.random.seed(42)
+    topics = [
+        "machine learning and artificial intelligence",
+        "natural language processing and text analysis",
+        "computer vision and image recognition",
+        "data science and statistical analysis",
+        "deep learning and neural networks",
+        "information retrieval and search engines",
+        "database systems and data management",
+        "software engineering and programming",
+        "cybersecurity and network protection",
+        "cloud computing and distributed systems",
+    ]
+
+    texts = []
+    for i in range(n_docs):
+        topic = topics[i % len(topics)]
+        variation = np.random.randint(1, 100)
+        text = (
+            f"This is document {i} about {topic}. Content variation {variation}. "
+            f"Additional information about {topic} with details and examples. "
+            f"Technical discussion of {topic} including implementation aspects."
+        )
+        texts.append(text)
+
+    return texts
+
+
+def benchmark_backend(
+    backend_name: str, texts: list[str], test_queries: list[str], backend_kwargs: dict[str, Any]
+) -> dict[str, float]:
+    """Benchmark a specific backend with the given configuration."""
+    from leann.api import LeannBuilder, LeannSearcher
+
+    print(f"\n🔧 Testing {backend_name.upper()} backend...")
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+        index_path = str(Path(temp_dir) / f"benchmark_{backend_name}.leann")
+
+        # Build index
+        print(f"📦 Building {backend_name} index with {len(texts)} documents...")
+        start_time = time.time()
+
+        builder = LeannBuilder(
+            backend_name=backend_name,
+            embedding_model="facebook/contriever",
+            embedding_mode="sentence-transformers",
+            **backend_kwargs,
+        )
+
+        for text in texts:
+            builder.add_text(text)
+
+        builder.build_index(index_path)
+        build_time = time.time() - start_time
+
+        # Measure index size
+        index_dir = Path(index_path).parent
+        index_files = list(index_dir.glob(f"{Path(index_path).stem}.*"))
+        total_size = sum(f.stat().st_size for f in index_files if f.is_file())
+        size_mb = total_size / (1024 * 1024)
+
+        print(f"   ✅ Build completed in {build_time:.2f}s, index size: {size_mb:.1f}MB")
+
+        # Search benchmark
+        print("🔍 Running search benchmark...")
+        searcher = LeannSearcher(index_path)
+
+        search_times = []
+        all_results = []
+
+        for query in test_queries:
+            start_time = time.time()
+            results = searcher.search(query, top_k=5)
+            search_time = time.time() - start_time
+            search_times.append(search_time)
+            all_results.append(results)
+
+        avg_search_time = np.mean(search_times) * 1000  # Convert to ms
+        print(f"   ✅ Average search time: {avg_search_time:.1f}ms")
+
+        # Check for valid scores (detect -inf issues)
+        all_scores = [
+            result.score
+            for results in all_results
+            for result in results
+            if result.score is not None
+        ]
+        valid_scores = [
+            score for score in all_scores if score != float("-inf") and score != float("inf")
+        ]
+        score_validity_rate = len(valid_scores) / len(all_scores) if all_scores else 0
+
+        # Clean up (ensure embedding server shutdown and object GC)
+        try:
+            if hasattr(searcher, "cleanup"):
+                searcher.cleanup()
+            del searcher
+            del builder
+            gc.collect()
+        except Exception as e:
+            print(f"⚠️  Warning: Resource cleanup error: {e}")
+
+        return {
+            "build_time": build_time,
+            "avg_search_time_ms": avg_search_time,
+            "index_size_mb": size_mb,
+            "score_validity_rate": score_validity_rate,
+        }
+
+
+def run_comparison(n_docs: int = 500, n_queries: int = 10):
+    """Run performance comparison between DiskANN and HNSW."""
+    print("🚀 Starting DiskANN vs HNSW Performance Comparison")
+    print(f"📊 Dataset: {n_docs} documents, {n_queries} test queries")
+
+    # Create test data
+    texts = create_test_texts(n_docs)
+    test_queries = [
+        "machine learning algorithms",
+        "natural language processing",
+        "computer vision techniques",
+        "data analysis methods",
+        "neural network architectures",
+        "database query optimization",
+        "software development practices",
+        "security vulnerabilities",
+        "cloud infrastructure",
+        "distributed computing",
+    ][:n_queries]
+
+    # HNSW benchmark
+    hnsw_results = benchmark_backend(
+        backend_name="hnsw",
+        texts=texts,
+        test_queries=test_queries,
+        backend_kwargs={
+            "is_recompute": True,  # Enable recompute for fair comparison
+            "M": 16,
+            "efConstruction": 200,
+        },
+    )
+
+    # DiskANN benchmark
+    diskann_results = benchmark_backend(
+        backend_name="diskann",
+        texts=texts,
+        test_queries=test_queries,
+        backend_kwargs={
+            "is_recompute": True,  # Enable graph partitioning
+            "num_neighbors": 32,
+            "search_list_size": 50,
+        },
+    )
+
+    # Performance comparison
+    print("\n📈 Performance Comparison Results")
+    print(f"{'=' * 60}")
+    print(f"{'Metric':<25} {'HNSW':<15} {'DiskANN':<15} {'Speedup':<10}")
+    print(f"{'-' * 60}")
+
+    # Build time comparison
+    build_speedup = hnsw_results["build_time"] / diskann_results["build_time"]
+    print(
+        f"{'Build Time (s)':<25} {hnsw_results['build_time']:<15.2f} {diskann_results['build_time']:<15.2f} {build_speedup:<10.2f}x"
+    )
+
+    # Search time comparison
+    search_speedup = hnsw_results["avg_search_time_ms"] / diskann_results["avg_search_time_ms"]
+    print(
+        f"{'Search Time (ms)':<25} {hnsw_results['avg_search_time_ms']:<15.1f} {diskann_results['avg_search_time_ms']:<15.1f} {search_speedup:<10.2f}x"
+    )
+
+    # Index size comparison
+    size_ratio = diskann_results["index_size_mb"] / hnsw_results["index_size_mb"]
+    print(
+        f"{'Index Size (MB)':<25} {hnsw_results['index_size_mb']:<15.1f} {diskann_results['index_size_mb']:<15.1f} {size_ratio:<10.2f}x"
+    )
+
+    # Score validity
+    print(
+        f"{'Score Validity (%)':<25} {hnsw_results['score_validity_rate'] * 100:<15.1f} {diskann_results['score_validity_rate'] * 100:<15.1f}"
+    )
+
+    print(f"{'=' * 60}")
+    print("\n🎯 Summary:")
+    if search_speedup > 1:
+        print(f"   DiskANN is {search_speedup:.2f}x faster than HNSW for search")
+    else:
+        print(f"   HNSW is {1 / search_speedup:.2f}x faster than DiskANN for search")
+
+    if size_ratio > 1:
+        print(f"   DiskANN uses {size_ratio:.2f}x more storage than HNSW")
+    else:
+        print(f"   DiskANN uses {1 / size_ratio:.2f}x less storage than HNSW")
+
+    print(
+        f"   Both backends achieved {min(hnsw_results['score_validity_rate'], diskann_results['score_validity_rate']) * 100:.1f}% score validity"
+    )
+
+
+if __name__ == "__main__":
+    import sys
+
+    try:
+        # Handle help request
+        if len(sys.argv) > 1 and sys.argv[1] in ["-h", "--help", "help"]:
+            print("DiskANN vs HNSW Performance Comparison")
+            print("=" * 50)
+            print(f"Usage: python {sys.argv[0]} [n_docs] [n_queries]")
+            print()
+            print("Arguments:")
+            print("  n_docs      Number of documents to index (default: 500)")
+            print("  n_queries   Number of test queries to run (default: 10)")
+            print()
+            print("Examples:")
+            print("  python benchmarks/diskann_vs_hnsw_speed_comparison.py")
+            print("  python benchmarks/diskann_vs_hnsw_speed_comparison.py 1000")
+            print("  python benchmarks/diskann_vs_hnsw_speed_comparison.py 2000 20")
+            sys.exit(0)
+
+        # Parse command line arguments
+        n_docs = int(sys.argv[1]) if len(sys.argv) > 1 else 500
+        n_queries = int(sys.argv[2]) if len(sys.argv) > 2 else 10
+
+        print("DiskANN vs HNSW Performance Comparison")
+        print("=" * 50)
+        print(f"Dataset: {n_docs} documents, {n_queries} queries")
+        print()
+
+        run_comparison(n_docs=n_docs, n_queries=n_queries)
+
+    except KeyboardInterrupt:
+        print("\n⚠️  Benchmark interrupted by user")
+        sys.exit(130)
+    except Exception as e:
+        print(f"\n❌ Benchmark failed: {e}")
+        sys.exit(1)
+    finally:
+        # Ensure clean exit (forceful to prevent rare hangs from atexit/threads)
+        try:
+            gc.collect()
+            print("\n🧹 Cleanup completed")
+            # Flush stdio to ensure message is visible before hard-exit
+            try:
+                import sys as _sys
+
+                _sys.stdout.flush()
+                _sys.stderr.flush()
+            except Exception:
+                pass
+        except Exception:
+            pass
+        # Use os._exit to bypass atexit handlers that may hang in rare cases
+        import os as _os
+
+        _os._exit(0)

benchmarks/faiss_only.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+"""Test only Faiss HNSW"""
+
+import os
+import sys
+import time
+
+import psutil
+
+
+def get_memory_usage():
+    process = psutil.Process()
+    return process.memory_info().rss / 1024 / 1024
+
+
+class MemoryTracker:
+    def __init__(self, name: str):
+        self.name = name
+        self.start_mem = get_memory_usage()
+        self.stages = []
+
+    def checkpoint(self, stage: str):
+        current_mem = get_memory_usage()
+        diff = current_mem - self.start_mem
+        print(f"[{self.name} - {stage}] Memory: {current_mem:.1f} MB (+{diff:.1f} MB)")
+        self.stages.append((stage, current_mem))
+        return current_mem
+
+    def summary(self):
+        peak_mem = max(mem for _, mem in self.stages)
+        print(f"Peak Memory: {peak_mem:.1f} MB")
+        return peak_mem
+
+
+def main():
+    try:
+        import faiss
+    except ImportError:
+        print("Faiss is not installed.")
+        print(
+            "Please install it with `uv pip install faiss-cpu` and you can  then run this script again"
+        )
+        sys.exit(1)
+
+    from llama_index.core import (
+        Settings,
+        SimpleDirectoryReader,
+        StorageContext,
+        VectorStoreIndex,
+    )
+    from llama_index.core.node_parser import SentenceSplitter
+    from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+    from llama_index.vector_stores.faiss import FaissVectorStore
+
+    tracker = MemoryTracker("Faiss HNSW")
+    tracker.checkpoint("Initial")
+
+    embed_model = HuggingFaceEmbedding(model_name="facebook/contriever")
+    Settings.embed_model = embed_model
+    tracker.checkpoint("After embedding model setup")
+
+    d = 768
+    faiss_index = faiss.IndexHNSWFlat(d, 32)
+    faiss_index.hnsw.efConstruction = 64
+    tracker.checkpoint("After Faiss index creation")
+
+    documents = SimpleDirectoryReader(
+        "data",
+        recursive=True,
+        encoding="utf-8",
+        required_exts=[".pdf", ".txt", ".md"],
+    ).load_data()
+    tracker.checkpoint("After document loading")
+
+    # Parse into chunks using the same splitter as LEANN
+    node_parser = SentenceSplitter(
+        chunk_size=256, chunk_overlap=20, separator=" ", paragraph_separator="\n\n"
+    )
+
+    tracker.checkpoint("After text splitter setup")
+
+    # Check if index already exists and try to load it
+    index_loaded = False
+    if os.path.exists("./storage_faiss"):
+        print("Loading existing Faiss HNSW index...")
+        try:
+            # Use the correct Faiss loading pattern from the example
+            vector_store = FaissVectorStore.from_persist_dir("./storage_faiss")
+            storage_context = StorageContext.from_defaults(
+                vector_store=vector_store, persist_dir="./storage_faiss"
+            )
+            from llama_index.core import load_index_from_storage
+
+            index = load_index_from_storage(storage_context=storage_context)
+            print("Index loaded from ./storage_faiss")
+            tracker.checkpoint("After loading existing index")
+            index_loaded = True
+        except Exception as e:
+            print(f"Failed to load existing index: {e}")
+            print("Cleaning up corrupted index and building new one...")
+            # Clean up corrupted index
+            import shutil
+
+            if os.path.exists("./storage_faiss"):
+                shutil.rmtree("./storage_faiss")
+
+    if not index_loaded:
+        print("Building new Faiss HNSW index...")
+
+        # Use the correct Faiss building pattern from the example
+        vector_store = FaissVectorStore(faiss_index=faiss_index)
+        storage_context = StorageContext.from_defaults(vector_store=vector_store)
+        index = VectorStoreIndex.from_documents(
+            documents, storage_context=storage_context, transformations=[node_parser]
+        )
+        tracker.checkpoint("After index building")
+
+        # Save index to disk using the correct pattern
+        index.storage_context.persist(persist_dir="./storage_faiss")
+        tracker.checkpoint("After index saving")
+
+    # Measure runtime memory overhead
+    print("\nMeasuring runtime memory overhead...")
+    runtime_start_mem = get_memory_usage()
+    print(f"Before load memory: {runtime_start_mem:.1f} MB")
+    tracker.checkpoint("Before load memory")
+
+    query_engine = index.as_query_engine(similarity_top_k=20)
+    queries = [
+        "什么是盘古大模型以及盘古开发过程中遇到了什么阴暗面,任务令一般在什么城市颁发",
+        "What is LEANN and how does it work?",
+        "华为诺亚方舟实验室的主要研究内容",
+    ]
+
+    for i, query in enumerate(queries):
+        start_time = time.time()
+        _ = query_engine.query(query)
+        query_time = time.time() - start_time
+        print(f"Query {i + 1} time: {query_time:.3f}s")
+        tracker.checkpoint(f"After query {i + 1}")
+
+    runtime_end_mem = get_memory_usage()
+    runtime_overhead = runtime_end_mem - runtime_start_mem
+
+    peak_memory = tracker.summary()
+    print(f"Peak Memory: {peak_memory:.1f} MB")
+    print(f"Runtime Memory Overhead: {runtime_overhead:.1f} MB")
+
+
+if __name__ == "__main__":
+    main()

benchmarks/generation_speed_bench.py

@@ -0,0 +1,114 @@
+import argparse
+import re
+import sys
+import time
+from pathlib import Path
+from statistics import mean
+
+from leann.chat import get_llm
+
+
+def parse_prompts_from_file(file_path: str) -> list[str]:
+    """
+    Parse a prompt dump file into individual prompt strings.
+
+    Splits by lines that look like: "PROMPT #<n>:".
+    Keeps the content from each marker up to the next marker (or EOF).
+    """
+    with open(file_path, "r", encoding="utf-8") as f:
+        text = f.read()
+
+    matches = list(re.finditer(r"^PROMPT\s+#\d+:\s*$", text, flags=re.MULTILINE))
+    if not matches:
+        # Fallback: try a more permissive pattern
+        matches = list(
+            re.finditer(r"^=+\nPROMPT\s+#\d+:\n=+\s*$", text, flags=re.MULTILINE)
+        )
+
+    prompts: list[str] = []
+    if not matches:
+        # No explicit markers; treat the whole file as a single prompt
+        return [text]
+
+    for i, m in enumerate(matches):
+        start = m.end()
+        end = matches[i + 1].start() if i + 1 < len(matches) else len(text)
+        block = text[start:end].strip()
+        # Reattach the marker line content above the block for full context
+        header_line_start = text.rfind("\n", 0, m.start()) + 1
+        header = text[header_line_start : m.end()].strip()
+        prompts.append(f"{header}\n{block}".strip())
+
+    return prompts
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=(
+            "Iterate prompts in a dump file, time generations, print outputs, and report last-10 average time."
+        )
+    )
+    parser.add_argument(
+        "--path",
+        default="benchmarks/data/prompts_g5/prompt_dump_nq_hnsw.txt",
+        help="Path to the prompt dump file",
+    )
+    parser.add_argument(
+        "--type",
+        default="ollama",
+        choices=["hf", "openai", "ollama", "gemini", "simulated"],
+        help="LLM backend type",
+    )
+    parser.add_argument(
+        "--model",
+        default="Qwen/Qwen3-4B",
+        help="Model identifier (depends on backend)",
+    )
+    parser.add_argument(
+        "--max_tokens",
+        type=int,
+        default=512,
+        help="Max new tokens to generate per prompt",
+    )
+    args = parser.parse_args()
+
+    llm_config = {"type": args.type, "model": args.model}
+    chat = get_llm(llm_config)
+
+    prompts = parse_prompts_from_file(args.path)
+    print(f"Found {len(prompts)} prompts in {args.path}")
+
+    times: list[float] = []
+    for idx, prompt in enumerate(prompts, start=1):
+        print("\n" + "=" * 80)
+        print(f"PROMPT {idx}/{len(prompts)}")
+        print("-" * 80)
+        start = time.perf_counter()
+        try:
+            output = chat.ask(prompt, max_tokens=args.max_tokens)
+        except Exception as e:
+            output = f"<error: {e}>"
+        elapsed = time.perf_counter() - start
+        times.append(elapsed)
+        print(f"Time: {elapsed:.3f}s")
+        print("-" * 80)
+        print(output)
+        print("=" * 80)
+
+    if times:
+        window = times[-10:] if len(times) >= 10 else times
+        avg_last_10 = mean(window)
+        print(
+            f"\nAverage time over last {len(window)} prompts: {avg_last_10:.3f}s"
+        )
+    else:
+        print("No prompts processed.")
+
+
+if __name__ == "__main__":
+    main()
+
+
+
+
+

benchmarks/micro_tpt.py

@@ -2,21 +2,20 @@
 
 import argparse
 import time
+from contextlib import contextmanager
 from dataclasses import dataclass
-from typing import Dict, List, Optional, Tuple
 
 import numpy as np
 import torch
 from torch import nn
-from torchao import quantize_
-from transformers import AutoModel, BitsAndBytesConfig
 from tqdm import tqdm
-from contextlib import contextmanager
+from transformers import AutoModel, BitsAndBytesConfig
+
 
 @dataclass
 class BenchmarkConfig:
     model_path: str
-    batch_sizes: List[int]
+    batch_sizes: list[int]
     seq_length: int
     num_runs: int
     use_fp16: bool = True
@@ -27,46 +26,58 @@ class BenchmarkConfig:
     use_linear8bitlt: bool = False
 
 
-class CUDAGraphContainer:
-    """Container for managing CUDA graphs for different batch sizes."""
+class GraphContainer:
+    """Container for managing graphs for different batch sizes (CUDA graphs on NVIDIA, regular on others)."""
 
     def __init__(self, model: nn.Module, seq_length: int):
         self.model = model
         self.seq_length = seq_length
-        self.graphs: Dict[int, CUDAGraphWrapper] = {}
+        self.graphs: dict[int, GraphWrapper] = {}
 
-    def get_or_create(self, batch_size: int) -> 'CUDAGraphWrapper':
+    def get_or_create(self, batch_size: int) -> "GraphWrapper":
         if batch_size not in self.graphs:
-            self.graphs[batch_size] = CUDAGraphWrapper(
-                self.model, batch_size, self.seq_length
-            )
+            self.graphs[batch_size] = GraphWrapper(self.model, batch_size, self.seq_length)
         return self.graphs[batch_size]
 
 
-class CUDAGraphWrapper:
-    """Wrapper for CUDA graph capture and replay."""
+class GraphWrapper:
+    """Wrapper for graph capture and replay (CUDA graphs on NVIDIA, regular on others)."""
 
     def __init__(self, model: nn.Module, batch_size: int, seq_length: int):
         self.model = model
+        self.device = self._get_device()
         self.static_input = self._create_random_batch(batch_size, seq_length)
         self.static_attention_mask = torch.ones_like(self.static_input)
 
         # Warm up
         self._warmup()
 
+        # Only use CUDA graphs on NVIDIA GPUs
+        if torch.cuda.is_available() and hasattr(torch.cuda, "CUDAGraph"):
             # Capture graph
             self.graph = torch.cuda.CUDAGraph()
             with torch.cuda.graph(self.graph):
                 self.static_output = self.model(
                     input_ids=self.static_input,
-                attention_mask=self.static_attention_mask
+                    attention_mask=self.static_attention_mask,
                 )
+            self.use_cuda_graph = True
+        else:
+            # For MPS or CPU, just store the model
+            self.use_cuda_graph = False
+            self.static_output = None
+
+    def _get_device(self) -> str:
+        if torch.cuda.is_available():
+            return "cuda"
+        elif torch.backends.mps.is_available():
+            return "mps"
+        else:
+            return "cpu"
 
     def _create_random_batch(self, batch_size: int, seq_length: int) -> torch.Tensor:
         return torch.randint(
-            0, 1000, (batch_size, seq_length), 
-            device="cuda", 
-            dtype=torch.long
+            0, 1000, (batch_size, seq_length), device=self.device, dtype=torch.long
         )
 
     def _warmup(self, num_warmup: int = 3):
@@ -74,14 +85,18 @@ class CUDAGraphWrapper:
             for _ in range(num_warmup):
                 self.model(
                     input_ids=self.static_input,
-                    attention_mask=self.static_attention_mask
+                    attention_mask=self.static_attention_mask,
                 )
 
     def __call__(self, input_ids: torch.Tensor, attention_mask: torch.Tensor) -> torch.Tensor:
+        if self.use_cuda_graph:
             self.static_input.copy_(input_ids)
             self.static_attention_mask.copy_(attention_mask)
             self.graph.replay()
             return self.static_output
+        else:
+            # For MPS/CPU, just run normally
+            return self.model(input_ids=input_ids, attention_mask=attention_mask)
 
 
 class ModelOptimizer:
@@ -95,8 +110,16 @@ class ModelOptimizer:
             raise ValueError("Cannot optimize None model")
 
         # Move to GPU
+        if torch.cuda.is_available():
             model = model.cuda()
-        print("- Model moved to GPU")
+            device = "cuda"
+        elif torch.backends.mps.is_available():
+            model = model.to("mps")
+            device = "mps"
+        else:
+            model = model.cpu()
+            device = "cpu"
+        print(f"- Model moved to {device}")
 
         # FP16
         if config.use_fp16 and not config.use_int4:
@@ -105,17 +128,22 @@ class ModelOptimizer:
             model = torch.compile(model)
             print("- Using FP16 precision")
 
-        # Check if using SDPA
-        if torch.version.cuda and float(torch.version.cuda[:3]) >= 11.6:
-            if hasattr(torch.nn.functional, 'scaled_dot_product_attention'):
+        # Check if using SDPA (only on CUDA)
+        if (
+            torch.cuda.is_available()
+            and torch.version.cuda
+            and float(torch.version.cuda[:3]) >= 11.6
+        ):
+            if hasattr(torch.nn.functional, "scaled_dot_product_attention"):
                 print("- Using PyTorch SDPA (scaled_dot_product_attention)")
             else:
                 print("- PyTorch SDPA not available")
 
-        # Flash Attention
-        if config.use_flash_attention:
+        # Flash Attention (only on CUDA)
+        if config.use_flash_attention and torch.cuda.is_available():
             try:
-                from flash_attn.flash_attention import FlashAttention
+                from flash_attn.flash_attention import FlashAttention  # noqa: F401
+
                 print("- Flash Attention 2 available")
                 if hasattr(model.config, "attention_mode"):
                     model.config.attention_mode = "flash_attention_2"
@@ -123,10 +151,12 @@ class ModelOptimizer:
             except ImportError:
                 print("- Flash Attention not available")
 
-        # Memory efficient attention
+        # Memory efficient attention (only on CUDA)
+        if torch.cuda.is_available():
             try:
-            from xformers.ops import memory_efficient_attention
-            if hasattr(model, 'enable_xformers_memory_efficient_attention'):
+                from xformers.ops import memory_efficient_attention  # noqa: F401
+
+                if hasattr(model, "enable_xformers_memory_efficient_attention"):
                     model.enable_xformers_memory_efficient_attention()
                     print("- Enabled xformers memory efficient attention")
                 else:
@@ -141,21 +171,38 @@ class ModelOptimizer:
 
 
 class Timer:
-    """Handles accurate GPU timing using CUDA events."""
+    """Handles accurate GPU timing using GPU events or CPU timing."""
 
     def __init__(self):
+        if torch.cuda.is_available():
             self.start_event = torch.cuda.Event(enable_timing=True)
             self.end_event = torch.cuda.Event(enable_timing=True)
+            self.use_gpu_timing = True
+        elif torch.backends.mps.is_available():
+            # MPS doesn't have events, use CPU timing
+            self.use_gpu_timing = False
+        else:
+            # CPU timing
+            self.use_gpu_timing = False
 
     @contextmanager
     def timing(self):
+        if self.use_gpu_timing:
             self.start_event.record()
             yield
             self.end_event.record()
             self.end_event.synchronize()
+        else:
+            # Use CPU timing for MPS/CPU
+            start_time = time.time()
+            yield
+            self.cpu_elapsed = time.time() - start_time
 
     def elapsed_time(self) -> float:
+        if self.use_gpu_timing:
             return self.start_event.elapsed_time(self.end_event) / 1000  # ms to seconds
+        else:
+            return self.cpu_elapsed
 
 
 class Benchmark:
@@ -168,14 +215,14 @@ class Benchmark:
             if self.model is None:
                 raise ValueError("Model initialization failed - model is None")
 
-            self.cuda_graphs = (
-                CUDAGraphContainer(self.model, config.seq_length)
-                if config.use_cuda_graphs
-                else None
-            )
+            # Only use CUDA graphs on NVIDIA GPUs
+            if config.use_cuda_graphs and torch.cuda.is_available():
+                self.graphs = GraphContainer(self.model, config.seq_length)
+            else:
+                self.graphs = None
             self.timer = Timer()
         except Exception as e:
-            print(f"ERROR in benchmark initialization: {str(e)}")
+            print(f"ERROR in benchmark initialization: {e!s}")
             raise
 
     def _load_model(self) -> nn.Module:
@@ -185,15 +232,17 @@ class Benchmark:
             # Int4 quantization using HuggingFace integration
             if self.config.use_int4:
                 import bitsandbytes as bnb
+
                 print(f"- bitsandbytes version: {bnb.__version__}")
 
-                # 检查是否使用自定义的8bit量化
-                if hasattr(self.config, 'use_linear8bitlt') and self.config.use_linear8bitlt:
+                # Check if using custom 8bit quantization
+                if hasattr(self.config, "use_linear8bitlt") and self.config.use_linear8bitlt:
                     print("- Using custom Linear8bitLt replacement for all linear layers")
 
-                    # 加载原始模型（不使用量化配置）
+                    # Load original model (without quantization config)
                     import bitsandbytes as bnb
                     import torch
+
                     # set default to half
                     torch.set_default_dtype(torch.float16)
                     compute_dtype = torch.float16 if self.config.use_fp16 else torch.float32
@@ -202,52 +251,58 @@ class Benchmark:
                         torch_dtype=compute_dtype,
                     )
 
-                    # 定义替换函数
+                    # Define replacement function
                     def replace_linear_with_linear8bitlt(model):
-                        """递归地将模型中的所有nn.Linear层替换为Linear8bitLt"""
+                        """Recursively replace all nn.Linear layers with Linear8bitLt"""
                         for name, module in list(model.named_children()):
                             if isinstance(module, nn.Linear):
-                                # 获取原始线性层的参数
+                                # Get original linear layer parameters
                                 in_features = module.in_features
                                 out_features = module.out_features
                                 bias = module.bias is not None
 
-                                # 创建8bit线性层
+                                # Create 8bit linear layer
                                 # print size
                                 print(f"in_features: {in_features}, out_features: {out_features}")
                                 new_module = bnb.nn.Linear8bitLt(
                                     in_features,
                                     out_features,
                                     bias=bias,
-                                    has_fp16_weights=False
+                                    has_fp16_weights=False,
                                 )
 
-                                # 复制权重和偏置
+                                # Copy weights and bias
                                 new_module.weight.data = module.weight.data
                                 if bias:
                                     new_module.bias.data = module.bias.data
 
-                                # 替换模块
+                                # Replace module
                                 setattr(model, name, new_module)
                             else:
-                                # 递归处理子模块
+                                # Process child modules recursively
                                 replace_linear_with_linear8bitlt(module)
 
                         return model
 
-                    # 替换所有线性层
+                    # Replace all linear layers
                     model = replace_linear_with_linear8bitlt(model)
                     # add torch compile
                     model = torch.compile(model)
 
-                    # 将模型移到GPU（量化发生在这里）
-                    device = "cuda" if torch.cuda.is_available() else "cpu"
+                    # Move model to GPU (quantization happens here)
+                    device = (
+                        "cuda"
+                        if torch.cuda.is_available()
+                        else "mps"
+                        if torch.backends.mps.is_available()
+                        else "cpu"
+                    )
                     model = model.to(device)
 
                     print("- All linear layers replaced with Linear8bitLt")
 
                 else:
-                    # 使用原来的Int4量化方法
+                    # Use original Int4 quantization method
                     print("- Using bitsandbytes for Int4 quantization")
 
                     # Create quantization config
@@ -257,7 +312,7 @@ class Benchmark:
                         load_in_4bit=True,
                         bnb_4bit_compute_dtype=compute_dtype,
                         bnb_4bit_use_double_quant=True,
-                        bnb_4bit_quant_type="nf4"
+                        bnb_4bit_quant_type="nf4",
                     )
 
                     print("- Quantization config:", quantization_config)
@@ -267,7 +322,7 @@ class Benchmark:
                         self.config.model_path,
                         quantization_config=quantization_config,
                         torch_dtype=compute_dtype,
-                        device_map="auto"  # Let HF decide on device mapping
+                        device_map="auto",  # Let HF decide on device mapping
                     )
 
                 # Check if model loaded successfully
@@ -279,7 +334,7 @@ class Benchmark:
                 # Apply optimizations directly here
                 print("\nApplying model optimizations:")
 
-                if hasattr(self.config, 'use_linear8bitlt') and self.config.use_linear8bitlt:
+                if hasattr(self.config, "use_linear8bitlt") and self.config.use_linear8bitlt:
                     print("- Model moved to GPU with Linear8bitLt quantization")
                 else:
                     # Skip moving to GPU since device_map="auto" already did that
@@ -289,16 +344,20 @@ class Benchmark:
                 print(f"- Using {compute_dtype} for compute dtype")
 
                 # Check CUDA and SDPA
-                if torch.version.cuda and float(torch.version.cuda[:3]) >= 11.6:
-                    if hasattr(torch.nn.functional, 'scaled_dot_product_attention'):
+                if (
+                    torch.cuda.is_available()
+                    and torch.version.cuda
+                    and float(torch.version.cuda[:3]) >= 11.6
+                ):
+                    if hasattr(torch.nn.functional, "scaled_dot_product_attention"):
                         print("- Using PyTorch SDPA (scaled_dot_product_attention)")
                     else:
                         print("- PyTorch SDPA not available")
 
-                # Try xformers if available
+                # Try xformers if available (only on CUDA)
+                if torch.cuda.is_available():
                     try:
-                    from xformers.ops import memory_efficient_attention
-                    if hasattr(model, 'enable_xformers_memory_efficient_attention'):
+                        if hasattr(model, "enable_xformers_memory_efficient_attention"):
                             model.enable_xformers_memory_efficient_attention()
                             print("- Enabled xformers memory efficient attention")
                         else:
@@ -310,58 +369,30 @@ class Benchmark:
                 model.eval()
                 print("- Model set to eval mode")
             # Int8 quantization using HuggingFace integration
-            # Int8 quantization using TorchAO
             elif self.config.use_int8:
-                print("- Using TorchAO for Int8 dynamic activation and Int8 weight quantization")
-                
-                # Import the quantize_ function and the quantization config
-                from torchao.quantization import quantize_, int8_dynamic_activation_int8_weight
-                print("- Successfully imported TorchAO")
-                
-                # Load model normally first
-                # set default to half
-                import torch
-                torch.set_default_dtype(torch.bfloat16)
-                model = AutoModel.from_pretrained(
-                    self.config.model_path,
-                    device_map="auto"
+                print("- Using INT8 quantization")
+                # For now, just use standard loading with INT8 config
+                compute_dtype = torch.float16 if self.config.use_fp16 else torch.float32
+                quantization_config = BitsAndBytesConfig(
+                    load_in_8bit=True,
+                    llm_int8_threshold=6.0,
+                    llm_int8_has_fp16_weight=False,
                 )
 
-                print("- Model loaded in full precision")
+                model = AutoModel.from_pretrained(
+                    self.config.model_path,
+                    quantization_config=quantization_config,
+                    torch_dtype=compute_dtype,
+                    device_map="auto",
+                )
+
+                if model is None:
+                    raise ValueError("Model loading returned None")
+
                 print(f"- Model type: {type(model)}")
-                
-                # Apply quantization - call the function to get the config, then apply it
-                # quantize_(model, int8_dynamic_activation_int8_weight())
-                # from torchao.quantization import quantize_, Int8DynamicActivationInt8WeightConfig,int8_dynamic_activation_int8_semi_sparse_weight,int4_weight_only,Int8DynActInt4WeightGPTQQuantizer,int8_dynamic_activation_int4_weight,Int8DynamicActivationInt4WeightConfig,Int4DynamicActivationInt4WeightConfig
-                from torchao.quantization import quantize_, Int8DynamicActivationInt8WeightConfig
-                quantize_(model, Int8DynamicActivationInt8WeightConfig())
-                print("- Model successfully quantized with int8 weights and int8 activations")
-                # add torch compile
-                model = torch.compile(model)
-                # For older PyTorch versions that have issues with tensor subclasses
-                from torchao.utils import unwrap_tensor_subclass
-                import torch
-                if hasattr(torch, '_version') and not torch.version >= "2.5.0":
-                    print("- Unwrapping tensor subclasses for compatibility with older PyTorch")
-                    unwrap_tensor_subclass(model)
-                
-                # Apply optimizations
-                if torch.version.cuda and float(torch.version.cuda[:3]) >= 11.6:
-                    if hasattr(torch.nn.functional, 'scaled_dot_product_attention'):
-                        print("- Using PyTorch SDPA (scaled_dot_product_attention)")
-                    else:
-                        print("- PyTorch SDPA not available")
-                
-                # Set to eval mode
                 model.eval()
                 print("- Model set to eval mode")
 
-                # For better performance with int8 dynamic quantization
-                torch._inductor.config.force_fuse_int_mm_with_mul = True
-                print("- Enabled fusion of int matmul with mul operations")
-
-                
-                
             else:
                 # Standard loading for FP16/FP32
                 model = AutoModel.from_pretrained(self.config.model_path)
@@ -371,6 +402,7 @@ class Benchmark:
                 # Apply standard optimizations
                 # set default to half
                 import torch
+
                 torch.set_default_dtype(torch.bfloat16)
                 model = ModelOptimizer.optimize(model, self.config)
                 model = model.half()
@@ -385,49 +417,60 @@ class Benchmark:
             return model
 
         except Exception as e:
-            print(f"ERROR loading model: {str(e)}")
+            print(f"ERROR loading model: {e!s}")
             import traceback
+
             traceback.print_exc()
             raise
 
     def _create_random_batch(self, batch_size: int) -> torch.Tensor:
+        device = (
+            "cuda"
+            if torch.cuda.is_available()
+            else "mps"
+            if torch.backends.mps.is_available()
+            else "cpu"
+        )
         return torch.randint(
-            0, 1000,
+            0,
+            1000,
             (batch_size, self.config.seq_length),
-            device="cuda",
-            dtype=torch.long
+            device=device,
+            dtype=torch.long,
         )
 
     def _run_inference(
-        self,
-        input_ids: torch.Tensor,
-        cuda_graph_wrapper: Optional[CUDAGraphWrapper] = None
-    ) -> Tuple[float, torch.Tensor]:
+        self, input_ids: torch.Tensor, graph_wrapper: GraphWrapper | None = None
+    ) -> tuple[float, torch.Tensor]:
         attention_mask = torch.ones_like(input_ids)
 
         with torch.no_grad(), self.timer.timing():
-            if cuda_graph_wrapper is not None:
-                output = cuda_graph_wrapper(input_ids, attention_mask)
+            if graph_wrapper is not None:
+                output = graph_wrapper(input_ids, attention_mask)
             else:
                 output = self.model(input_ids=input_ids, attention_mask=attention_mask)
 
         return self.timer.elapsed_time(), output
 
-    def run(self) -> Dict[int, Dict[str, float]]:
+    def run(self) -> dict[int, dict[str, float]]:
         results = {}
 
         # Reset peak memory stats
+        if torch.cuda.is_available():
             torch.cuda.reset_peak_memory_stats()
+        elif torch.backends.mps.is_available():
+            # MPS doesn't have reset_peak_memory_stats, skip it
+            pass
+        else:
+            print("- No GPU memory stats available")
 
         for batch_size in self.config.batch_sizes:
             print(f"\nTesting batch size: {batch_size}")
             times = []
 
-            # Get or create CUDA graph for this batch size
-            cuda_graph_wrapper = (
-                self.cuda_graphs.get_or_create(batch_size)
-                if self.cuda_graphs is not None
-                else None
+            # Get or create graph for this batch size
+            graph_wrapper = (
+                self.graphs.get_or_create(batch_size) if self.graphs is not None else None
             )
 
             # Pre-allocate input tensor
@@ -437,7 +480,7 @@ class Benchmark:
             # Run benchmark
             for i in tqdm(range(self.config.num_runs), desc=f"Batch size {batch_size}"):
                 try:
-                    elapsed_time, output = self._run_inference(input_ids, cuda_graph_wrapper)
+                    elapsed_time, output = self._run_inference(input_ids, graph_wrapper)
                     if i == 0:  # Only print on first run
                         print(f"Output shape: {output.last_hidden_state.shape}")
                     times.append(elapsed_time)
@@ -464,8 +507,19 @@ class Benchmark:
             print(f"Throughput: {throughput:.2f} sequences/second")
 
         # Log memory usage
+        if torch.cuda.is_available():
             peak_memory_gb = torch.cuda.max_memory_allocated() / (1024**3)
+        elif torch.backends.mps.is_available():
+            # MPS doesn't have max_memory_allocated, use 0
+            peak_memory_gb = 0.0
+        else:
+            peak_memory_gb = 0.0
+            print("- No GPU memory usage available")
+
+        if peak_memory_gb > 0:
             print(f"\nPeak GPU memory usage: {peak_memory_gb:.2f} GB")
+        else:
+            print("\n- GPU memory usage not available")
 
         # Add memory info to results
         for batch_size in results:
@@ -485,7 +539,7 @@ def main():
     parser.add_argument(
         "--batch_sizes",
         type=str,
-        default="1,2,4,8,10,16,20,32,40,64,128,256,512,1024,2048,4096,8192",
+        default="1,2,4,8,16,32",
         help="Comma-separated list of batch sizes",
     )
     parser.add_argument(
@@ -518,12 +572,12 @@ def main():
     parser.add_argument(
         "--use_cuda_graphs",
         action="store_true",
-        help="Enable CUDA Graphs optimization",
+        help="Enable CUDA Graphs optimization (only on NVIDIA GPUs)",
     )
     parser.add_argument(
         "--use_flash_attention",
         action="store_true",
-        help="Enable Flash Attention 2 if available",
+        help="Enable Flash Attention 2 if available (only on NVIDIA GPUs)",
     )
     parser.add_argument(
         "--use_linear8bitlt",
@@ -568,7 +622,15 @@ def main():
         os.makedirs("results", exist_ok=True)
 
         # Generate filename based on configuration
-        precision_type = "int4" if config.use_int4 else "fp16" if config.use_fp16 else "fp32"
+        precision_type = (
+            "int4"
+            if config.use_int4
+            else "int8"
+            if config.use_int8
+            else "fp16"
+            if config.use_fp16
+            else "fp32"
+        )
         model_name = os.path.basename(config.model_path)
         output_file = f"results/benchmark_{model_name}_{precision_type}.json"
 
@@ -576,17 +638,20 @@ def main():
         with open(output_file, "w") as f:
             json.dump(
                 {
-                    "config": {k: str(v) if isinstance(v, list) else v for k, v in vars(config).items()},
-                    "results": {str(k): v for k, v in results.items()}
+                    "config": {
+                        k: str(v) if isinstance(v, list) else v for k, v in vars(config).items()
+                    },
+                    "results": {str(k): v for k, v in results.items()},
                 },
                 f,
-                indent=2
+                indent=2,
             )
         print(f"Results saved to {output_file}")
 
     except Exception as e:
         print(f"Benchmark failed: {e}")
         import traceback
+
         traceback.print_exc()
 
 

benchmarks/run_all.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# 公共参数
+INDEX_PATH="benchmarks/data/indices/rpj_wiki/rpj_wiki"
+NUM_QUERIES=20
+BATCH_SIZE=128
+LLM_MODEL="qwen3:4b"
+TOP_K=3
+
+# 日志目录（带时间戳）
+LOG_DIR="logs/eval_runs_$(date +%Y%m%d_%H%M%S)"
+mkdir -p "$LOG_DIR"
+
+# dataset -> ef 列表
+declare -A EF_MAP=(
+  [nq_open.jsonl]="32 62 190"
+  [trivia_qa.jsonl]="77 150 249"
+  [gpqa.jsonl]="41 72 124"
+  [hotpot_qa.jsonl]="137 299 1199"
+)
+
+# 按指定顺序遍历
+ORDERED_DATASETS=(nq_open.jsonl trivia_qa.jsonl gpqa.jsonl hotpot_qa.jsonl)
+
+for dataset in "${ORDERED_DATASETS[@]}"; do
+  for ef in ${EF_MAP[$dataset]}; do
+    log_file="${LOG_DIR}/${dataset%.jsonl}_ef${ef}.log"
+
+    # 展示并记录将要执行的命令
+    cmd=(python benchmarks/run_evaluation.py "$INDEX_PATH" \
+      --num-queries "$NUM_QUERIES" \
+      --ef "$ef" \
+      --batch-size "$BATCH_SIZE" \
+      --llm-model "$LLM_MODEL" \
+      --top-k "$TOP_K" \
+      --queries-file "$dataset")
+
+    echo "=== Running dataset=${dataset} ef=${ef} ===" | tee -a "$log_file"
+    printf 'CMD: '; printf '%q ' "${cmd[@]}" | tee -a "$log_file"; echo | tee -a "$log_file"
+
+    # 同时输出到命令行和日志文件
+    "${cmd[@]}" 2>&1 | tee -a "$log_file"
+
+    echo | tee -a "$log_file"
+  done
+done
+
+echo "All runs completed. Logs in: $LOG_DIR"

benchmarks/run_evaluation.py

@@ -0,0 +1,452 @@
+#!/usr/bin/env python3
+"""
+This script runs a recall evaluation on a given LEANN index.
+It correctly compares results by fetching the text content for both the new search
+results and the golden standard results, making the comparison robust to ID changes.
+"""
+
+import argparse
+import json
+import sys
+import time
+from pathlib import Path
+
+import numpy as np
+from leann.api import LeannBuilder, LeannChat, LeannSearcher
+
+
+def download_data_if_needed(data_root: Path, download_embeddings: bool = False):
+    """Checks if the data directory exists, and if not, downloads it from HF Hub."""
+    if not data_root.exists():
+        print(f"Data directory '{data_root}' not found.")
+        print("Downloading evaluation data from Hugging Face Hub... (this may take a moment)")
+        try:
+            from huggingface_hub import snapshot_download
+
+            if download_embeddings:
+                # Download everything including embeddings (large files)
+                snapshot_download(
+                    repo_id="LEANN-RAG/leann-rag-evaluation-data",
+                    repo_type="dataset",
+                    local_dir=data_root,
+                    local_dir_use_symlinks=False,
+                )
+                print("Data download complete (including embeddings)!")
+            else:
+                # Download only specific folders, excluding embeddings
+                allow_patterns = [
+                    "ground_truth/**",
+                    "indices/**",
+                    "queries/**",
+                    "*.md",
+                    "*.txt",
+                ]
+                snapshot_download(
+                    repo_id="LEANN-RAG/leann-rag-evaluation-data",
+                    repo_type="dataset",
+                    local_dir=data_root,
+                    local_dir_use_symlinks=False,
+                    allow_patterns=allow_patterns,
+                )
+                print("Data download complete (excluding embeddings)!")
+        except ImportError:
+            print(
+                "Error: huggingface_hub is not installed. Please install it to download the data:"
+            )
+            print("uv pip install -e '.[dev]'")
+            sys.exit(1)
+        except Exception as e:
+            print(f"An error occurred during data download: {e}")
+            sys.exit(1)
+
+
+def download_embeddings_if_needed(data_root: Path, dataset_type: str | None = None):
+    """Download embeddings files specifically."""
+    embeddings_dir = data_root / "embeddings"
+
+    if dataset_type:
+        # Check if specific dataset embeddings exist
+        target_file = embeddings_dir / dataset_type / "passages_00.pkl"
+        if target_file.exists():
+            print(f"Embeddings for {dataset_type} already exist")
+            return str(target_file)
+
+    print("Downloading embeddings from HuggingFace Hub...")
+    try:
+        from huggingface_hub import snapshot_download
+
+        # Download only embeddings folder
+        snapshot_download(
+            repo_id="LEANN-RAG/leann-rag-evaluation-data",
+            repo_type="dataset",
+            local_dir=data_root,
+            local_dir_use_symlinks=False,
+            allow_patterns=["embeddings/**/*.pkl"],
+        )
+        print("Embeddings download complete!")
+
+        if dataset_type:
+            target_file = embeddings_dir / dataset_type / "passages_00.pkl"
+            if target_file.exists():
+                return str(target_file)
+
+        return str(embeddings_dir)
+
+    except Exception as e:
+        print(f"Error downloading embeddings: {e}")
+        sys.exit(1)
+
+
+# --- Helper Function to get Golden Passages ---
+def get_golden_texts(searcher: LeannSearcher, golden_ids: list[int]) -> set:
+    """
+    Retrieves the text for golden passage IDs directly from the LeannSearcher's
+    passage manager.
+    """
+    golden_texts = set()
+    for gid in golden_ids:
+        try:
+            # PassageManager uses string IDs
+            passage_data = searcher.passage_manager.get_passage(str(gid))
+            golden_texts.add(passage_data["text"])
+        except KeyError:
+            print(f"Warning: Golden passage ID '{gid}' not found in the index's passage data.")
+    return golden_texts
+
+
+def load_queries(file_path: Path) -> list[str]:
+    queries = []
+    with open(file_path, encoding="utf-8") as f:
+        for line in f:
+            data = json.loads(line)
+            queries.append(data["query"])
+    return queries
+
+
+def build_index_from_embeddings(embeddings_file: str, output_path: str, backend: str = "hnsw"):
+    """
+    Build a LEANN index from pre-computed embeddings.
+
+    Args:
+        embeddings_file: Path to pickle file with (ids, embeddings) tuple
+        output_path: Path where to save the index
+        backend: Backend to use ("hnsw" or "diskann")
+    """
+    print(f"Building {backend} index from embeddings: {embeddings_file}")
+
+    # Create builder with appropriate parameters
+    if backend == "hnsw":
+        builder_kwargs = {
+            "M": 32,  # Graph degree
+            "efConstruction": 256,  # Construction complexity
+            "is_compact": True,  # Use compact storage
+            "is_recompute": True,  # Enable pruning for better recall
+        }
+    elif backend == "diskann":
+        builder_kwargs = {
+            "complexity": 64,
+            "graph_degree": 32,
+            "search_memory_maximum": 8.0,  # GB
+            "build_memory_maximum": 16.0,  # GB
+        }
+    else:
+        builder_kwargs = {}
+
+    builder = LeannBuilder(
+        backend_name=backend,
+        embedding_model="facebook/contriever-msmarco",  # Model used to create embeddings
+        dimensions=768,  # Will be auto-detected from embeddings
+        **builder_kwargs,
+    )
+
+    # Build index from precomputed embeddings
+    builder.build_index_from_embeddings(output_path, embeddings_file)
+    print(f"Index saved to: {output_path}")
+    return output_path
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Run recall evaluation on a LEANN index.")
+    parser.add_argument(
+        "index_path",
+        type=str,
+        nargs="?",
+        help="Path to the LEANN index to evaluate or build (optional).",
+    )
+    parser.add_argument(
+        "--mode",
+        choices=["evaluate", "build"],
+        default="evaluate",
+        help="Mode: 'evaluate' existing index or 'build' from embeddings",
+    )
+    parser.add_argument(
+        "--embeddings-file",
+        type=str,
+        help="Path to embeddings pickle file (optional for build mode)",
+    )
+    parser.add_argument(
+        "--backend",
+        choices=["hnsw", "diskann"],
+        default="hnsw",
+        help="Backend to use for building index (default: hnsw)",
+    )
+    parser.add_argument(
+        "--num-queries", type=int, default=10, help="Number of queries to evaluate."
+    )
+    parser.add_argument("--top-k", type=int, default=3, help="The 'k' value for recall@k.")
+    parser.add_argument(
+        "--ef-search", type=int, default=120, help="The 'efSearch' parameter for HNSW."
+    )
+    parser.add_argument(
+        "--batch-size",
+        type=int,
+        default=0,
+        help="Batch size for HNSW batched search (0 disables batching)",
+    )
+    parser.add_argument(
+        "--queries-file",
+        type=str,
+        default="nq_open.jsonl",
+        help=(
+            "Queries file to use. Provide a filename under benchmarks/data/queries "
+            "or an absolute path to a .jsonl file (default: nq_open.jsonl)."
+        ),
+    )
+    parser.add_argument(
+        "--llm-type",
+        type=str,
+        choices=["ollama", "hf", "openai", "gemini", "simulated"],
+        default="ollama",
+        help="LLM backend type to optionally query during evaluation (default: ollama)",
+    )
+    parser.add_argument(
+        "--llm-model",
+        type=str,
+        default="qwen3:1.7b",
+        help="LLM model identifier for the chosen backend (default: qwen3:1.7b)",
+    )
+    args = parser.parse_args()
+
+    # --- Path Configuration ---
+    # Assumes a project structure where the script is in 'benchmarks/'
+    # and evaluation data is in 'benchmarks/data/'.
+    script_dir = Path(__file__).resolve().parent
+    data_root = script_dir / "data"
+
+    # Download data based on mode
+    if args.mode == "build":
+        # For building mode, we need embeddings
+        download_data_if_needed(data_root, download_embeddings=False)  # Basic data first
+
+        # Auto-detect dataset type and download embeddings
+        if args.embeddings_file:
+            embeddings_file = args.embeddings_file
+            # Try to detect dataset type from embeddings file path
+            if "rpj_wiki" in str(embeddings_file):
+                dataset_type = "rpj_wiki"
+            elif "dpr" in str(embeddings_file):
+                dataset_type = "dpr"
+            else:
+                dataset_type = "dpr"  # Default
+        else:
+            # Auto-detect from index path if provided, otherwise default to DPR
+            if args.index_path:
+                index_path_str = str(args.index_path)
+                if "rpj_wiki" in index_path_str:
+                    dataset_type = "rpj_wiki"
+                elif "dpr" in index_path_str:
+                    dataset_type = "dpr"
+                else:
+                    dataset_type = "dpr"  # Default to DPR
+            else:
+                dataset_type = "dpr"  # Default to DPR
+
+            embeddings_file = download_embeddings_if_needed(data_root, dataset_type)
+
+        # Auto-generate index path if not provided
+        if not args.index_path:
+            indices_dir = data_root / "indices" / dataset_type
+            indices_dir.mkdir(parents=True, exist_ok=True)
+            args.index_path = str(indices_dir / f"{dataset_type}_from_embeddings")
+            print(f"Auto-generated index path: {args.index_path}")
+
+        print(f"Building index from embeddings: {embeddings_file}")
+        built_index_path = build_index_from_embeddings(
+            embeddings_file, args.index_path, args.backend
+        )
+        print(f"Index built successfully: {built_index_path}")
+
+        # Ask if user wants to run evaluation
+        eval_response = input("Run evaluation on the built index? (y/n): ").strip().lower()
+        if eval_response != "y":
+            print("Index building complete. Exiting.")
+            return
+    else:
+        # For evaluation mode, don't need embeddings
+        download_data_if_needed(data_root, download_embeddings=False)
+
+        # Auto-detect index path if not provided
+        if not args.index_path:
+            # Default to using downloaded indices
+            indices_dir = data_root / "indices"
+
+            # Try common datasets in order of preference
+            for dataset in ["dpr", "rpj_wiki"]:
+                dataset_dir = indices_dir / dataset
+                if dataset_dir.exists():
+                    # Look for index files
+                    index_files = list(dataset_dir.glob("*.index")) + list(
+                        dataset_dir.glob("*_disk.index")
+                    )
+                    if index_files:
+                        args.index_path = str(
+                            index_files[0].with_suffix("")
+                        )  # Remove .index extension
+                        print(f"Using index: {args.index_path}")
+                        break
+
+            if not args.index_path:
+                print("No indices found. The data download should have included pre-built indices.")
+                print(
+                    "Please check the benchmarks/data/indices/ directory or provide --index-path manually."
+                )
+                sys.exit(1)
+
+    # Detect dataset type from index path to select the correct ground truth
+    index_path_str = str(args.index_path)
+    if "rpj_wiki" in index_path_str:
+        dataset_type = "rpj_wiki"
+    elif "dpr" in index_path_str:
+        dataset_type = "dpr"
+    else:
+        # Fallback: try to infer from the index directory name
+        dataset_type = Path(args.index_path).name
+        print(f"WARNING: Could not detect dataset type from path, inferred '{dataset_type}'.")
+
+    # Resolve queries file (supports absolute path or name under data/queries)
+    queries_file_candidate = Path(args.queries_file)
+    if queries_file_candidate.is_absolute():
+        queries_file = queries_file_candidate
+    else:
+        queries_file = data_root / "queries" / args.queries_file
+
+    if not queries_file.exists():
+        print(f"Error: Queries file not found: {queries_file}")
+        print("Tip: Use --queries-file with a filename under benchmarks/data/queries or an absolute path.")
+        sys.exit(1)
+
+    # Infer ground-truth file from the queries filename
+    qname = queries_file.name.lower()
+    if "hotpot" in qname:
+        task_key = "hotpot"
+    elif "trivia" in qname:
+        task_key = "trivia"
+    elif "gpqa" in qname:
+        task_key = "gpqa"
+    elif "nq" in qname:
+        task_key = "nq"
+    else:
+        print(
+            "Error: Could not infer task from queries filename. Supported names include 'nq', 'hotpot', 'trivia', 'gpqa'."
+        )
+        print(f"Filename was: {queries_file.name}")
+        sys.exit(1)
+
+    golden_results_file = data_root / "ground_truth" / dataset_type / f"flat_results_{task_key}_k3.json"
+    if not golden_results_file.exists():
+        gt_dir = data_root / "ground_truth" / dataset_type
+        try:
+            available = sorted(p.name for p in gt_dir.glob("flat_results_*_k3.json"))
+        except Exception:
+            available = []
+        print(
+            f"Error: Ground truth file not found for task '{task_key}' under dataset '{dataset_type}': {golden_results_file}"
+        )
+        if available:
+            print("Available ground truth files:")
+            for name in available:
+                print(f"  - {name}")
+        else:
+            print(f"No ground truth files found in {gt_dir}")
+        sys.exit(1)
+
+    print(f"INFO: Detected dataset type: {dataset_type}")
+    print(f"INFO: Using queries file: {queries_file}")
+    print(f"INFO: Using ground truth file: {golden_results_file}")
+
+    try:
+        searcher = LeannSearcher(args.index_path)
+        queries = load_queries(queries_file)
+
+        with open(golden_results_file) as f:
+            golden_results_data = json.load(f)
+
+        num_eval_queries = min(args.num_queries, len(queries))
+        queries = queries[:num_eval_queries]
+
+        print(f"\nRunning evaluation on {num_eval_queries} queries...")
+        recall_scores = []
+        search_times = []
+
+        for i in range(num_eval_queries):
+            start_time = time.time()
+            new_results = searcher.search(
+                queries[i],
+                top_k=args.top_k,
+                complexity=args.ef_search,
+                batch_size=args.batch_size,
+            )
+            search_times.append(time.time() - start_time)
+
+            # Optional: also call the LLM with configurable backend/model (does not affect recall)
+            # llm_config = {"type": args.llm_type, "model": args.llm_model}
+            # chat = LeannChat(args.index_path, llm_config=llm_config, searcher=searcher)
+            # answer = chat.ask(
+            #     queries[i],
+            #     top_k=args.top_k,
+            #     complexity=args.ef_search,
+            #     batch_size=args.batch_size,
+            # )
+            # print(f"Answer: {answer}")
+            # Correct Recall Calculation: Based on TEXT content
+            new_texts = {result.text for result in new_results}
+
+            # Get golden texts directly from the searcher's passage manager
+            golden_ids = golden_results_data["indices"][i][: args.top_k]
+            golden_texts = get_golden_texts(searcher, golden_ids)
+
+            overlap = len(new_texts & golden_texts)
+            recall = overlap / len(golden_texts) if golden_texts else 0
+            recall_scores.append(recall)
+
+            print("\n--- EVALUATION RESULTS ---")
+            print(f"Query: {queries[i]}")
+            print(f"New Results: {new_texts}")
+            print(f"Golden Results: {golden_texts}")
+            print(f"Overlap: {overlap}")
+            print(f"Recall: {recall}")
+            print(f"Search Time: {search_times[-1]:.4f}s")
+            print("--------------------------------")
+
+        avg_recall = np.mean(recall_scores) if recall_scores else 0
+        avg_time = np.mean(search_times) if search_times else 0
+
+        print(f"search time: {search_times}")
+
+        print("\n🎉 --- Evaluation Complete ---")
+        print(f"Avg. Recall@{args.top_k} (efSearch={args.ef_search}): {avg_recall:.4f}")
+        print(f"Avg. Search Time: {avg_time:.4f}s")
+
+        # avg last 10 search times
+        avg_last_10_search_times = np.mean(search_times[-10:])
+        print(f"Avg. Last 10 Search Times: {avg_last_10_search_times:.4f}s")
+
+    except Exception as e:
+        print(f"\n❌ An error occurred during evaluation: {e}")
+        import traceback
+
+        traceback.print_exc()
+
+
+if __name__ == "__main__":
+    main()

benchmarks/run_speed_bench_all.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Absolute paths (adjust if needed)
+PROMPTS_DIR="/home/tony/yichuan/leann/benchmarks/data/prompts_g5"
+SCRIPT_PATH="/home/tony/yichuan/leann/benchmarks/generation_speed_bench.py"
+
+# Common args
+MAX_TOKENS=2048
+OLLAMA_MODEL="qwen3:4b"
+HF_MODEL="Qwen/Qwen3-4B"
+
+# Logs
+LOG_DIR="/home/tony/yichuan/leann/logs/speed_bench_$(date +%Y%m%d_%H%M%S)"
+mkdir -p "$LOG_DIR"
+
+echo "Scanning: $PROMPTS_DIR"
+
+# Iterate all .txt files under PROMPTS_DIR
+while IFS= read -r -d '' file; do
+  base_name=$(basename "$file")
+  stem_name="${base_name%.*}"
+
+  # 1) Ollama
+  log_ollama="${LOG_DIR}/${stem_name}_ollama.log"
+  cmd_ollama=(python "$SCRIPT_PATH" \
+    --path "$file" \
+    --type ollama \
+    --model "$OLLAMA_MODEL" \
+    --max_tokens "$MAX_TOKENS")
+
+  echo "=== Running (ollama) file=${file} model=${OLLAMA_MODEL} ===" | tee -a "$log_ollama"
+  printf 'CMD: '; printf '%q ' "${cmd_ollama[@]}" | tee -a "$log_ollama"; echo | tee -a "$log_ollama"
+  "${cmd_ollama[@]}" 2>&1 | tee -a "$log_ollama"
+  echo | tee -a "$log_ollama"
+
+  # 2) HF
+  log_hf="${LOG_DIR}/${stem_name}_hf.log"
+  cmd_hf=(python "$SCRIPT_PATH" \
+    --path "$file" \
+    --type hf \
+    --model "$HF_MODEL" \
+    --max_tokens "$MAX_TOKENS")
+
+  echo "=== Running (hf) file=${file} model=${HF_MODEL} ===" | tee -a "$log_hf"
+  printf 'CMD: '; printf '%q ' "${cmd_hf[@]}" | tee -a "$log_hf"; echo | tee -a "$log_hf"
+  "${cmd_hf[@]}" 2>&1 | tee -a "$log_hf"
+  echo | tee -a "$log_hf"
+
+done < <(find "$PROMPTS_DIR" -type f -name '*.txt' -print0)
+
+
+echo "All runs completed. Logs in: $LOG_DIR"
+
+

benchmarks/simple_mac_tpt_test.py

@@ -0,0 +1,317 @@
+import time
+from dataclasses import dataclass
+
+import numpy as np
+import torch
+from torch import nn
+from tqdm import tqdm
+from transformers import AutoModel
+
+# Add MLX imports
+try:
+    import mlx.core as mx
+    from mlx_lm.utils import load
+
+    MLX_AVAILABLE = True
+except ImportError:
+    print("MLX not available. Install with: uv pip install mlx mlx-lm")
+    MLX_AVAILABLE = False
+
+
+@dataclass
+class BenchmarkConfig:
+    model_path: str = "facebook/contriever-msmarco"
+    batch_sizes: list[int] = None
+    seq_length: int = 256
+    num_runs: int = 5
+    use_fp16: bool = True
+    use_int4: bool = False
+    use_int8: bool = False
+    use_cuda_graphs: bool = False
+    use_flash_attention: bool = False
+    use_linear8bitlt: bool = False
+    use_mlx: bool = False  # New flag for MLX testing
+
+    def __post_init__(self):
+        if self.batch_sizes is None:
+            self.batch_sizes = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024]
+
+
+class MLXBenchmark:
+    """MLX-specific benchmark for embedding models"""
+
+    def __init__(self, config: BenchmarkConfig):
+        self.config = config
+        self.model, self.tokenizer = self._load_model()
+
+    def _load_model(self):
+        """Load MLX model and tokenizer following the API pattern"""
+        print(f"Loading MLX model from {self.config.model_path}...")
+        try:
+            model, tokenizer = load(self.config.model_path)
+            print("MLX model loaded successfully")
+            return model, tokenizer
+        except Exception as e:
+            print(f"Error loading MLX model: {e}")
+            raise
+
+    def _create_random_batch(self, batch_size: int):
+        """Create random input batches for MLX testing - same as PyTorch"""
+        return torch.randint(0, 1000, (batch_size, self.config.seq_length), dtype=torch.long)
+
+    def _run_inference(self, input_ids: torch.Tensor) -> float:
+        """Run MLX inference with same input as PyTorch"""
+        start_time = time.time()
+        try:
+            # Convert PyTorch tensor to MLX array
+            input_ids_mlx = mx.array(input_ids.numpy())
+
+            # Get embeddings
+            embeddings = self.model(input_ids_mlx)
+
+            # Mean pooling (following the API pattern)
+            pooled = embeddings.mean(axis=1)
+
+            # Convert to numpy (following the API pattern)
+            pooled_numpy = np.array(pooled.tolist(), dtype=np.float32)
+
+            # Force computation
+            _ = pooled_numpy.shape
+
+        except Exception as e:
+            print(f"MLX inference error: {e}")
+            return float("inf")
+        end_time = time.time()
+
+        return end_time - start_time
+
+    def run(self) -> dict[int, dict[str, float]]:
+        """Run the MLX benchmark across all batch sizes"""
+        results = {}
+
+        print(f"Starting MLX benchmark with model: {self.config.model_path}")
+        print(f"Testing batch sizes: {self.config.batch_sizes}")
+
+        for batch_size in self.config.batch_sizes:
+            print(f"\n=== Testing MLX batch size: {batch_size} ===")
+            times = []
+
+            # Create input batch (same as PyTorch)
+            input_ids = self._create_random_batch(batch_size)
+
+            # Warm up
+            print("Warming up...")
+            for _ in range(3):
+                try:
+                    self._run_inference(input_ids[:2])  # Warm up with smaller batch
+                except Exception as e:
+                    print(f"Warmup error: {e}")
+                    break
+
+            # Run benchmark
+            for _i in tqdm(range(self.config.num_runs), desc=f"MLX Batch size {batch_size}"):
+                try:
+                    elapsed_time = self._run_inference(input_ids)
+                    if elapsed_time != float("inf"):
+                        times.append(elapsed_time)
+                except Exception as e:
+                    print(f"Error during MLX inference: {e}")
+                    break
+
+            if not times:
+                print(f"Skipping batch size {batch_size} due to errors")
+                continue
+
+            # Calculate statistics
+            avg_time = np.mean(times)
+            std_time = np.std(times)
+            throughput = batch_size / avg_time
+
+            results[batch_size] = {
+                "avg_time": avg_time,
+                "std_time": std_time,
+                "throughput": throughput,
+                "min_time": np.min(times),
+                "max_time": np.max(times),
+            }
+
+            print(f"MLX Results for batch size {batch_size}:")
+            print(f"  Avg Time: {avg_time:.4f}s ± {std_time:.4f}s")
+            print(f"  Min Time: {np.min(times):.4f}s")
+            print(f"  Max Time: {np.max(times):.4f}s")
+            print(f"  Throughput: {throughput:.2f} sequences/second")
+
+        return results
+
+
+class Benchmark:
+    def __init__(self, config: BenchmarkConfig):
+        self.config = config
+        self.device = (
+            "cuda"
+            if torch.cuda.is_available()
+            else "mps"
+            if torch.backends.mps.is_available()
+            else "cpu"
+        )
+        self.model = self._load_model()
+
+    def _load_model(self) -> nn.Module:
+        print(f"Loading model from {self.config.model_path}...")
+
+        model = AutoModel.from_pretrained(self.config.model_path)
+        if self.config.use_fp16:
+            model = model.half()
+        model = torch.compile(model)
+        model = model.to(self.device)
+
+        model.eval()
+        return model
+
+    def _create_random_batch(self, batch_size: int) -> torch.Tensor:
+        return torch.randint(
+            0,
+            1000,
+            (batch_size, self.config.seq_length),
+            device=self.device,
+            dtype=torch.long,
+        )
+
+    def _run_inference(self, input_ids: torch.Tensor) -> float:
+        attention_mask = torch.ones_like(input_ids)
+        # print shape of input_ids and attention_mask
+        print(f"input_ids shape: {input_ids.shape}")
+        print(f"attention_mask shape: {attention_mask.shape}")
+        start_time = time.time()
+        with torch.no_grad():
+            self.model(input_ids=input_ids, attention_mask=attention_mask)
+        if torch.cuda.is_available():
+            torch.cuda.synchronize()
+        if torch.backends.mps.is_available():
+            torch.mps.synchronize()
+        end_time = time.time()
+
+        return end_time - start_time
+
+    def run(self) -> dict[int, dict[str, float]]:
+        results = {}
+
+        if torch.cuda.is_available():
+            torch.cuda.reset_peak_memory_stats()
+
+        for batch_size in self.config.batch_sizes:
+            print(f"\nTesting batch size: {batch_size}")
+            times = []
+
+            input_ids = self._create_random_batch(batch_size)
+
+            for _i in tqdm(range(self.config.num_runs), desc=f"Batch size {batch_size}"):
+                try:
+                    elapsed_time = self._run_inference(input_ids)
+                    times.append(elapsed_time)
+                except Exception as e:
+                    print(f"Error during inference: {e}")
+                    break
+
+            if not times:
+                continue
+
+            avg_time = np.mean(times)
+            std_time = np.std(times)
+            throughput = batch_size / avg_time
+
+            results[batch_size] = {
+                "avg_time": avg_time,
+                "std_time": std_time,
+                "throughput": throughput,
+            }
+
+            print(f"Avg Time: {avg_time:.4f}s ± {std_time:.4f}s")
+            print(f"Throughput: {throughput:.2f} sequences/second")
+
+        if torch.cuda.is_available():
+            peak_memory_gb = torch.cuda.max_memory_allocated() / (1024**3)
+        else:
+            peak_memory_gb = 0.0
+
+        for batch_size in results:
+            results[batch_size]["peak_memory_gb"] = peak_memory_gb
+
+        return results
+
+
+def run_benchmark():
+    """Main function to run the benchmark with optimized parameters."""
+    config = BenchmarkConfig()
+
+    try:
+        benchmark = Benchmark(config)
+        results = benchmark.run()
+
+        max_throughput = max(results[batch_size]["throughput"] for batch_size in results)
+        avg_throughput = np.mean([results[batch_size]["throughput"] for batch_size in results])
+
+        return {
+            "max_throughput": max_throughput,
+            "avg_throughput": avg_throughput,
+            "results": results,
+        }
+
+    except Exception as e:
+        print(f"Benchmark failed: {e}")
+        return {"max_throughput": 0.0, "avg_throughput": 0.0, "error": str(e)}
+
+
+def run_mlx_benchmark():
+    """Run MLX-specific benchmark"""
+    if not MLX_AVAILABLE:
+        print("MLX not available, skipping MLX benchmark")
+        return {
+            "max_throughput": 0.0,
+            "avg_throughput": 0.0,
+            "error": "MLX not available",
+        }
+
+    config = BenchmarkConfig(model_path="mlx-community/all-MiniLM-L6-v2-4bit", use_mlx=True)
+
+    try:
+        benchmark = MLXBenchmark(config)
+        results = benchmark.run()
+
+        if not results:
+            return {
+                "max_throughput": 0.0,
+                "avg_throughput": 0.0,
+                "error": "No valid results",
+            }
+
+        max_throughput = max(results[batch_size]["throughput"] for batch_size in results)
+        avg_throughput = np.mean([results[batch_size]["throughput"] for batch_size in results])
+
+        return {
+            "max_throughput": max_throughput,
+            "avg_throughput": avg_throughput,
+            "results": results,
+        }
+
+    except Exception as e:
+        print(f"MLX benchmark failed: {e}")
+        return {"max_throughput": 0.0, "avg_throughput": 0.0, "error": str(e)}
+
+
+if __name__ == "__main__":
+    print("=== PyTorch Benchmark ===")
+    pytorch_result = run_benchmark()
+    print(f"PyTorch Max throughput: {pytorch_result['max_throughput']:.2f} sequences/second")
+    print(f"PyTorch Average throughput: {pytorch_result['avg_throughput']:.2f} sequences/second")
+
+    print("\n=== MLX Benchmark ===")
+    mlx_result = run_mlx_benchmark()
+    print(f"MLX Max throughput: {mlx_result['max_throughput']:.2f} sequences/second")
+    print(f"MLX Average throughput: {mlx_result['avg_throughput']:.2f} sequences/second")
+
+    # Compare results
+    if pytorch_result["max_throughput"] > 0 and mlx_result["max_throughput"] > 0:
+        speedup = mlx_result["max_throughput"] / pytorch_result["max_throughput"]
+        print("\n=== Comparison ===")
+        print(f"MLX is {speedup:.2f}x {'faster' if speedup > 1 else 'slower'} than PyTorch")

data/huawei_pangu.md

@@ -0,0 +1,82 @@
+# 盘古之殇：华为诺亚盘古大模型研发历程的心酸与黑暗
+
+各位好，
+
+我是一名盘古大模型团队，华为诺亚方舟实验室的员工。
+
+首先为自证身份，列举一些细节：
+
+1. 现诺亚主任，前算法应用部部长，后改名为小模型实验室的主任王云鹤。前诺亚主任：姚骏（大家称姚老师）。几个实验室主任：唐睿明（明哥，明队，已离职），尚利峰，张维（维哥），郝建业（郝老师），刘武龙（称呼为武龙所）等。其他骨干成员和专家陆续有很多人离职。
+2. 我们隶属于“四野”这个组织。四野下属有许多纵队，基础语言大模型是四纵。王云鹤的小模型是十六纵队。我们参加过苏州的集结，有各种月份的时间节点。在苏州攻关会颁发任务令，需要在节点前达成目标。苏州集结会把各地的人员都集中在苏州研究所，平常住宾馆，比如在甪直的酒店，与家人孩子天各一方。
+3. 在苏州集结的时候周六默认上班，非常辛苦，不过周六有下午茶，有一次还有小龙虾。在苏州研究所的工位搬迁过一次，从一栋楼换到了另一栋。苏州研究所楼栋都是欧式装修，门口有大坡，里面景色很不错。去苏州集结一般至少要去一周，甚至更久，多的人甚至一两个月都回不了家。
+4. 诺亚曾经传说是研究型的，但是来了之后因为在四野做大模型项目，项目成员完全变成了交付型的，且充满了例会，评审，汇报。很多时候做实验都要申请。团队需要对接终端小艺，华为云，ICT等诸多业务线，交付压力不小。
+5. 诺亚研发的盘古模型早期内部代号叫做“盘古智子”，一开始只有内部需要申请试用的网页版，到后续迫于压力在welink上接入和公测开放。
+
+这些天发生关于质疑盘古大模型抄袭千问的事情闹的沸沸扬扬。作为一个盘古团队的成员，我最近夜夜辗转反侧，难以入眠。盘古的品牌受到如此大的影响，一方面，我自私的为我的职业发展担忧，也为自己过去的努力工作感到不值。另一方面，由于有人开始揭露这些事情我内心又感到大快人心。在多少个日日夜夜，我们对内部某些人一次次靠着造假而又获得了无数利益的行为咬牙切齿而又无能为力。这种压抑和羞辱也逐渐消磨了我对华为的感情，让我在这里的时日逐渐浑浑噩噩，迷茫无措，时常怀疑自己的人生和自我价值。
+
+我承认我是一个懦弱的人，作为一个小小的打工人，我不仅不敢和王云鹤等内部手眼通天的人做对，更不敢和华为这样的庞然大物做对。我很怕失去我的工作，毕竟我也有家人和孩子，所以我打心眼里很佩服揭露者。但是，看到内部还在试图洗地掩盖事实，蒙蔽公众的时候，我实在不能容忍了。我也希望勇敢一次，顺从自己本心。就算自损八百，我也希望能伤敌一千。我决定把我在这里的所见所闻（部分来自于同事口述）公布出来，关于盘古大模型的“传奇故事”：
+
+华为确实主要在昇腾卡上训练大模型（小模型实验室有不少英伟达的卡，他们之前也会用来训练，后面转移到昇腾）。曾经我被华为“打造世界第二选择”的决心而折服，我本身也曾经对华为有深厚的感情。我们陪着昇腾一步步摸爬滚打，从充满bug到现在能训出模型，付出了巨大的心血和代价。
+
+最初我们的算力非常有限，在910A上训练模型。那会只支持fp16，训练的稳定性远不如bf16。盘古的moe开始很早，23年就主要是训练38Bmoe模型和后续的71B dense模型。71B的dense模型通过扩增变成了第一代的135Bdense模型，后面主力模型也逐渐在910B上训练。
+
+71B和135B模型都有一个巨大的硬伤就是tokenizer。当时使用的tokenizer编码效率极低，每个单个的符号，数字，空格，乃至汉字都会占用一个token。可想而知这会非常浪费算力，且使得模型的效果很差。这时候小模型实验室正好有个自己训的词表。姚老师当时怀疑是不是模型的tokenizer不好（虽然事后来看，他的怀疑是无疑正确的），于是就决定，让71B和135B换tokenizer，因为小模型实验室曾经尝试过。团队缝合了两个tokenizer，开始了tokenizer的更换。71B模型的更换失败了，而135B因为采用了更精细的embedding初始化策略，续训了至少1T的数据后词表总算更换成功，但可想而知，效果并不会变好。
+
+于此同期，阿里和智谱等国内其他公司在GPU上训练，且已经摸索出了正确的方法，盘古和竞品的差距越来越大。内部一个230B从头训练的dense模型又因为各种原因训练失败，导致项目的状况几乎陷入绝境。面临几个节点的压力以及内部对盘古的强烈质疑时，团队的士气低迷到了极点。团队在算力极其有限的时候，做出了很多努力和挣扎。比如，团队偶然发现当时的38B moe并没有预期moe的效果。于是去掉了moe参数，还原为了13B的dense模型。由于38B的moe源自很早的pangu alpha 13B，架构相对落后，团队进行了一系列的操作，比如切换绝对位置编码到rope，去掉bias，切换为rmsnorm。同时鉴于tokenizer的一些失败和换词表的经验，这个模型的词表也更换为了王云鹤的小模型实验室7B模型所使用的词表。后面这个13B模型进行了扩增续训，变成了第二代38B dense模型（在几个月内这个模型都是主要的盘古中档位模型），曾经具有一定的竞争力。但是，由于更大的135B模型架构落后，且更换词表模型损伤巨大（后续分析发现当时更换的缝合词表有更严重的bug），续训后也与千问等当时国内领先模型存在很大差距。这时由于内部的质疑声和领导的压力也越来越大。团队的状态几乎陷入了绝境。
+
+在这种情况下，王云鹤和他的小模型实验室出手了。他们声称是从旧的135B参数继承改造而来，通过训练短短的几百B数据，各项指标平均提升了十个点左右。实际上，这就是他们套壳应用到大模型的第一次杰作。华为的外行领导内行，使得领导完全对于这种扯淡的事情没有概念，他们只会觉得肯定是有什么算法创新。经过内部的分析，他们实际上是使用Qwen 1.5 110B续训而来，通过加层，扩增ffn维度，添加盘古pi论文的一些机制得来，凑够了大概135B的参数。实际上，旧的135B有107层，而这个模型只有82层，各种配置也都不一样。新的来路不明的135B训练完很多参数的分布也和Qwen 110B几乎一模一样。连模型代码的类名当时都是Qwen，甚至懒得改名。后续这个模型就是所谓的135B V2。而这个模型当时也提供给了很多下游，甚至包括外部客户。
+
+这件事对于我们这些认真诚实做事的同事们带来了巨大的冲击，内部很多人其实都知道这件事，甚至包括终端和华为云。我们都戏称以后别叫盘古模型了，叫千古吧。当时团队成员就想向bcg举报了，毕竟这已经是重大的业务造假了。但是后面据说被领导拦了下来，因为更高级别的领导（比如姚老师，以及可能熊总和查老）其实后面也知道了，但是并不管，因为通过套壳拿出好的结果，对他们也是有利的。这件事使得当时团队几位最强的同事开始心灰意冷，离职跑路也逐渐成为挂在嘴边的事。
+
+此时，盘古似乎迎来了转机。由于前面所述的这些盘古模型基本都是续训和改造而来，当时诺亚完全没有掌握从头训练的技术，何况还是在昇腾的NPU上进行训练。在当时团队的核心成员的极力争取下，盘古开始了第三代模型的训练，付出了巨大的努力后，在数据架构和训练算法方面都与业界逐渐接轨，而这其中的艰辛和小模型实验室的人一点关系都没有。
+
+一开始团队成员毫无信心，只从一个13B的模型开始训练，但是后面发现效果还不错，于是这个模型后续再次进行了一次参数扩增，变成了第三代的38B，代号38B V3。想必很多产品线的兄弟都对这个模型很熟悉。当时这个模型的tokenizer是基于llama的词表进行扩展的（也是业界常见的做法）。而当时王云鹤的实验室做出来了另一个词表（也就是后续pangu系列的词表）。当时两个词表还被迫进行了一次赛马，最终没有明显的好坏结论。于是，领导当即决定，应该统一词表，使用王云鹤他们的。于是，在后续从头训练的135B V3（也就是对外的Pangu Ultra），便是采用了这个tokenizer。这也解释了很多使用我们模型的兄弟的疑惑，为什么当时同为V3代的两个不同档位的模型，会使用不同的tokenizer。
+
+
+我们打心眼里觉得，135B V3是我们四纵团队当时的骄傲。这是第一个真正意义上的，华为全栈自研，正经从头训练的千亿级别的模型，且效果与24年同期竞品可比的。写到这里我已经热泪盈眶，太不容易了。当时为了稳定训练，团队做了大量实验对比，并且多次在模型梯度出现异常的时候进行及时回退重启。这个模型真正做到了后面技术报告所说的训练全程没有一个loss spike。我们克服了不知道多少困难，我们做到了，我们愿用生命和荣誉保证这个模型训练的真实性。多少个凌晨，我们为了它的训练而不眠。在被内部心声骂的一文不值的时候，我们有多么不甘，有多少的委屈，我们挺住了。
+
+我们这帮人是真的在为打磨国产算力底座燃烧自己的青春啊……客居他乡，我们放弃了家庭，放弃了假期，放弃了健康，放弃了娱乐，抛头颅洒热血，其中的艰辛与困苦，寥寥数笔不足以概括其万一。在各种动员大会上，当时口号中喊出的盘古必胜，华为必胜，我们心里是真的深深被感动。
+
+然而，我们的所有辛苦的成果，经常被小模型实验室轻飘飘的拿走了。数据，直接要走。代码，直接要走，还要求我们配合适配到能一键运行。我们当时戏称小模型实验室为点鼠标实验室。我们付出辛苦，他们取得荣耀。果然应了那句话，你在负重前行是因为有人替你岁月静好。在这种情况下，越来越多的战友再也坚持不下去了，选择了离开。看到身边那些优秀的同事一个个离职，我的内心又感叹又难过。在这种作战一样的环境下，我们比起同事来说更像是战友。他们在技术上也有无数值得我学习的地方，堪称良师。看到他们去了诸如字节Seed，Deepseek，月之暗面，腾讯和快手等等很多出色的团队，我打心眼里为他们高兴和祝福，脱离了这个辛苦却肮脏的地方。我至今还对一位离职同事的话记忆犹新，ta说：“来这里是我技术生涯中的耻辱，在这里再呆每一天都是浪费生命”。话虽难听却让我无言以对。我担心我自己技术方面的积累不足，以及没法适应互联网公司高淘汰的环境，让我多次想离职的心始终没有迈出这一步。
+
+盘古除了dense模型，后续也启动了moe的探索。一开始训练的是一个224B的moe模型。而与之平行的，小模型实验室也开启了第二次主要的套壳行动（次要的插曲可能还包括一些别的模型，比如math模型），即这次流传甚广的pangu pro moe 72B。这个模型内部自称是从小模型实验室的7B扩增上来的（就算如此，这也与技术报告不符，何况是套壳qwen 2.5的14b续训）。还记得他们训了没几天，内部的评测就立刻追上了当时的38B V3。AI系统实验室很多兄弟因为需要适配模型，都知道他们的套壳行动，只是迫于各种原因，无法伸张正义。实际上，对于后续训了很久很久的这个模型，Honestagi能够分析出这个量级的相似性我已经很诧异了，因为这个模型为了续训洗参数，所付出的算力甚至早就足够从头训一个同档位的模型了。听同事说他们为了洗掉千问的水印，采取了不少办法，甚至包括故意训了脏数据。这也为学术界研究模型血缘提供了一个前所未有的特殊模范吧。以后新的血缘方法提出可以拿出来溜溜。
+
+24年底和25年初，在Deepseek v3和r1发布之后，由于其惊艳的技术水平，团队受到了巨大的冲击，也受到了更大的质疑。于是为了紧跟潮流，盘古模仿Deepseek的模型尺寸，开启了718B moe的训练。这个时候，小模型实验室再次出手了。他们选择了套壳Deepseekv3续训。他们通过冻住Deepseek加载的参数，进行训练。连任务加载ckpt的目录都是deepseekv3，改都不改，何其嚣张？与之相反，一些有真正技术信仰的同事，在从头训练另一个718B的moe。但其中出现了各种各样的问题。但是很显然，这个模型怎么可能比直接套壳的好呢？如果不是团队leader坚持，早就被叫停了。
+
+华为的流程管理之繁重，严重拖累了大模型的研发节奏，例如版本管理，模型血缘，各种流程化，各种可追溯。讽刺的是，小模型实验室的模型似乎从来不受这些流程的约束，想套壳就套壳，想续训就续训，算力源源不断的伸手拿走。这种强烈到近乎魔幻的对比，说明了当前流程管理的情况：只许州官放火，不许百姓点灯。何其可笑？何其可悲？何其可恶？何其可耻！
+
+HonestAGI的事情出来后，内部让大家不停的研讨分析，如何公关和“回应”。诚然，这个原文的分析也许不够有力，给了王云鹤与小模型实验室他们狡辩和颠倒黑白的机会。为此，这两天我内心感到作呕，时时怀疑自己的人生意义以及苍天无眼。我不奉陪了，我要离职了，同时我也在申请从盘古部分技术报告的作者名单中移除。曾经在这些技术报告上署名是我一生都无法抹除的污点。当时我没想到，他们竟然猖狂到敢开源。我没想到，他们敢如此愚弄世人，大肆宣发。当时，我也许是存了侥幸心理，没有拒绝署名。我相信很多扎实做事的战友，也只是被迫上了贼船，或者不知情。但这件事已经无法挽回，我希望我的余生能够坚持扎实做真正有意义的事，为我当时的软弱和不坚定赎罪。
+
+深夜写到这里，我已经泪流满面，泣不成声。还记得一些出色的同事离职时，我苦笑问他们要不要发个长长的心声惯例帖，揭露一下现状。对方说：不了，浪费时间，而且我也怕揭露出来你们过的更糟。我当时一下黯然神伤，因为曾经共同为了理想奋斗过的战友已经彻底对华为彻底灰心了。当时大家调侃，我们用着当年共产党的小米加步枪，组织却有着堪比当年国民党的作风。
+
+曾几何时，我为我们用着小米加步枪打败洋枪洋炮而自豪。
+
+现在，我累了，我想投降。
+
+其实时至今日，我还是真心希望华为能认真吸取教训，能做好盘古，把盘古做到世界一流，把昇腾变成英伟达的水平。内部的劣币驱逐良币，使得诺亚乃至华为在短时间内急剧流失了大量出色的大模型人才。相信他们也正在如Deepseek等各个团队闪耀着，施展着他们的抱负才华，为中美在AI的激烈竞赛中奉献力量。我时常感叹，华为不是没有人才，而是根本不知道怎么留住人才。如果给这些人合适的环境，合适的资源，更少的枷锁，更少的政治斗争，盘古何愁不成？
+
+最后：我以生命，人格和荣誉发誓，我写的以上所有内容均为真实（至少在我有限的认知范围内）。我没有那么高的技术水平以及机会去做详尽扎实的分析，也不敢直接用内部记录举证，怕因为信息安全抓到。但是我相信我很多曾经的战友，会为我作证。在华为内部的兄弟，包括我们曾经服务过的产品线兄弟们，相信本文的无数细节能和你们的印象对照，印证我的说法。你们可能也曾经被蒙骗，但这些残酷的真相不会被尘封。我们奋战过的痕迹，也不应该被扭曲和埋葬。
+
+写了这么多，某些人肯定想把我找出来，抹杀掉。公司搞不好也想让我噤声乃至追责。如果真的这样，我，乃至我的家人的人身乃至生命安全可能都会受到威胁。为了自我保护，我近期每天会跟大家报平安。
+
+如果我消失了，就当是我为了真理和理想，为了华为乃至中国能够更好地发展算力和AI而牺牲了吧，我愿埋葬于那片曾经奋斗过的地方。
+
+诺亚，再见
+
+2025年7月6日凌晨      写于深圳
+
+---
+
+各位好，
+
+感谢大家的关心与祝福。我目前暂时安全，但公司应该在进行排查与某些名单收集，后续情况未知。
+
+我补充一些细节，以免某些人继续颠倒黑白。
+
+关于135B V2，小模型实验室在迅速地完成套壳并拿完所有套壳带来的好处后（比如任务令表彰和及时激励），因为不想继续支撑下游应用和模型迭代，又把这个烫手山芋甩给了四纵。确实技高一筹，直接把四纵的兄弟们拉下水。同事提供过去一个老旧的模型，最终拿回了一个当时一个魔改的先进的千问。做大模型的人，自己做的模型就像自己孩子一样熟悉，不要把别人都当傻子。就像自家儿子出门一趟，回来个别人家孩子。
+
+盘古report的署名是不符合学术规范的。例如，135B V3有不少有技术贡献的人，因为作者名额数量限制，劳动成果没有得到应有的回报，团队内曾经有不小的意见。这个模型当时是大家智慧和汗水的结晶，甚至是团队当时的精神支柱，支撑着不少兄弟们继续留在诺亚。所谓的名额限制，以及挂名了一些毫无技术贡献的人（如一些小模型实验室的人），让兄弟们何其心寒。
+
+---
+
+暂时平安。另外，支持我勇于说出真相的战友们 https://github.com/HW-whistleblower/True-Story-of-Pangu/issues/317

demo.ipynb

@@ -1,362 +1,116 @@
 {
  "cells": [
   {
-   "cell_type": "code",
-   "execution_count": 1,
+   "cell_type": "markdown",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Initializing leann-backend-diskann...\n",
-      "INFO: Registering backend 'diskann'\n",
-      "INFO: DiskANN backend loaded successfully\n",
-      "INFO: LeannBuilder initialized with 'diskann' backend.\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/home/ubuntu/LEANN_clean/leann/.venv/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
-      "  from .autonotebook import tqdm as notebook_tqdm\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "INFO: Computing embeddings for 6 chunks using 'sentence-transformers/all-mpnet-base-v2'...\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Batches: 100%|██████████| 1/1 [00:00<00:00,  2.91it/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "INFO: Building DiskANN index for 6 vectors with metric Metric.INNER_PRODUCT...\n",
-      "Using Inner Product search, so need to pre-process base data into temp file. Please ensure there is additional (n*(d+1)*4) bytes for storing pre-processed base vectors, apart from the interim indices created by DiskANN and the final index.\n",
-      "Pre-processing base file by adding extra coordinate\n",
-      "✅ DiskANN index built successfully at 'knowledge'\n",
-      "Writing bin: knowledge_disk.index_max_base_norm.bin\n",
-      "bin: #pts = 1, #dims = 1, size = 12B\n",
-      "Finished writing bin.\n",
-      "Time for preprocessing data for inner product: 0.000172 seconds\n",
-      "Reading max_norm_of_base from knowledge_disk.index_max_base_norm.bin\n",
-      "Reading bin file knowledge_disk.index_max_base_norm.bin ...\n",
-      "Opening bin file knowledge_disk.index_max_base_norm.bin... \n",
-      "Metadata: #pts = 1, #dims = 1...\n",
-      "done.\n",
-      "max_norm_of_base: 1\n",
-      "! Using prepped_base file at knowledge_prepped_base.bin\n",
-      "Starting index build: R=32 L=64 Query RAM budget: 4.02653e+09 Indexing ram budget: 8 T: 8\n",
-      "getting bin metadata\n",
-      "Time for getting bin metadata: 0.000019 seconds\n",
-      "Compressing 769-dimensional data into 512 bytes per vector.\n",
-      "Opened: knowledge_prepped_base.bin, size: 18464, cache_size: 18464\n",
-      "Training data with 6 samples loaded.\n",
-      "Reading bin file knowledge_pq_pivots.bin ...\n",
-      "Opening bin file knowledge_pq_pivots.bin... \n",
-      "Metadata: #pts = 256, #dims = 769...\n",
-      "done.\n",
-      "PQ pivot file exists. Not generating again\n",
-      "Opened: knowledge_prepped_base.bin, size: 18464, cache_size: 18464\n",
-      "Reading bin file knowledge_pq_pivots.bin ...\n",
-      "Opening bin file knowledge_pq_pivots.bin... \n",
-      "Metadata: #pts = 4, #dims = 1...\n",
-      "done.\n",
-      "Reading bin file knowledge_pq_pivots.bin ...\n",
-      "Opening bin file knowledge_pq_pivots.bin... \n",
-      "Metadata: #pts = 256, #dims = 769...\n",
-      "done.\n",
-      "Reading bin file knowledge_pq_pivots.bin ...\n",
-      "Opening bin file knowledge_pq_pivots.bin... \n",
-      "Metadata: #pts = 769, #dims = 1...\n",
-      "done.\n",
-      "Reading bin file knowledge_pq_pivots.bin ...\n",
-      "Opening bin file knowledge_pq_pivots.bin... \n",
-      "Metadata: #pts = 513, #dims = 1...\n",
-      "done.\n",
-      "Loaded PQ pivot information\n",
-      "Processing points  [0, 6)...done.\n",
-      "Time for generating quantized data: 0.055587 seconds\n",
-      "Full index fits in RAM budget, should consume at most 2.03973e-05GiBs, so building in one shot\n",
-      "L2: Using AVX2 distance computation DistanceL2Float\n",
-      "Passed, empty search_params while creating index config\n",
-      "Using only first 6 from file.. \n",
-      "Starting index build with 6 points... \n",
-      "0% of index build completed.Starting final cleanup..done. Link time: 0.00011s\n",
-      "Index built with degree: max:5  avg:5  min:5  count(deg<2):0\n",
-      "Not saving tags as they are not enabled.\n",
-      "Time taken for save: 0.000148s.\n",
-      "Time for building merged vamana index: 0.000836 seconds\n",
-      "Opened: knowledge_prepped_base.bin, size: 18464, cache_size: 18464\n",
-      "Vamana index file size=168\n",
-      "Opened: knowledge_disk.index, cache_size: 67108864\n",
-      "medoid: 0B\n",
-      "max_node_len: 3100B\n",
-      "nnodes_per_sector: 1B\n",
-      "# sectors: 6\n",
-      "Sector #0written\n",
-      "Finished writing 28672B\n",
-      "Writing bin: knowledge_disk.index\n",
-      "bin: #pts = 9, #dims = 1, size = 80B\n",
-      "Finished writing bin.\n",
-      "Output disk index file written to knowledge_disk.index\n",
-      "Finished writing 28672B\n",
-      "Time for generating disk layout: 0.040268 seconds\n",
-      "Opened: knowledge_prepped_base.bin, size: 18464, cache_size: 18464\n",
-      "Loading base knowledge_prepped_base.bin. #points: 6. #dim: 769.\n",
-      "Wrote 1 points to sample file: knowledge_sample_data.bin\n",
-      "Indexing time: 0.0970594\n",
-      "INFO: Leann metadata saved to knowledge.leann.meta.json\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Opened file : knowledge_disk.index\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "✅ DiskANN index loaded successfully.\n",
-      "INFO: LeannSearcher initialized with 'diskann' backend using index 'knowledge.leann'.\n",
-      "Since data is floating point, we assume that it has been appropriately pre-processed (normalization for cosine, and convert-to-l2 by adding extra dimension for MIPS). So we shall invoke an l2 distance function.\n",
-      "L2: Using AVX2 distance computation DistanceL2Float\n",
-      "L2: Using AVX2 distance computation DistanceL2Float\n",
-      "Before index load\n",
-      "Reading bin file knowledge_pq_compressed.bin ...\n",
-      "Opening bin file knowledge_pq_compressed.bin... \n",
-      "Metadata: #pts = 6, #dims = 512...\n",
-      "done.\n",
-      "Reading bin file knowledge_pq_pivots.bin ...\n",
-      "Opening bin file knowledge_pq_pivots.bin... \n",
-      "Metadata: #pts = 4, #dims = 1...\n",
-      "done.\n",
-      "Offsets: 4096 791560 794644 796704\n",
-      "Reading bin file knowledge_pq_pivots.bin ...\n",
-      "Opening bin file knowledge_pq_pivots.bin... \n",
-      "Metadata: #pts = 256, #dims = 769...\n",
-      "done.\n",
-      "Reading bin file knowledge_pq_pivots.bin ...\n",
-      "Opening bin file knowledge_pq_pivots.bin... \n",
-      "Metadata: #pts = 769, #dims = 1...\n",
-      "done.\n",
-      "Reading bin file knowledge_pq_pivots.bin ...\n",
-      "Opening bin file knowledge_pq_pivots.bin... \n",
-      "Metadata: #pts = 513, #dims = 1...\n",
-      "done.\n",
-      "Loaded PQ Pivots: #ctrs: 256, #dims: 769, #chunks: 512\n",
-      "Loaded PQ centroids and in-memory compressed vectors. #points: 6 #dim: 769 #aligned_dim: 776 #chunks: 512\n",
-      "Loading index metadata from knowledge_disk.index\n",
-      "Disk-Index File Meta-data: # nodes per sector: 1, max node len (bytes): 3100, max node degree: 5\n",
-      "Disk-Index Meta: nodes per sector: 1, max node len: 3100, max node degree: 5\n",
-      "Setting up thread-specific contexts for nthreads: 8\n",
-      "allocating ctx: 0x7a33f7204000 to thread-id:134367072315200\n",
-      "allocating ctx: 0x7a33f6805000 to thread-id:134355206802368\n",
-      "allocating ctx: 0x7a33f5e72000 to thread-id:134355217288000\n",
-      "allocating ctx: 0x7a33f5e61000 to thread-id:134355227773632\n",
-      "allocating ctx: 0x7a33f5e50000 to thread-id:134355196316736\n",
-      "allocating ctx: 0x7a33f5e3f000 to thread-id:134355164859840\n",
-      "allocating ctx: 0x7a33f5e2e000 to thread-id:134355175345472\n",
-      "allocating ctx: 0x7a33f5e1d000 to thread-id:134355185831104\n",
-      "Loading centroid data from medoids vector data of 1 medoid(s)\n",
-      "Reading bin file knowledge_disk.index_max_base_norm.bin ...\n",
-      "Opening bin file knowledge_disk.index_max_base_norm.bin... \n",
-      "Metadata: #pts = 1, #dims = 1...\n",
-      "done.\n",
-      "Setting re-scaling factor of base vectors to 1\n",
-      "load_from_separate_paths done.\n",
-      "Reading (with alignment) bin file knowledge_sample_data.bin ...Metadata: #pts = 1, #dims = 769, aligned_dim = 776... allocating aligned memory of 3104 bytes... done. Copying data to mem_aligned buffer... done.\n",
-      "reserve ratio: 1\n",
-      "Graph traversal completed, hops: 3\n",
-      "Loading the cache list into memory....done.\n",
-      "After index load\n",
-      "INFO: Computing embeddings for 1 chunks using 'sentence-transformers/all-mpnet-base-v2'...\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Batches: 100%|██████████| 1/1 [00:00<00:00, 60.54it/s]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "INFO: DiskANN ZMQ mode enabled - ensuring embedding server is running\n",
-      "INFO: Starting session-level embedding server as a background process...\n",
-      "INFO: Running command from project root: /home/ubuntu/LEANN_clean/leann\n",
-      "INFO: Server process started with PID: 424761\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "✅ Embedding server is up and ready for this session.\n",
-      "[EmbeddingServer LOG]: Initializing leann-backend-diskann...\n",
-      "[EmbeddingServer LOG]: WARNING: Could not import DiskANN backend: cannot import name '_diskannpy' from partially initialized module 'packages.leann-backend-diskann.leann_backend_diskann' (most likely due to a circular import) (/home/ubuntu/LEANN_clean/leann/packages/leann-backend-diskann/leann_backend_diskann/__init__.py)\n",
-      "[EmbeddingServer LOG]: INFO: Initializing embedding server thread on port 5555\n",
-      "[EmbeddingServer LOG]: INFO: Using CUDA device\n",
-      "[EmbeddingServer LOG]: INFO: Loading model sentence-transformers/all-mpnet-base-v2\n",
-      "[EmbeddingServer LOG]: INFO: Using FP16 precision with model: sentence-transformers/all-mpnet-base-v2\n",
-      "[EmbeddingServer LOG]: INFO: Loaded 6 demo documents\n",
-      "[EmbeddingServer LOG]: INFO: ZMQ ROUTER server listening on port 5555\n",
-      "[EmbeddingServer LOG]: INFO: Embedding server ready to serve requests\n",
-      "[EmbeddingServer LOG]: INFO: Received ZMQ request from client 006b8b45, size 3 bytes\n",
-      "[EmbeddingServer LOG]: INFO: Request for 1 node embeddings: [0]\n",
-      "[EmbeddingServer LOG]: DEBUG: Node ID range: 0 to 0\n",
-      "[EmbeddingServer LOG]: Time taken for text lookup: 0.000028 seconds\n",
-      "[EmbeddingServer LOG]: INFO: Total batch size: 1, max_batch_size: 128\n",
-      "[EmbeddingServer LOG]: INFO: Processing batch of size 1\n",
-      "[EmbeddingServer LOG]: Time taken for tokenization (batch): 0.019294 seconds\n",
-      "[EmbeddingServer LOG]: Batch size: 1, Sequence length: 256\n",
-      "[EmbeddingServer LOG]: Time taken for transfer to device (batch): 0.000210 seconds\n",
-      "[EmbeddingServer LOG]: Time taken for embedding (batch): 3.065444 seconds\n",
-      "[EmbeddingServer LOG]: Time taken for mean pooling (batch): 0.041810 seconds\n",
-      "[EmbeddingServer LOG]: INFO: Serialize time: 0.000194 seconds\n",
-      "[EmbeddingServer LOG]: INFO: ZMQ E2E time: 3.128073 seconds\n",
-      "[EmbeddingServer LOG]: INFO: Received ZMQ request from client 006b8b45, size 7 bytes\n",
-      "[EmbeddingServer LOG]: INFO: Request for 5 node embeddings: [1, 2, 3, 4, 5]\n",
-      "[EmbeddingServer LOG]: DEBUG: Node ID range: 1 to 5\n",
-      "[EmbeddingServer LOG]: Time taken for text lookup: 0.000042 seconds\n",
-      "[EmbeddingServer LOG]: INFO: Total batch size: 5, max_batch_size: 128\n",
-      "[EmbeddingServer LOG]: INFO: Processing batch of size 5\n",
-      "[EmbeddingServer LOG]: Time taken for tokenization (batch): 0.001791 seconds\n",
-      "[EmbeddingServer LOG]: Batch size: 5, Sequence length: 256\n",
-      "[EmbeddingServer LOG]: Time taken for transfer to device (batch): 0.000112 seconds\n",
-      "[EmbeddingServer LOG]: Time taken for embedding (batch): 3.674183 seconds\n",
-      "[EmbeddingServer LOG]: Time taken for mean pooling (batch): 0.000372 seconds\n",
-      "[EmbeddingServer LOG]: INFO: Serialize time: 0.000177 seconds\n",
-      "[EmbeddingServer LOG]: INFO: ZMQ E2E time: 3.677425 seconds\n",
-      "[EmbeddingServer LOG]: INFO: Received ZMQ request from client 006b8b45, size 7 bytes\n",
-      "[EmbeddingServer LOG]: INFO: Request for 5 node embeddings: [3, 4, 2, 1, 0]\n",
-      "[EmbeddingServer LOG]: DEBUG: Node ID range: 0 to 4\n",
-      "[EmbeddingServer LOG]: Time taken for text lookup: 0.000030 seconds\n",
-      "[EmbeddingServer LOG]: INFO: Total batch size: 5, max_batch_size: 128\n",
-      "[EmbeddingServer LOG]: INFO: Processing batch of size 5\n",
-      "[EmbeddingServer LOG]: Time taken for tokenization (batch): 0.001550 seconds\n",
-      "[EmbeddingServer LOG]: Batch size: 5, Sequence length: 256\n",
-      "[EmbeddingServer LOG]: Time taken for transfer to device (batch): 0.000097 seconds\n",
-      "[EmbeddingServer LOG]: Time taken for embedding (batch): 0.009335 seconds\n",
-      "[EmbeddingServer LOG]: Time taken for mean pooling (batch): 0.000154 seconds\n",
-      "[EmbeddingServer LOG]: INFO: Serialize time: 0.000073 seconds\n",
-      "[EmbeddingServer LOG]: INFO: ZMQ E2E time: 0.011773 seconds\n",
-      "[EmbeddingServer LOG]: INFO: Received ZMQ request from client 006b8b45, size 7 bytes\n",
-      "[EmbeddingServer LOG]: INFO: Request for 5 node embeddings: [0, 1, 2, 4, 5]\n",
-      "[EmbeddingServer LOG]: DEBUG: Node ID range: 0 to 5\n",
-      "[EmbeddingServer LOG]: Time taken for text lookup: 0.000020 seconds\n",
-      "[EmbeddingServer LOG]: INFO: Total batch size: 5, max_batch_size: 128\n",
-      "[EmbeddingServer LOG]: INFO: Processing batch of size 5\n",
-      "[EmbeddingServer LOG]: Time taken for tokenization (batch): 0.001041 seconds\n",
-      "[EmbeddingServer LOG]: Batch size: 5, Sequence length: 256\n",
-      "[EmbeddingServer LOG]: Time taken for transfer to device (batch): 0.000125 seconds\n",
-      "[EmbeddingServer LOG]: Time taken for embedding (batch): 0.008972 seconds\n",
-      "[EmbeddingServer LOG]: Time taken for mean pooling (batch): 0.000151 seconds\n",
-      "[EmbeddingServer LOG]: INFO: Serialize time: 0.000048 seconds\n",
-      "[EmbeddingServer LOG]: INFO: ZMQ E2E time: 0.010853 seconds\n",
-      "[EmbeddingServer LOG]: INFO: Received ZMQ request from client 006b8b45, size 7 bytes\n",
-      "[EmbeddingServer LOG]: INFO: Request for 5 node embeddings: [3, 1, 0, 2, 5]\n",
-      "[EmbeddingServer LOG]: DEBUG: Node ID range: 0 to 5\n",
-      "[EmbeddingServer LOG]: Time taken for text lookup: 0.000020 seconds\n",
-      "[EmbeddingServer LOG]: INFO: Total batch size: 5, max_batch_size: 128\n",
-      "[EmbeddingServer LOG]: INFO: Processing batch of size 5\n",
-      "[EmbeddingServer LOG]: Time taken for tokenization (batch): 0.001350 seconds\n",
-      "[EmbeddingServer LOG]: Batch size: 5, Sequence length: 256\n",
-      "[EmbeddingServer LOG]: Time taken for transfer to device (batch): 0.000088 seconds\n",
-      "[EmbeddingServer LOG]: Time taken for embedding (batch): 0.008869 seconds\n",
-      "[EmbeddingServer LOG]: Time taken for mean pooling (batch): 0.000146 seconds\n",
-      "[EmbeddingServer LOG]: INFO: Serialize time: 0.000063 seconds\n",
-      "[EmbeddingServer LOG]: INFO: ZMQ E2E time: 0.011054 seconds\n",
-      "[EmbeddingServer LOG]: INFO: Received ZMQ request from client 006b8b45, size 7 bytes\n",
-      "[EmbeddingServer LOG]: INFO: Request for 5 node embeddings: [0, 2, 3, 4, 5]\n",
-      "[EmbeddingServer LOG]: DEBUG: Node ID range: 0 to 5\n",
-      "[EmbeddingServer LOG]: Time taken for text lookup: 0.000022 seconds\n",
-      "[EmbeddingServer LOG]: INFO: Total batch size: 5, max_batch_size: 128\n",
-      "[EmbeddingServer LOG]: INFO: Processing batch of size 5\n",
-      "[EmbeddingServer LOG]: Time taken for tokenization (batch): 0.001195 seconds\n",
-      "[EmbeddingServer LOG]: Batch size: 5, Sequence length: 256\n",
-      "[EmbeddingServer LOG]: Time taken for transfer to device (batch): 0.000087 seconds\n",
-      "[EmbeddingServer LOG]: Time taken for embedding (batch): 0.008903 seconds\n",
-      "[EmbeddingServer LOG]: Time taken for mean pooling (batch): 0.000145 seconds\n",
-      "[EmbeddingServer LOG]: INFO: Serialize time: 0.000060 seconds\n",
-      "[EmbeddingServer LOG]: INFO: ZMQ E2E time: 0.010921 seconds\n",
-      "[EmbeddingServer LOG]: INFO: Received ZMQ request from client 006b8b45, size 7 bytes\n",
-      "[EmbeddingServer LOG]: INFO: Request for 5 node embeddings: [1, 0, 3, 4, 5]\n",
-      "[EmbeddingServer LOG]: DEBUG: Node ID range: 0 to 5\n",
-      "[EmbeddingServer LOG]: Time taken for text lookup: 0.000020 seconds\n",
-      "[EmbeddingServer LOG]: INFO: Total batch size: 5, max_batch_size: 128\n",
-      "[EmbeddingServer LOG]: INFO: Processing batch of size 5\n",
-      "[EmbeddingServer LOG]: Time taken for tokenization (batch): 0.001188 seconds\n",
-      "[EmbeddingServer LOG]: Batch size: 5, Sequence length: 256\n",
-      "[EmbeddingServer LOG]: Time taken for transfer to device (batch): 0.000087 seconds\n",
-      "[EmbeddingServer LOG]: Time taken for embedding (batch): 0.008858 seconds\n",
-      "[EmbeddingServer LOG]: Time taken for mean pooling (batch): 0.000153 seconds\n",
-      "[EmbeddingServer LOG]: INFO: Serialize time: 0.000052 seconds\n",
-      "[EmbeddingServer LOG]: INFO: ZMQ E2E time: 0.010886 seconds\n",
-      "reserve ratio: Score: -0.481 - C++ is a powerful programming language1\n",
-      "Graph traversal completed, hops: 3\n",
-      "\n",
-      "Score: -1.049 - Java is a powerful programming language\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "[EmbeddingServer LOG]: INFO: ZMQ socket timeout, continuing to listen\n",
-      "[EmbeddingServer LOG]: INFO: ZMQ socket timeout, continuing to listen\n",
-      "[EmbeddingServer LOG]: INFO: ZMQ socket timeout, continuing to listen\n",
-      "[EmbeddingServer LOG]: INFO: ZMQ socket timeout, continuing to listen\n",
-      "[EmbeddingServer LOG]: INFO: ZMQ socket timeout, continuing to listen\n",
-      "[EmbeddingServer LOG]: INFO: ZMQ socket timeout, continuing to listen\n",
-      "[EmbeddingServer LOG]: INFO: ZMQ socket timeout, continuing to listen\n",
-      "[EmbeddingServer LOG]: INFO: ZMQ socket timeout, continuing to listen\n",
-      "[EmbeddingServer LOG]: INFO: ZMQ socket timeout, continuing to listen\n",
-      "[EmbeddingServer LOG]: INFO: ZMQ socket timeout, continuing to listen\n"
-     ]
-    }
-   ],
    "source": [
-    "from leann.api import LeannBuilder, LeannSearcher\n",
-    "import leann_backend_diskann\n",
-    "# 1. Build index (no embeddings stored!)\n",
-    "builder = LeannBuilder(backend_name=\"diskann\")\n",
-    "builder.add_text(\"Python is a powerful programming language\")\n",
+    "# Quick Start \n",
+    "\n",
+    "**Home GitHub Repository:** [LEANN on GitHub](https://github.com/yichuan-w/LEANN)\n",
+    "\n",
+    "**Important for Colab users:** Set your runtime type to T4 GPU for optimal performance. Go to Runtime → Change runtime type → Hardware accelerator → T4 GPU."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# install this if you are using colab\n",
+    "! uv pip install leann-core leann-backend-hnsw --no-deps\n",
+    "! uv pip install leann --no-deps\n",
+    "# For Colab environment, we need to set some environment variables\n",
+    "import os\n",
+    "\n",
+    "os.environ[\"LEANN_LOG_LEVEL\"] = \"INFO\"  # Enable more detailed logging"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pathlib import Path\n",
+    "\n",
+    "INDEX_DIR = Path(\"./\").resolve()\n",
+    "INDEX_PATH = str(INDEX_DIR / \"demo.leann\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Build the index"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from leann.api import LeannBuilder\n",
+    "\n",
+    "builder = LeannBuilder(backend_name=\"hnsw\")\n",
+    "builder.add_text(\"C# is a powerful programming language and it is good at game development\")\n",
+    "builder.add_text(\n",
+    "    \"Python is a powerful programming language and it is good at machine learning tasks\"\n",
+    ")\n",
     "builder.add_text(\"Machine learning transforms industries\")\n",
     "builder.add_text(\"Neural networks process complex data\")\n",
-    "builder.add_text(\"Java is a powerful programming language\")\n",
-    "builder.add_text(\"C++ is a powerful programming language\")\n",
-    "builder.add_text(\"C# is a powerful programming language\")\n",
-    "builder.build_index(\"knowledge.leann\")\n",
+    "builder.add_text(\"Leann is a great storage saving engine for RAG on your MacBook\")\n",
+    "builder.build_index(INDEX_PATH)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Search with real-time embeddings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from leann.api import LeannSearcher\n",
     "\n",
-    "# 2. Search with real-time embeddings\n",
-    "searcher = LeannSearcher(\"knowledge.leann\")\n",
-    "results = searcher.search(\"C++ programming languages\", top_k=2,recompute_beighbor_embeddings=True)\n",
+    "searcher = LeannSearcher(INDEX_PATH)\n",
+    "results = searcher.search(\"programming languages\", top_k=2)\n",
+    "results"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Chat with LEANN using retrieved results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from leann.api import LeannChat\n",
     "\n",
-    "for result in results:\n",
-    "    print(f\"Score: {result['score']:.3f} - {result['text']}\")"
+    "llm_config = {\n",
+    "    \"type\": \"hf\",\n",
+    "    \"model\": \"Qwen/Qwen3-0.6B\",\n",
+    "}\n",
+    "\n",
+    "chat = LeannChat(index_path=INDEX_PATH, llm_config=llm_config)\n",
+    "response = chat.ask(\n",
+    "    \"Compare the two retrieved programming languages and tell me their advantages.\",\n",
+    "    top_k=2,\n",
+    "    llm_kwargs={\"max_tokens\": 128},\n",
+    ")\n",
+    "response"
    ]
   }
  ],
@@ -376,7 +130,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.11"
+   "version": "3.11.12"
   }
  },
  "nbformat": 4,

docs/CONTRIBUTING.md

@@ -0,0 +1,220 @@
+# 🤝 Contributing
+
+We welcome contributions! Leann is built by the community, for the community.
+
+## Ways to Contribute
+
+- 🐛 **Bug Reports**: Found an issue? Let us know!
+- 💡 **Feature Requests**: Have an idea? We'd love to hear it!
+- 🔧 **Code Contributions**: PRs welcome for all skill levels
+- 📖 **Documentation**: Help make Leann more accessible
+- 🧪 **Benchmarks**: Share your performance results
+
+## 🚀 Development Setup
+
+### Prerequisites
+
+1. **Install uv** (fast Python package installer):
+   ```bash
+   curl -LsSf https://astral.sh/uv/install.sh | sh
+   ```
+
+2. **Clone the repository**:
+   ```bash
+   git clone https://github.com/LEANN-RAG/LEANN-RAG.git
+   cd LEANN-RAG
+   ```
+
+3. **Install system dependencies**:
+
+   **macOS:**
+   ```bash
+   brew install llvm libomp boost protobuf zeromq pkgconf
+   ```
+
+   **Ubuntu/Debian:**
+   ```bash
+   sudo apt-get install libomp-dev libboost-all-dev protobuf-compiler \
+                        libabsl-dev libmkl-full-dev libaio-dev libzmq3-dev
+   ```
+
+4. **Build from source**:
+   ```bash
+   # macOS
+   CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv sync
+
+   # Ubuntu/Debian
+   uv sync
+   ```
+
+## 🔨 Pre-commit Hooks
+
+We use pre-commit hooks to ensure code quality and consistency. This runs automatically before each commit.
+
+### Setup Pre-commit
+
+1. **Install pre-commit** (already included when you run `uv sync`):
+   ```bash
+   uv pip install pre-commit
+   ```
+
+2. **Install the git hooks**:
+   ```bash
+   pre-commit install
+   ```
+
+3. **Run pre-commit manually** (optional):
+   ```bash
+   pre-commit run --all-files
+   ```
+
+### Pre-commit Checks
+
+Our pre-commit configuration includes:
+- **Trailing whitespace removal**
+- **End-of-file fixing**
+- **YAML validation**
+- **Large file prevention**
+- **Merge conflict detection**
+- **Debug statement detection**
+- **Code formatting with ruff**
+- **Code linting with ruff**
+
+## 🧪 Testing
+
+### Running Tests
+
+```bash
+# Run all tests
+uv run pytest
+
+# Run specific test file
+uv run pytest test/test_filename.py
+
+# Run with coverage
+uv run pytest --cov=leann
+```
+
+### Writing Tests
+
+- Place tests in the `test/` directory
+- Follow the naming convention `test_*.py`
+- Use descriptive test names that explain what's being tested
+- Include both positive and negative test cases
+
+## 📝 Code Style
+
+We use `ruff` for both linting and formatting to ensure consistent code style.
+
+### Format Your Code
+
+```bash
+# Format all files
+ruff format
+
+# Check formatting without changing files
+ruff format --check
+```
+
+### Lint Your Code
+
+```bash
+# Run linter with auto-fix
+ruff check --fix
+
+# Just check without fixing
+ruff check
+```
+
+### Style Guidelines
+
+- Follow PEP 8 conventions
+- Use descriptive variable names
+- Add type hints where appropriate
+- Write docstrings for all public functions and classes
+- Keep functions focused and single-purpose
+
+## 🚦 CI/CD
+
+Our CI pipeline runs automatically on all pull requests. It includes:
+
+1. **Linting and Formatting**: Ensures code follows our style guidelines
+2. **Multi-platform builds**: Tests on Ubuntu and macOS
+3. **Python version matrix**: Tests on Python 3.9-3.13
+4. **Wheel building**: Ensures packages can be built and distributed
+
+### CI Commands
+
+The CI uses the same commands as pre-commit to ensure consistency:
+```bash
+# Linting
+ruff check .
+
+# Format checking
+ruff format --check .
+```
+
+Make sure your code passes these checks locally before pushing!
+
+## 🔄 Pull Request Process
+
+1. **Fork the repository** and create your branch from `main`:
+   ```bash
+   git checkout -b feature/your-feature-name
+   ```
+
+2. **Make your changes**:
+   - Write clean, documented code
+   - Add tests for new functionality
+   - Update documentation as needed
+
+3. **Run pre-commit checks**:
+   ```bash
+   pre-commit run --all-files
+   ```
+
+4. **Test your changes**:
+   ```bash
+   uv run pytest
+   ```
+
+5. **Commit with descriptive messages**:
+   ```bash
+   git commit -m "feat: add new search algorithm"
+   ```
+
+   Follow [Conventional Commits](https://www.conventionalcommits.org/):
+   - `feat:` for new features
+   - `fix:` for bug fixes
+   - `docs:` for documentation changes
+   - `test:` for test additions/changes
+   - `refactor:` for code refactoring
+   - `perf:` for performance improvements
+
+6. **Push and create a pull request**:
+   - Provide a clear description of your changes
+   - Reference any related issues
+   - Include examples or screenshots if applicable
+
+## 📚 Documentation
+
+When adding new features or making significant changes:
+
+1. Update relevant documentation in `/docs`
+2. Add docstrings to new functions/classes
+3. Update README.md if needed
+4. Include usage examples
+
+## 🤔 Getting Help
+
+- **Discord**: Join our community for discussions
+- **Issues**: Check existing issues or create a new one
+- **Discussions**: For general questions and ideas
+
+## 📄 License
+
+By contributing, you agree that your contributions will be licensed under the same license as the project (MIT).
+
+---
+
+Thank you for contributing to LEANN! Every contribution, no matter how small, helps make the project better for everyone. 🌟

docs/RELEASE.md

@@ -0,0 +1,22 @@
+# Release Guide
+
+## Setup (One-time)
+
+Add `PYPI_API_TOKEN` to GitHub Secrets:
+1. Get token: https://pypi.org/manage/account/token/
+2. Add to secrets: Settings → Secrets → Actions → `PYPI_API_TOKEN`
+
+## Release (One-click)
+
+1. Go to: https://github.com/yichuan-w/LEANN/actions/workflows/release-manual.yml
+2. Click "Run workflow"
+3. Enter version: `0.1.2`
+4. Click green "Run workflow" button
+
+That's it! The workflow will automatically:
+- ✅ Update version in all packages
+- ✅ Build all packages
+- ✅ Publish to PyPI
+- ✅ Create GitHub tag and release
+
+Check progress: https://github.com/yichuan-w/LEANN/actions

docs/THINKING_BUDGET_FEATURE.md

@@ -0,0 +1,123 @@
+# Thinking Budget Feature Implementation
+
+## Overview
+
+This document describes the implementation of the **thinking budget** feature for LEANN, which allows users to control the computational effort for reasoning models like GPT-Oss:20b.
+
+## Feature Description
+
+The thinking budget feature provides three levels of computational effort for reasoning models:
+- **`low`**: Fast responses, basic reasoning (default for simple queries)
+- **`medium`**: Balanced speed and reasoning depth
+- **`high`**: Maximum reasoning effort, best for complex analytical questions
+
+## Implementation Details
+
+### 1. Command Line Interface
+
+Added `--thinking-budget` parameter to both CLI and RAG examples:
+
+```bash
+# LEANN CLI
+leann ask my-index --llm ollama --model gpt-oss:20b --thinking-budget high
+
+# RAG Examples
+python apps/email_rag.py --llm ollama --llm-model gpt-oss:20b --thinking-budget high
+python apps/document_rag.py --llm openai --llm-model o3 --thinking-budget medium
+```
+
+### 2. LLM Backend Support
+
+#### Ollama Backend (`packages/leann-core/src/leann/chat.py`)
+
+```python
+def ask(self, prompt: str, **kwargs) -> str:
+    # Handle thinking budget for reasoning models
+    options = kwargs.copy()
+    thinking_budget = kwargs.get("thinking_budget")
+    if thinking_budget:
+        options.pop("thinking_budget", None)
+        if thinking_budget in ["low", "medium", "high"]:
+            options["reasoning"] = {"effort": thinking_budget, "exclude": False}
+```
+
+**API Format**: Uses Ollama's `reasoning` parameter with `effort` and `exclude` fields.
+
+#### OpenAI Backend (`packages/leann-core/src/leann/chat.py`)
+
+```python
+def ask(self, prompt: str, **kwargs) -> str:
+    # Handle thinking budget for reasoning models
+    thinking_budget = kwargs.get("thinking_budget")
+    if thinking_budget and thinking_budget in ["low", "medium", "high"]:
+        # Check if this is an o-series model
+        o_series_models = ["o3", "o3-mini", "o4-mini", "o1", "o3-pro", "o3-deep-research"]
+        if any(model in self.model for model in o_series_models):
+            params["reasoning_effort"] = thinking_budget
+```
+
+**API Format**: Uses OpenAI's `reasoning_effort` parameter for o-series models.
+
+### 3. Parameter Propagation
+
+The thinking budget parameter is properly propagated through the LEANN architecture:
+
+1. **CLI** (`packages/leann-core/src/leann/cli.py`): Captures `--thinking-budget` argument
+2. **Base RAG** (`apps/base_rag_example.py`): Adds parameter to argument parser
+3. **LeannChat** (`packages/leann-core/src/leann/api.py`): Passes `llm_kwargs` to LLM
+4. **LLM Interface**: Handles the parameter in backend-specific implementations
+
+## Files Modified
+
+### Core Implementation
+- `packages/leann-core/src/leann/chat.py`: Added thinking budget support to OllamaChat and OpenAIChat
+- `packages/leann-core/src/leann/cli.py`: Added `--thinking-budget` argument
+- `apps/base_rag_example.py`: Added thinking budget parameter to RAG examples
+
+### Documentation
+- `README.md`: Added thinking budget parameter to usage examples
+- `docs/configuration-guide.md`: Added detailed documentation and usage guidelines
+
+### Examples
+- `examples/thinking_budget_demo.py`: Comprehensive demo script with usage examples
+
+## Usage Examples
+
+### Basic Usage
+```bash
+# High reasoning effort for complex questions
+leann ask my-index --llm ollama --model gpt-oss:20b --thinking-budget high
+
+# Medium reasoning for balanced performance
+leann ask my-index --llm openai --model gpt-4o --thinking-budget medium
+
+# Low reasoning for fast responses
+leann ask my-index --llm ollama --model gpt-oss:20b --thinking-budget low
+```
+
+### RAG Examples
+```bash
+# Email RAG with high reasoning
+python apps/email_rag.py --llm ollama --llm-model gpt-oss:20b --thinking-budget high
+
+# Document RAG with medium reasoning
+python apps/document_rag.py --llm openai --llm-model gpt-4o --thinking-budget medium
+```
+
+## Supported Models
+
+### Ollama Models
+- **GPT-Oss:20b**: Primary target model with reasoning capabilities
+- **Other reasoning models**: Any Ollama model that supports the `reasoning` parameter
+
+### OpenAI Models
+- **o3, o3-mini, o4-mini, o1**: o-series reasoning models with `reasoning_effort` parameter
+- **GPT-OSS models**: Models that support reasoning capabilities
+
+## Testing
+
+The implementation includes comprehensive testing:
+- Parameter handling verification
+- Backend-specific API format validation
+- CLI argument parsing tests
+- Integration with existing LEANN architecture

docs/ast_chunking_guide.md

@@ -0,0 +1,128 @@
+# AST-Aware Code chunking guide
+
+## Overview
+
+This guide covers best practices for using AST-aware code chunking in LEANN. AST chunking provides better semantic understanding of code structure compared to traditional text-based chunking.
+
+## Quick Start
+
+### Basic Usage
+
+```bash
+# Enable AST chunking for mixed content (code + docs)
+python -m apps.document_rag --enable-code-chunking --data-dir ./my_project
+
+# Specialized code repository indexing
+python -m apps.code_rag --repo-dir ./my_codebase
+
+# Global CLI with AST support
+leann build my-code-index --docs ./src --use-ast-chunking
+```
+
+### Installation
+
+```bash
+# Install LEANN with AST chunking support
+uv pip install -e "."
+```
+
+## Best Practices
+
+### When to Use AST Chunking
+
+✅ **Recommended for:**
+- Code repositories with multiple languages
+- Mixed documentation and code content
+- Complex codebases with deep function/class hierarchies
+- When working with Claude Code for code assistance
+
+❌ **Not recommended for:**
+- Pure text documents
+- Very large files (>1MB)
+- Languages not supported by tree-sitter
+
+### Optimal Configuration
+
+```bash
+# Recommended settings for most codebases
+python -m apps.code_rag \
+    --repo-dir ./src \
+    --ast-chunk-size 768 \
+    --ast-chunk-overlap 96 \
+    --exclude-dirs .git __pycache__ node_modules build dist
+```
+
+### Supported Languages
+
+| Extension | Language | Status |
+|-----------|----------|--------|
+| `.py` | Python | ✅ Full support |
+| `.java` | Java | ✅ Full support |
+| `.cs` | C# | ✅ Full support |
+| `.ts`, `.tsx` | TypeScript | ✅ Full support |
+| `.js`, `.jsx` | JavaScript | ✅ Via TypeScript parser |
+
+## Integration Examples
+
+### Document RAG with Code Support
+
+```python
+# Enable code chunking in document RAG
+python -m apps.document_rag \
+    --enable-code-chunking \
+    --data-dir ./project \
+    --query "How does authentication work in the codebase?"
+```
+
+### Claude Code Integration
+
+When using with Claude Code MCP server, AST chunking provides better context for:
+- Code completion and suggestions
+- Bug analysis and debugging
+- Architecture understanding
+- Refactoring assistance
+
+## Troubleshooting
+
+### Common Issues
+
+1. **Fallback to Traditional Chunking**
+   - Normal behavior for unsupported languages
+   - Check logs for specific language support
+
+2. **Performance with Large Files**
+   - Adjust `--max-file-size` parameter
+   - Use `--exclude-dirs` to skip unnecessary directories
+
+3. **Quality Issues**
+   - Try different `--ast-chunk-size` values (512, 768, 1024)
+   - Adjust overlap for better context preservation
+
+### Debug Mode
+
+```bash
+export LEANN_LOG_LEVEL=DEBUG
+python -m apps.code_rag --repo-dir ./my_code
+```
+
+## Migration from Traditional Chunking
+
+Existing workflows continue to work without changes. To enable AST chunking:
+
+```bash
+# Before
+python -m apps.document_rag --chunk-size 256
+
+# After (maintains traditional chunking for non-code files)
+python -m apps.document_rag --enable-code-chunking --chunk-size 256 --ast-chunk-size 768
+```
+
+## References
+
+- [astchunk GitHub Repository](https://github.com/yilinjz/astchunk)
+- [LEANN MCP Integration](../packages/leann-mcp/README.md)
+- [Research Paper](https://arxiv.org/html/2506.15655v1)
+
+---
+
+**Note**: AST chunking maintains full backward compatibility while enhancing code understanding capabilities.

docs/code/embedding_model_compare.py

@@ -0,0 +1,98 @@
+"""
+Comparison between Sentence Transformers and OpenAI embeddings
+
+This example shows how different embedding models handle complex queries
+and demonstrates the differences between local and API-based embeddings.
+"""
+
+import numpy as np
+from leann.embedding_compute import compute_embeddings
+
+# OpenAI API key should be set as environment variable
+# export OPENAI_API_KEY="your-api-key-here"
+
+# Test data
+conference_text = "[Title]: COLING 2025 Conference\n[URL]: https://coling2025.org/"
+browser_text = "[Title]: Browser Use Tool\n[URL]: https://github.com/browser-use"
+
+# Two queries with same intent but different wording
+query1 = "Tell me my browser history about some conference i often visit"
+query2 = "browser history about conference I often visit"
+
+texts = [query1, query2, conference_text, browser_text]
+
+
+def cosine_similarity(a, b):
+    return np.dot(a, b)  # Already normalized
+
+
+def analyze_embeddings(embeddings, model_name):
+    print(f"\n=== {model_name} Results ===")
+
+    # Results for Query 1
+    sim1_conf = cosine_similarity(embeddings[0], embeddings[2])
+    sim1_browser = cosine_similarity(embeddings[0], embeddings[3])
+
+    print(f"Query 1: '{query1}'")
+    print(f"  → Conference similarity: {sim1_conf:.4f} {'✓' if sim1_conf > sim1_browser else ''}")
+    print(
+        f"  → Browser similarity:    {sim1_browser:.4f} {'✓' if sim1_browser > sim1_conf else ''}"
+    )
+    print(f"  Winner: {'Conference' if sim1_conf > sim1_browser else 'Browser'}")
+
+    # Results for Query 2
+    sim2_conf = cosine_similarity(embeddings[1], embeddings[2])
+    sim2_browser = cosine_similarity(embeddings[1], embeddings[3])
+
+    print(f"\nQuery 2: '{query2}'")
+    print(f"  → Conference similarity: {sim2_conf:.4f} {'✓' if sim2_conf > sim2_browser else ''}")
+    print(
+        f"  → Browser similarity:    {sim2_browser:.4f} {'✓' if sim2_browser > sim2_conf else ''}"
+    )
+    print(f"  Winner: {'Conference' if sim2_conf > sim2_browser else 'Browser'}")
+
+    # Show the impact
+    print("\n=== Impact Analysis ===")
+    print(f"Conference similarity change: {sim2_conf - sim1_conf:+.4f}")
+    print(f"Browser similarity change:    {sim2_browser - sim1_browser:+.4f}")
+
+    if sim1_conf > sim1_browser and sim2_browser > sim2_conf:
+        print("❌ FLIP: Adding 'browser history' flips winner from Conference to Browser!")
+    elif sim1_conf > sim1_browser and sim2_conf > sim2_browser:
+        print("✅ STABLE: Conference remains winner in both queries")
+    elif sim1_browser > sim1_conf and sim2_browser > sim2_conf:
+        print("✅ STABLE: Browser remains winner in both queries")
+    else:
+        print("🔄 MIXED: Results vary between queries")
+
+    return {
+        "query1_conf": sim1_conf,
+        "query1_browser": sim1_browser,
+        "query2_conf": sim2_conf,
+        "query2_browser": sim2_browser,
+    }
+
+
+# Test Sentence Transformers
+print("Testing Sentence Transformers (facebook/contriever)...")
+try:
+    st_embeddings = compute_embeddings(texts, "facebook/contriever", mode="sentence-transformers")
+    st_results = analyze_embeddings(st_embeddings, "Sentence Transformers (facebook/contriever)")
+except Exception as e:
+    print(f"❌ Sentence Transformers failed: {e}")
+    st_results = None
+
+# Test OpenAI
+print("\n" + "=" * 60)
+print("Testing OpenAI (text-embedding-3-small)...")
+try:
+    openai_embeddings = compute_embeddings(texts, "text-embedding-3-small", mode="openai")
+    openai_results = analyze_embeddings(openai_embeddings, "OpenAI (text-embedding-3-small)")
+except Exception as e:
+    print(f"❌ OpenAI failed: {e}")
+    openai_results = None
+
+# Compare results
+if st_results and openai_results:
+    print("\n" + "=" * 60)
+    print("=== COMPARISON SUMMARY ===")

docs/configuration-guide.md

@@ -0,0 +1,384 @@
+# LEANN Configuration Guide
+
+This guide helps you optimize LEANN for different use cases and understand the trade-offs between various configuration options.
+
+## Getting Started: Simple is Better
+
+When first trying LEANN, start with a small dataset to quickly validate your approach:
+
+**For document RAG**: The default `data/` directory works perfectly - includes 2 AI research papers, Pride and Prejudice literature, and a technical report
+```bash
+python -m apps.document_rag --query "What techniques does LEANN use?"
+```
+
+**For other data sources**: Limit the dataset size for quick testing
+```bash
+# WeChat: Test with recent messages only
+python -m apps.wechat_rag --max-items 100 --query "What did we discuss about the project timeline?"
+
+# Browser history: Last few days
+python -m apps.browser_rag --max-items 500 --query "Find documentation about vector databases"
+
+# Email: Recent inbox
+python -m apps.email_rag --max-items 200 --query "Who sent updates about the deployment status?"
+```
+
+Once validated, scale up gradually:
+- 100 documents → 1,000 → 10,000 → full dataset (`--max-items -1`)
+- This helps identify issues early before committing to long processing times
+
+## Embedding Model Selection: Understanding the Trade-offs
+
+Based on our experience developing LEANN, embedding models fall into three categories:
+
+### Small Models (< 100M parameters)
+**Example**: `sentence-transformers/all-MiniLM-L6-v2` (22M params)
+- **Pros**: Lightweight, fast for both indexing and inference
+- **Cons**: Lower semantic understanding, may miss nuanced relationships
+- **Use when**: Speed is critical, handling simple queries, interactive mode, or just experimenting with LEANN. If time is not a constraint, consider using a larger/better embedding model
+
+### Medium Models (100M-500M parameters)
+**Example**: `facebook/contriever` (110M params), `BAAI/bge-base-en-v1.5` (110M params)
+- **Pros**: Balanced performance, good multilingual support, reasonable speed
+- **Cons**: Requires more compute than small models
+- **Use when**: Need quality results without extreme compute requirements, general-purpose RAG applications
+
+### Large Models (500M+ parameters)
+**Example**: `Qwen/Qwen3-Embedding-0.6B` (600M params), `intfloat/multilingual-e5-large` (560M params)
+- **Pros**: Best semantic understanding, captures complex relationships, excellent multilingual support. **Qwen3-Embedding-0.6B achieves nearly OpenAI API performance!**
+- **Cons**: Slower inference, longer index build times
+- **Use when**: Quality is paramount and you have sufficient compute resources. **Highly recommended** for production use
+
+### Quick Start: Cloud and Local Embedding Options
+
+**OpenAI Embeddings (Fastest Setup)**
+For immediate testing without local model downloads(also if you [do not have GPU](https://github.com/yichuan-w/LEANN/issues/43) and do not care that much about your document leak, you should use this, we compute the embedding and recompute using openai API):
+```bash
+# Set OpenAI embeddings (requires OPENAI_API_KEY)
+--embedding-mode openai --embedding-model text-embedding-3-small
+```
+
+**Ollama Embeddings (Privacy-Focused)**
+For local embeddings with complete privacy:
+```bash
+# First, pull an embedding model
+ollama pull nomic-embed-text
+
+# Use Ollama embeddings
+--embedding-mode ollama --embedding-model nomic-embed-text
+```
+
+<details>
+<summary><strong>Cloud vs Local Trade-offs</strong></summary>
+
+**OpenAI Embeddings** (`text-embedding-3-small/large`)
+- **Pros**: No local compute needed, consistently fast, high quality
+- **Cons**: Requires API key, costs money, data leaves your system, [known limitations with certain languages](https://yichuan-w.github.io/blog/lessons_learned_in_dev_leann/)
+- **When to use**: Prototyping, non-sensitive data, need immediate results
+
+**Local Embeddings**
+- **Pros**: Complete privacy, no ongoing costs, full control, can sometimes outperform OpenAI embeddings
+- **Cons**: Slower than cloud APIs, requires local compute resources
+- **When to use**: Production systems, sensitive data, cost-sensitive applications
+
+</details>
+
+## Index Selection: Matching Your Scale
+
+### HNSW (Hierarchical Navigable Small World)
+**Best for**: Small to medium datasets (< 10M vectors) - **Default and recommended for extreme low storage**
+- Full recomputation required
+- High memory usage during build phase
+- Excellent recall (95%+)
+
+```bash
+# Optimal for most use cases
+--backend-name hnsw --graph-degree 32 --build-complexity 64
+```
+
+### DiskANN
+**Best for**: Large datasets, especially when you want `recompute=True`.
+
+**Key advantages:**
+- **Faster search** on large datasets (3x+ speedup vs HNSW in many cases)
+- **Smart storage**: `recompute=True` enables automatic graph partitioning for smaller indexes
+- **Better scaling**: Designed for 100k+ documents
+
+**Recompute behavior:**
+- `recompute=True` (recommended): Pure PQ traversal + final reranking - faster and enables partitioning
+- `recompute=False`: PQ + partial real distances during traversal - slower but higher accuracy
+
+```bash
+# Recommended for most use cases
+--backend-name diskann --graph-degree 32 --build-complexity 64
+```
+
+**Performance Benchmark**: Run `uv run benchmarks/diskann_vs_hnsw_speed_comparison.py` to compare DiskANN and HNSW on your system.
+
+## LLM Selection: Engine and Model Comparison
+
+### LLM Engines
+
+**OpenAI** (`--llm openai`)
+- **Pros**: Best quality, consistent performance, no local resources needed
+- **Cons**: Costs money ($0.15-2.5 per million tokens), requires internet, data privacy concerns
+- **Models**: `gpt-4o-mini` (fast, cheap), `gpt-4o` (best quality), `o3` (reasoning), `o3-mini` (reasoning, cheaper)
+- **Thinking Budget**: Use `--thinking-budget low/medium/high` for o-series reasoning models (o3, o3-mini, o4-mini)
+- **Note**: Our current default, but we recommend switching to Ollama for most use cases
+
+**Ollama** (`--llm ollama`)
+- **Pros**: Fully local, free, privacy-preserving, good model variety
+- **Cons**: Requires local GPU/CPU resources, slower than cloud APIs, need to install extra [ollama app](https://github.com/ollama/ollama?tab=readme-ov-file#ollama) and pre-download models by `ollama pull`
+- **Models**: `qwen3:0.6b` (ultra-fast), `qwen3:1.7b` (balanced), `qwen3:4b` (good quality), `qwen3:7b` (high quality), `deepseek-r1:1.5b` (reasoning)
+- **Thinking Budget**: Use `--thinking-budget low/medium/high` for reasoning models like GPT-Oss:20b
+
+**HuggingFace** (`--llm hf`)
+- **Pros**: Free tier available, huge model selection, direct model loading (vs Ollama's server-based approach)
+- **Cons**: More complex initial setup
+- **Models**: `Qwen/Qwen3-1.7B-FP8`
+
+## Parameter Tuning Guide
+
+### Search Complexity Parameters
+
+**`--build-complexity`** (index building)
+- Controls thoroughness during index construction
+- Higher = better recall but slower build
+- Recommendations:
+  - 32: Quick prototyping
+  - 64: Balanced (default)
+  - 128: Production systems
+  - 256: Maximum quality
+
+**`--search-complexity`** (query time)
+- Controls search thoroughness
+- Higher = better results but slower
+- Recommendations:
+  - 16: Fast/Interactive search
+  - 32: High quality with diversity
+  - 64+: Maximum accuracy
+
+### Top-K Selection
+
+**`--top-k`** (number of retrieved chunks)
+- More chunks = better context but slower LLM processing
+- Should be always smaller than `--search-complexity`
+- Guidelines:
+  - 10-20: General questions (default: 20)
+  - 30+: Complex multi-hop reasoning requiring comprehensive context
+
+**Trade-off formula**:
+- Retrieval time ∝ log(n) × search_complexity
+- LLM processing time ∝ top_k × chunk_size
+- Total context = top_k × chunk_size tokens
+
+### Thinking Budget for Reasoning Models
+
+**`--thinking-budget`** (reasoning effort level)
+- Controls the computational effort for reasoning models
+- Options: `low`, `medium`, `high`
+- Guidelines:
+  - `low`: Fast responses, basic reasoning (default for simple queries)
+  - `medium`: Balanced speed and reasoning depth
+  - `high`: Maximum reasoning effort, best for complex analytical questions
+- **Supported Models**:
+  - **Ollama**: `gpt-oss:20b`, `gpt-oss:120b`
+  - **OpenAI**: `o3`, `o3-mini`, `o4-mini`, `o1` (o-series reasoning models)
+- **Note**: Models without reasoning support will show a warning and proceed without reasoning parameters
+- **Example**: `--thinking-budget high` for complex analytical questions
+
+**📖 For detailed usage examples and implementation details, check out [Thinking Budget Documentation](THINKING_BUDGET_FEATURE.md)**
+
+**💡 Quick Examples:**
+```bash
+# OpenAI o-series reasoning model
+python apps/document_rag.py --query "What are the main techniques LEANN explores?" \
+  --index-dir hnswbuild --backend hnsw \
+  --llm openai --llm-model o3 --thinking-budget medium
+
+# Ollama reasoning model
+python apps/document_rag.py --query "What are the main techniques LEANN explores?" \
+  --index-dir hnswbuild --backend hnsw \
+  --llm ollama --llm-model gpt-oss:20b --thinking-budget high
+```
+
+### Graph Degree (HNSW/DiskANN)
+
+**`--graph-degree`**
+- Number of connections per node in the graph
+- Higher = better recall but more memory
+- HNSW: 16-32 (default: 32)
+- DiskANN: 32-128 (default: 64)
+
+
+## Performance Optimization Checklist
+
+### If Embedding is Too Slow
+
+1. **Switch to smaller model**:
+   ```bash
+   # From large model
+   --embedding-model Qwen/Qwen3-Embedding-0.6B
+   # To small model
+   --embedding-model sentence-transformers/all-MiniLM-L6-v2
+   ```
+
+2. **Limit dataset size for testing**:
+   ```bash
+   --max-items 1000  # Process first 1k items only
+   ```
+
+3. **Use MLX on Apple Silicon** (optional optimization):
+   ```bash
+   --embedding-mode mlx --embedding-model mlx-community/Qwen3-Embedding-0.6B-8bit
+   ```
+    MLX might not be the best choice, as we tested and found that it only offers 1.3x acceleration compared to HF, so maybe using ollama is a better choice for embedding generation
+
+4. **Use Ollama**
+   ```bash
+   --embedding-mode ollama --embedding-model nomic-embed-text
+   ```
+   To discover additional embedding models in ollama, check out https://ollama.com/search?c=embedding or read more about embedding models at https://ollama.com/blog/embedding-models, please do check the model size that works best for you
+### If Search Quality is Poor
+
+1. **Increase retrieval count**:
+   ```bash
+   --top-k 30  # Retrieve more candidates
+   ```
+
+2. **Upgrade embedding model**:
+   ```bash
+   # For English
+   --embedding-model BAAI/bge-base-en-v1.5
+   # For multilingual
+   --embedding-model intfloat/multilingual-e5-large
+   ```
+
+## Understanding the Trade-offs
+
+Every configuration choice involves trade-offs:
+
+| Factor | Small/Fast | Large/Quality |
+|--------|------------|---------------|
+| Embedding Model | `all-MiniLM-L6-v2` | `Qwen/Qwen3-Embedding-0.6B` |
+| Chunk Size | 512 tokens | 128 tokens |
+| Index Type | HNSW | DiskANN |
+| LLM | `qwen3:1.7b` | `gpt-4o` |
+
+The key is finding the right balance for your specific use case. Start small and simple, measure performance, then scale up only where needed.
+
+## Low-resource setups
+
+If you don’t have a local GPU or builds/searches are too slow, use one or more of the options below.
+
+### 1) Use OpenAI embeddings (no local compute)
+
+Fastest path with zero local GPU requirements. Set your API key and use OpenAI embeddings during build and search:
+
+```bash
+export OPENAI_API_KEY=sk-...
+
+# Build with OpenAI embeddings
+leann build my-index \
+  --embedding-mode openai \
+  --embedding-model text-embedding-3-small
+
+# Search with OpenAI embeddings (recompute at query time)
+leann search my-index "your query" \
+  --recompute
+```
+
+### 2) Run remote builds with SkyPilot (cloud GPU)
+
+Offload embedding generation and index building to a GPU VM using [SkyPilot](https://skypilot.readthedocs.io/en/latest/). A template is provided at `sky/leann-build.yaml`.
+
+```bash
+# One-time: install and configure SkyPilot
+pip install skypilot
+
+# Launch with defaults (L4:1) and mount ./data to ~/leann-data; the build runs automatically
+sky launch -c leann-gpu sky/leann-build.yaml
+
+# Override parameters via -e key=value (optional)
+sky launch -c leann-gpu sky/leann-build.yaml \
+  -e index_name=my-index \
+  -e backend=hnsw \
+  -e embedding_mode=sentence-transformers \
+  -e embedding_model=Qwen/Qwen3-Embedding-0.6B
+
+# Copy the built index back to your local .leann (use rsync)
+rsync -Pavz leann-gpu:~/.leann/indexes/my-index ./.leann/indexes/
+```
+
+### 3) Disable recomputation to trade storage for speed
+
+If you need lower latency and have more storage/memory, disable recomputation. This stores full embeddings and avoids recomputing at search time.
+
+```bash
+# Build without recomputation (HNSW requires non-compact in this mode)
+leann build my-index --no-recompute --no-compact
+
+# Search without recomputation
+leann search my-index "your query" --no-recompute
+```
+
+When to use:
+- Extreme low latency requirements (high QPS, interactive assistants)
+- Read-heavy workloads where storage is cheaper than latency
+- No always-available GPU
+
+Constraints:
+- HNSW: when `--no-recompute` is set, LEANN automatically disables compact mode during build
+- DiskANN: supported; `--no-recompute` skips selective recompute during search
+
+Storage impact:
+- Storing N embeddings of dimension D with float32 requires approximately N × D × 4 bytes
+- Example: 1,000,000 chunks × 768 dims × 4 bytes ≈ 2.86 GB (plus graph/metadata)
+
+Converting an existing index (rebuild required):
+```bash
+# Rebuild in-place (ensure you still have original docs or can regenerate chunks)
+leann build my-index --force --no-recompute --no-compact
+```
+
+Python API usage:
+```python
+from leann import LeannSearcher
+
+searcher = LeannSearcher("/path/to/my-index.leann")
+results = searcher.search("your query", top_k=10, recompute_embeddings=False)
+```
+
+Trade-offs:
+- Lower latency and fewer network hops at query time
+- Significantly higher storage (10–100× vs selective recomputation)
+- Slightly larger memory footprint during build and search
+
+Quick benchmark results (`benchmarks/benchmark_no_recompute.py` with 5k texts, complexity=32):
+
+- HNSW
+
+  ```text
+  recompute=True:  search_time=0.818s, size=1.1MB
+  recompute=False: search_time=0.012s, size=16.6MB
+  ```
+
+- DiskANN
+
+  ```text
+  recompute=True:  search_time=0.041s, size=5.9MB
+  recompute=False: search_time=0.013s, size=24.6MB
+  ```
+
+Conclusion:
+- **HNSW**: `no-recompute` is significantly faster (no embedding recomputation) but requires much more storage (stores all embeddings)
+- **DiskANN**: `no-recompute` uses PQ + partial real distances during traversal (slower but higher accuracy), while `recompute=True` uses pure PQ traversal + final reranking (faster traversal, enables build-time partitioning for smaller storage)
+
+
+
+## Further Reading
+
+- [Lessons Learned Developing LEANN](https://yichuan-w.github.io/blog/lessons_learned_in_dev_leann/)
+- [LEANN Technical Paper](https://arxiv.org/abs/2506.08276)
+- [DiskANN Original Paper](https://papers.nips.cc/paper/2019/file/09853c7fb1d3f8ee67a61b6bf4a7f8e6-Paper.pdf)
+- [SSD-based Graph Partitioning](https://github.com/SonglinLife/SSD_BASED_PLAN)

docs/faq.md

@@ -0,0 +1,10 @@
+# FAQ
+
+## 1. My building time seems long
+
+You can speed up the process by using a lightweight embedding model. Add this to your arguments:
+
+```bash
+--embedding-model sentence-transformers/all-MiniLM-L6-v2
+```
+**Model sizes:** `all-MiniLM-L6-v2` (30M parameters), `facebook/contriever` (~100M parameters), `Qwen3-0.6B` (600M parameters)

docs/features.md

@@ -0,0 +1,23 @@
+# ✨ Detailed Features
+
+## 🔥 Core Features
+
+- **🔄 Real-time Embeddings** - Eliminate heavy embedding storage with dynamic computation using optimized ZMQ servers and highly optimized search paradigm (overlapping and batching) with highly optimized embedding engine
+- **🧠 AST-Aware Code Chunking** - Intelligent code chunking that preserves semantic boundaries (functions, classes, methods) for Python, Java, C#, and TypeScript files
+- **📈 Scalable Architecture** - Handles millions of documents on consumer hardware; the larger your dataset, the more LEANN can save
+- **🎯 Graph Pruning** - Advanced techniques to minimize the storage overhead of vector search to a limited footprint
+- **🏗️ Pluggable Backends** - HNSW/FAISS (default), with optional DiskANN for large-scale deployments
+
+## 🛠️ Technical Highlights
+- **🔄 Recompute Mode** - Highest accuracy scenarios while eliminating vector storage overhead
+- **⚡ Zero-copy Operations** - Minimize IPC overhead by transferring distances instead of embeddings
+- **🚀 High-throughput Embedding Pipeline** - Optimized batched processing for maximum efficiency
+- **🎯 Two-level Search** - Novel coarse-to-fine search overlap for accelerated query processing (optional)
+- **💾 Memory-mapped Indices** - Fast startup with raw text mapping to reduce memory overhead
+- **🚀 MLX Support** - Ultra-fast recompute/build with quantized embedding models, accelerating building and search ([minimal example](../examples/mlx_demo.py))
+
+## 🎨 Developer Experience
+
+- **Simple Python API** - Get started in minutes
+- **Extensible backend system** - Easy to add new algorithms
+- **Comprehensive examples** - From basic usage to production deployment

docs/metadata_filtering.md

@@ -0,0 +1,300 @@
+# LEANN Metadata Filtering Usage Guide
+
+## Overview
+
+Leann possesses metadata filtering capabilities that allow you to filter search results based on arbitrary metadata fields set during chunking. This feature enables use cases like spoiler-free book search, document filtering by date/type, code search by file type, and potentially much more.
+
+## Basic Usage
+
+### Adding Metadata to Your Documents
+
+When building your index, add metadata to each text chunk:
+
+```python
+from leann.api import LeannBuilder
+
+builder = LeannBuilder("hnsw")
+
+# Add text with metadata
+builder.add_text(
+    text="Chapter 1: Alice falls down the rabbit hole",
+    metadata={
+        "chapter": 1,
+        "character": "Alice",
+        "themes": ["adventure", "curiosity"],
+        "word_count": 150
+    }
+)
+
+builder.build_index("alice_in_wonderland_index")
+```
+
+### Searching with Metadata Filters
+
+Use the `metadata_filters` parameter in search calls:
+
+```python
+from leann.api import LeannSearcher
+
+searcher = LeannSearcher("alice_in_wonderland_index")
+
+# Search with filters
+results = searcher.search(
+    query="What happens to Alice?",
+    top_k=10,
+    metadata_filters={
+        "chapter": {"<=": 5},           # Only chapters 1-5
+        "spoiler_level": {"!=": "high"} # No high spoilers
+    }
+)
+```
+
+## Filter Syntax
+
+### Basic Structure
+
+```python
+metadata_filters = {
+    "field_name": {"operator": value},
+    "another_field": {"operator": value}
+}
+```
+
+### Supported Operators
+
+#### Comparison Operators
+- `"=="`: Equal to
+- `"!="`: Not equal to
+- `"<"`: Less than
+- `"<="`: Less than or equal
+- `">"`: Greater than
+- `">="`: Greater than or equal
+
+```python
+# Examples
+{"chapter": {"==": 1}}           # Exactly chapter 1
+{"page": {">": 100}}            # Pages after 100
+{"rating": {">=": 4.0}}         # Rating 4.0 or higher
+{"word_count": {"<": 500}}      # Short passages
+```
+
+#### Membership Operators
+- `"in"`: Value is in list
+- `"not_in"`: Value is not in list
+
+```python
+# Examples
+{"character": {"in": ["Alice", "Bob"]}}      # Alice OR Bob
+{"genre": {"not_in": ["horror", "thriller"]}} # Exclude genres
+{"tags": {"in": ["fiction", "adventure"]}}   # Any of these tags
+```
+
+#### String Operators
+- `"contains"`: String contains substring
+- `"starts_with"`: String starts with prefix
+- `"ends_with"`: String ends with suffix
+
+```python
+# Examples
+{"title": {"contains": "alice"}}        # Title contains "alice"
+{"filename": {"ends_with": ".py"}}      # Python files
+{"author": {"starts_with": "Dr."}}      # Authors with "Dr." prefix
+```
+
+#### Boolean Operators
+- `"is_true"`: Field is truthy
+- `"is_false"`: Field is falsy
+
+```python
+# Examples
+{"is_published": {"is_true": True}}     # Published content
+{"is_draft": {"is_false": False}}       # Not drafts
+```
+
+### Multiple Operators on Same Field
+
+You can apply multiple operators to the same field (AND logic):
+
+```python
+metadata_filters = {
+    "word_count": {
+        ">=": 100,    # At least 100 words
+        "<=": 500     # At most 500 words
+    }
+}
+```
+
+### Compound Filters
+
+Multiple fields are combined with AND logic:
+
+```python
+metadata_filters = {
+    "chapter": {"<=": 10},              # Up to chapter 10
+    "character": {"==": "Alice"},       # About Alice
+    "spoiler_level": {"!=": "high"}     # No major spoilers
+}
+```
+
+## Use Case Examples
+
+### 1. Spoiler-Free Book Search
+
+```python
+# Reader has only read up to chapter 5
+def search_spoiler_free(query, max_chapter):
+    return searcher.search(
+        query=query,
+        metadata_filters={
+            "chapter": {"<=": max_chapter},
+            "spoiler_level": {"in": ["none", "low"]}
+        }
+    )
+
+results = search_spoiler_free("What happens to Alice?", max_chapter=5)
+```
+
+### 2. Document Management by Date
+
+```python
+# Find recent documents
+recent_docs = searcher.search(
+    query="project updates",
+    metadata_filters={
+        "date": {">=": "2024-01-01"},
+        "document_type": {"==": "report"}
+    }
+)
+```
+
+### 3. Code Search by File Type
+
+```python
+# Search only Python files
+python_code = searcher.search(
+    query="authentication function",
+    metadata_filters={
+        "file_extension": {"==": ".py"},
+        "lines_of_code": {"<": 100}
+    }
+)
+```
+
+### 4. Content Filtering by Audience
+
+```python
+# Age-appropriate content
+family_content = searcher.search(
+    query="adventure stories",
+    metadata_filters={
+        "age_rating": {"in": ["G", "PG"]},
+        "content_warnings": {"not_in": ["violence", "adult_themes"]}
+    }
+)
+```
+
+### 5. Multi-Book Series Management
+
+```python
+# Search across first 3 books only
+early_series = searcher.search(
+    query="character development",
+    metadata_filters={
+        "series": {"==": "Harry Potter"},
+        "book_number": {"<=": 3}
+    }
+)
+```
+
+## Running the Example
+
+You can see metadata filtering in action with our spoiler-free book RAG example:
+
+```bash
+# Don't forget to set up the environment
+uv venv
+source .venv/bin/activate
+
+# Set your OpenAI API key (required for embeddings, but you can update the example locally and use ollama instead)
+export OPENAI_API_KEY="your-api-key-here"
+
+# Run the spoiler-free book RAG example
+uv run examples/spoiler_free_book_rag.py
+```
+
+This example demonstrates:
+- Building an index with metadata (chapter numbers, characters, themes, locations)
+- Searching with filters to avoid spoilers (e.g., only show results up to chapter 5)
+- Different scenarios for readers at various points in the book
+
+The example uses Alice's Adventures in Wonderland as sample data and shows how you can search for information without revealing plot points from later chapters.
+
+## Advanced Patterns
+
+### Custom Chunking with metadata
+
+```python
+def chunk_book_with_metadata(book_text, book_info):
+    chunks = []
+
+    for chapter_num, chapter_text in parse_chapters(book_text):
+        # Extract entities, themes, etc.
+        characters = extract_characters(chapter_text)
+        themes = classify_themes(chapter_text)
+        spoiler_level = assess_spoiler_level(chapter_text, chapter_num)
+
+        # Create chunks with rich metadata
+        for paragraph in split_paragraphs(chapter_text):
+            chunks.append({
+                "text": paragraph,
+                "metadata": {
+                    "book_title": book_info["title"],
+                    "chapter": chapter_num,
+                    "characters": characters,
+                    "themes": themes,
+                    "spoiler_level": spoiler_level,
+                    "word_count": len(paragraph.split()),
+                    "reading_level": calculate_reading_level(paragraph)
+                }
+            })
+
+    return chunks
+```
+
+## Performance Considerations
+
+### Efficient Filtering Strategies
+
+1. **Post-search filtering**: Applies filters after vector search, which should be efficient for typical result sets (10-100 results).
+
+2. **Metadata design**: Keep metadata fields simple and avoid deeply nested structures.
+
+### Best Practices
+
+1. **Consistent metadata schema**: Use consistent field names and value types across your documents.
+
+2. **Reasonable metadata size**: Keep metadata reasonably sized to avoid storage overhead.
+
+3. **Type consistency**: Use consistent data types for the same fields (e.g., always integers for chapter numbers).
+
+4. **Index multiple granularities**: Consider chunking at different levels (paragraph, section, chapter) with appropriate metadata.
+
+### Adding Metadata to Existing Indices
+
+To add metadata filtering to existing indices, you'll need to rebuild them with metadata:
+
+```python
+# Read existing passages and add metadata
+def add_metadata_to_existing_chunks(chunks):
+    for chunk in chunks:
+        # Extract or assign metadata based on content
+        chunk["metadata"] = extract_metadata(chunk["text"])
+    return chunks
+
+# Rebuild index with metadata
+enhanced_chunks = add_metadata_to_existing_chunks(existing_chunks)
+builder = LeannBuilder("hnsw")
+for chunk in enhanced_chunks:
+    builder.add_text(chunk["text"], chunk["metadata"])
+builder.build_index("enhanced_index")
+```

docs/normalized_embeddings.md

@@ -0,0 +1,75 @@
+# Normalized Embeddings Support in LEANN
+
+LEANN now automatically detects normalized embedding models and sets the appropriate distance metric for optimal performance.
+
+## What are Normalized Embeddings?
+
+Normalized embeddings are vectors with L2 norm = 1 (unit vectors). These embeddings are optimized for cosine similarity rather than Maximum Inner Product Search (MIPS).
+
+## Automatic Detection
+
+When you create a `LeannBuilder` instance with a normalized embedding model, LEANN will:
+
+1. **Automatically set `distance_metric="cosine"`** if not specified
+2. **Show a warning** if you manually specify a different distance metric
+3. **Provide optimal search performance** with the correct metric
+
+## Supported Normalized Embedding Models
+
+### OpenAI
+All OpenAI text embedding models are normalized:
+- `text-embedding-ada-002`
+- `text-embedding-3-small`
+- `text-embedding-3-large`
+
+### Voyage AI
+All Voyage AI embedding models are normalized:
+- `voyage-2`
+- `voyage-3`
+- `voyage-large-2`
+- `voyage-multilingual-2`
+- `voyage-code-2`
+
+### Cohere
+All Cohere embedding models are normalized:
+- `embed-english-v3.0`
+- `embed-multilingual-v3.0`
+- `embed-english-light-v3.0`
+- `embed-multilingual-light-v3.0`
+
+## Example Usage
+
+```python
+from leann.api import LeannBuilder
+
+# Automatic detection - will use cosine distance
+builder = LeannBuilder(
+    backend_name="hnsw",
+    embedding_model="text-embedding-3-small",
+    embedding_mode="openai"
+)
+# Warning: Detected normalized embeddings model 'text-embedding-3-small'...
+# Automatically setting distance_metric='cosine'
+
+# Manual override (not recommended)
+builder = LeannBuilder(
+    backend_name="hnsw",
+    embedding_model="text-embedding-3-small",
+    embedding_mode="openai",
+    distance_metric="mips"  # Will show warning
+)
+# Warning: Using 'mips' distance metric with normalized embeddings...
+```
+
+## Non-Normalized Embeddings
+
+Models like `facebook/contriever` and other sentence-transformers models that are not normalized will continue to use MIPS by default, which is optimal for them.
+
+## Why This Matters
+
+Using the wrong distance metric with normalized embeddings can lead to:
+- **Poor search quality** due to HNSW's early termination with narrow score ranges
+- **Incorrect ranking** of search results
+- **Suboptimal performance** compared to using the correct metric
+
+For more details on why this happens, see our analysis in the [embedding detection code](../packages/leann-core/src/leann/api.py) which automatically handles normalized embeddings and MIPS distance metric issues.

docs/roadmap.md

@@ -0,0 +1,21 @@
+# 📈 Roadmap
+
+## 🎯 Q2 2025
+
+- [X] HNSW backend integration
+- [X] DiskANN backend with MIPS/L2/Cosine support
+- [X] Real-time embedding pipeline
+- [X] Memory-efficient graph pruning
+
+## 🚀 Q3 2025
+
+- [ ] Advanced caching strategies
+- [ ] Add contextual-retrieval https://www.anthropic.com/news/contextual-retrieval
+- [ ] Add sleep-time-compute and summarize agent! to summarilze the file on computer!
+- [ ] Add OpenAI recompute API
+
+## 🌟 Q4 2025
+
+- [ ] Integration with LangChain/LlamaIndex
+- [ ] Visual similarity search
+- [ ] Query rewrtiting, rerank and expansion

examples/basic_demo.py

@@ -1,16 +1,23 @@
 """
 Simple demo showing basic leann usage
-Run: uv run python examples/simple_demo.py
+Run: uv run python examples/basic_demo.py
 """
 
 import argparse
-from leann import LeannBuilder, LeannSearcher, LeannChat
+
+from leann import LeannBuilder, LeannChat, LeannSearcher
 
 
 def main():
-    parser = argparse.ArgumentParser(description="Simple demo of Leann with selectable embedding models.")
-    parser.add_argument("--embedding_model", type=str, default="sentence-transformers/all-mpnet-base-v2",
-                        help="The embedding model to use, e.g., 'sentence-transformers/all-mpnet-base-v2' or 'text-embedding-ada-002'.")
+    parser = argparse.ArgumentParser(
+        description="Simple demo of Leann with selectable embedding models."
+    )
+    parser.add_argument(
+        "--embedding_model",
+        type=str,
+        default="sentence-transformers/all-mpnet-base-v2",
+        help="The embedding model to use, e.g., 'sentence-transformers/all-mpnet-base-v2' or 'text-embedding-ada-002'.",
+    )
     args = parser.parse_args()
 
     print(f"=== Leann Simple Demo with {args.embedding_model} ===")
@@ -74,7 +81,7 @@ def main():
         print()
 
     print("Demo completed! Try running:")
-    print("   uv run python examples/document_search.py")
+    print("   uv run python apps/document_rag.py")
 
 
 if __name__ == "__main__":

examples/document_search.py

@@ -1,146 +0,0 @@
-#!/usr/bin/env python3
-"""
-Document search demo with recompute mode
-"""
-
-import os
-from pathlib import Path
-import shutil
-import time
-
-# Import backend packages to trigger plugin registration
-try:
-    import leann_backend_diskann
-    import leann_backend_hnsw
-    print("INFO: Backend packages imported successfully.")
-except ImportError as e:
-    print(f"WARNING: Could not import backend packages. Error: {e}")
-
-# Import upper-level API from leann-core
-from leann.api import LeannBuilder, LeannSearcher, LeannChat
-
-
-def load_sample_documents():
-    """Create sample documents for demonstration"""
-    docs = [
-        {"title": "Intro to Python", "content": "Python is a high-level, interpreted language known for simplicity."},
-        {"title": "ML Basics", "content": "Machine learning builds systems that learn from data."},
-        {"title": "Data Structures", "content": "Data structures like arrays, lists, and graphs organize data."},
-    ]
-    return docs
-
-def main():
-    print("==========================================================")
-    print("=== Leann Document Search Demo (DiskANN + Recompute) ===")
-    print("==========================================================")
-    
-    INDEX_DIR = Path("./test_indices")
-    INDEX_PATH = str(INDEX_DIR / "documents.diskann")
-    BACKEND_TO_TEST = "diskann"
-
-    if INDEX_DIR.exists():
-        print(f"--- Cleaning up old index directory: {INDEX_DIR} ---")
-        shutil.rmtree(INDEX_DIR)
-
-    # --- 1. Build index ---
-    print(f"\n[PHASE 1] Building index using '{BACKEND_TO_TEST}' backend...")
-    
-    builder = LeannBuilder(
-        backend_name=BACKEND_TO_TEST, 
-        graph_degree=32, 
-        complexity=64
-    )
-    
-    documents = load_sample_documents()
-    print(f"Loaded {len(documents)} sample documents.")
-    for doc in documents:
-        builder.add_text(doc["content"], metadata={"title": doc["title"]})
-        
-    builder.build_index(INDEX_PATH)
-    print(f"\nIndex built!")
-
-    # --- 2. Basic search demo ---
-    print(f"\n[PHASE 2] Basic search using '{BACKEND_TO_TEST}' backend...")
-    searcher = LeannSearcher(index_path=INDEX_PATH)
-    
-    query = "What is machine learning?"
-    print(f"\nQuery: '{query}'")
-    
-    print("\n--- Basic search mode (PQ computation) ---")
-    start_time = time.time()
-    results = searcher.search(query, top_k=2)
-    basic_time = time.time() - start_time
-    
-    print(f"⏱️  Basic search time: {basic_time:.3f} seconds")
-    print(">>> Basic search results <<<")
-    for i, res in enumerate(results, 1):
-        print(f"  {i}. ID: {res['id']}, Score: {res['score']:.4f}, Text: '{res['text']}', Metadata: {res['metadata']}")
-
-    # --- 3. Recompute search demo ---
-    print(f"\n[PHASE 3] Recompute search using embedding server...")
-    
-    print("\n--- Recompute search mode (get real embeddings via network) ---")
-    
-    # Configure recompute parameters
-    recompute_params = {
-        "recompute_beighbor_embeddings": True,  # Enable network recomputation
-        "USE_DEFERRED_FETCH": False,           # Don't use deferred fetch
-        "skip_search_reorder": True,           # Skip search reordering
-        "dedup_node_dis": True,               # Enable node distance deduplication
-        "prune_ratio": 0.1,                   # Pruning ratio 10%
-        "batch_recompute": False,             # Don't use batch recomputation
-        "global_pruning": False,              # Don't use global pruning
-        "zmq_port": 5555,                     # ZMQ port
-        "embedding_model": "sentence-transformers/all-mpnet-base-v2"
-    }
-    
-    print("Recompute parameter configuration:")
-    for key, value in recompute_params.items():
-        print(f"  {key}: {value}")
-    
-    print(f"\n🔄 Executing Recompute search...")
-    try:
-        start_time = time.time()
-        recompute_results = searcher.search(query, top_k=2, **recompute_params)
-        recompute_time = time.time() - start_time
-        
-        print(f"⏱️  Recompute search time: {recompute_time:.3f} seconds")
-        print(">>> Recompute search results <<<")
-        for i, res in enumerate(recompute_results, 1):
-            print(f"  {i}. ID: {res['id']}, Score: {res['score']:.4f}, Text: '{res['text']}', Metadata: {res['metadata']}")
-        
-        # Compare results
-        print(f"\n--- Result comparison ---")
-        print(f"Basic search time: {basic_time:.3f} seconds")
-        print(f"Recompute time: {recompute_time:.3f} seconds")
-        
-        print("\nBasic search vs Recompute results:")
-        for i in range(min(len(results), len(recompute_results))):
-            basic_score = results[i]['score']
-            recompute_score = recompute_results[i]['score']
-            score_diff = abs(basic_score - recompute_score)
-            print(f"  Position {i+1}: PQ={basic_score:.4f}, Recompute={recompute_score:.4f}, Difference={score_diff:.4f}")
-        
-        if recompute_time > basic_time:
-            print(f"✅ Recompute mode working correctly (more accurate but slower)")
-        else:
-            print(f"ℹ️  Recompute time is unusually fast, network recomputation may not be enabled")
-            
-    except Exception as e:
-        print(f"❌ Recompute search failed: {e}")
-        print("This usually indicates an embedding server connection issue")
-
-    # --- 4. Chat demo ---
-    print(f"\n[PHASE 4] Starting chat session...")
-    chat = LeannChat(index_path=INDEX_PATH)
-    chat_response = chat.ask(query)
-    print(f"You: {query}")
-    print(f"Leann: {chat_response}")
-
-    print("\n==========================================================")
-    print("✅ Demo finished successfully!")
-    print("==========================================================")
-
-
-if __name__ == "__main__":
-    main()

examples/main_cli_example.py

@@ -1,81 +0,0 @@
-import faulthandler
-faulthandler.enable()
-
-from llama_index.core import SimpleDirectoryReader, Settings
-from llama_index.core.readers.base import BaseReader
-from llama_index.node_parser.docling import DoclingNodeParser
-from llama_index.readers.docling import DoclingReader
-from docling_core.transforms.chunker.hybrid_chunker import HybridChunker
-import asyncio
-import os
-import dotenv
-from leann.api import LeannBuilder, LeannSearcher, LeannChat
-import leann_backend_hnsw # Import to ensure backend registration
-import shutil
-from pathlib import Path
-
-dotenv.load_dotenv()
-
-reader = DoclingReader(export_type=DoclingReader.ExportType.JSON)
-file_extractor: dict[str, BaseReader] = {
-    ".docx": reader,
-    ".pptx": reader, 
-    ".pdf": reader,
-    ".xlsx": reader,
-}
-node_parser = DoclingNodeParser(
-    chunker=HybridChunker(tokenizer="Qwen/Qwen3-Embedding-4B", max_tokens=64)
-)
-print("Loading documents...")
-documents = SimpleDirectoryReader(
-    "examples/data", 
-    recursive=True, 
-    file_extractor=file_extractor,
-    encoding="utf-8",
-    required_exts=[".pdf", ".docx", ".pptx", ".xlsx"]
-).load_data(show_progress=True)
-print("Documents loaded.")
-all_texts = []
-for doc in documents:
-    nodes = node_parser.get_nodes_from_documents([doc])
-    for node in nodes:
-        all_texts.append(node.text)
-
-INDEX_DIR = Path("./test_pdf_index")
-INDEX_PATH = str(INDEX_DIR / "pdf_documents.leann")
-
-if not INDEX_DIR.exists():
-    print(f"--- Index directory not found, building new index ---")
-    
-    print(f"\n[PHASE 1] Building Leann index...")
-
-    # CSR compact mode with recompute
-    builder = LeannBuilder(
-        backend_name="hnsw",
-        embedding_model="facebook/contriever",
-        graph_degree=32, 
-        complexity=64,
-        is_compact=True,
-        is_recompute=True
-    )
-
-    print(f"Loaded {len(all_texts)} text chunks from documents.")
-    for chunk_text in all_texts:
-        builder.add_text(chunk_text)
-        
-    builder.build_index(INDEX_PATH)
-    print(f"\nLeann index built at {INDEX_PATH}!")
-else:
-    print(f"--- Using existing index at {INDEX_DIR} ---")
-
-async def main():
-    print(f"\n[PHASE 2] Starting Leann chat session...")
-    chat = LeannChat(index_path=INDEX_PATH)
-    
-    query = "Based on the paper, what are the main techniques LEANN explores to reduce the storage overhead and DLPM explore to achieve Fairness and Efiiciency trade-off?"
-    print(f"You: {query}")
-    chat_response = chat.ask(query, top_k=20, recompute_beighbor_embeddings=True,embedding_model="facebook/contriever")
-    print(f"Leann: {chat_response}")
-
-if __name__ == "__main__":
-    asyncio.run(main())

examples/mlx_demo.py

@@ -0,0 +1,43 @@
+import os
+
+from leann.api import LeannBuilder, LeannChat
+
+# Define the path for our new MLX-based index
+INDEX_PATH = "./mlx_diskann_index/leann"
+
+if os.path.exists(INDEX_PATH + ".meta.json"):
+    print(f"Index already exists at {INDEX_PATH}. Skipping build.")
+else:
+    print("Initializing LeannBuilder with MLX support...")
+    # 1. Configure LeannBuilder to use MLX
+    builder = LeannBuilder(
+        backend_name="hnsw",
+        embedding_model="mlx-community/Qwen3-Embedding-0.6B-4bit-DWQ",
+        embedding_mode="mlx",
+    )
+
+    # 2. Add documents
+    print("Adding documents...")
+    docs = [
+        "MLX is an array framework for machine learning on Apple silicon.",
+        "It was designed by Apple's machine learning research team.",
+        "The mlx-community organization provides pre-trained models in MLX format.",
+        "It supports operations on multi-dimensional arrays.",
+        "Leann can now use MLX for its embedding models.",
+    ]
+    for doc in docs:
+        builder.add_text(doc)
+
+    # 3. Build the index
+    print(f"Building the MLX-based index at: {INDEX_PATH}")
+    builder.build_index(INDEX_PATH)
+    print("\nSuccessfully built the index with MLX embeddings!")
+    print(f"Check the metadata file: {INDEX_PATH}.meta.json")
+
+
+chat = LeannChat(index_path=INDEX_PATH)
+# add query
+query = "MLX is an array framework for machine learning on Apple silicon."
+print(f"Query: {query}")
+response = chat.ask(query, top_k=3, recompute_beighbor_embeddings=True, complexity=3, beam_width=1)
+print(f"Response: {response}")

examples/spoiler_free_book_rag.py

@@ -0,0 +1,250 @@
+#!/usr/bin/env python3
+"""
+Spoiler-Free Book RAG Example using LEANN Metadata Filtering
+
+This example demonstrates how to use LEANN's metadata filtering to create
+a spoiler-free book RAG system where users can search for information
+up to a specific chapter they've read.
+
+Usage:
+    python spoiler_free_book_rag.py
+"""
+
+import os
+import sys
+from typing import Any, Optional
+
+# Add LEANN to path (adjust path as needed)
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../packages/leann-core/src"))
+
+from leann.api import LeannBuilder, LeannSearcher
+
+
+def chunk_book_with_metadata(book_title: str = "Sample Book") -> list[dict[str, Any]]:
+    """
+    Create sample book chunks with metadata for demonstration.
+
+    In a real implementation, this would parse actual book files (epub, txt, etc.)
+    and extract chapter boundaries, character mentions, etc.
+
+    Args:
+        book_title: Title of the book
+
+    Returns:
+        List of chunk dictionaries with text and metadata
+    """
+    # Sample book chunks with metadata
+    # In practice, you'd use proper text processing libraries
+
+    sample_chunks = [
+        {
+            "text": "Alice was beginning to get very tired of sitting by her sister on the bank, and of having nothing to do.",
+            "metadata": {
+                "book": book_title,
+                "chapter": 1,
+                "page": 1,
+                "characters": ["Alice", "Sister"],
+                "themes": ["boredom", "curiosity"],
+                "location": "riverbank",
+            },
+        },
+        {
+            "text": "So she was considering in her own mind (as well as she could, for the hot day made her feel very sleepy and stupid), whether the pleasure of making a daisy-chain would be worth the trouble of getting up and picking the daisies, when suddenly a White Rabbit with pink eyes ran close by her.",
+            "metadata": {
+                "book": book_title,
+                "chapter": 1,
+                "page": 2,
+                "characters": ["Alice", "White Rabbit"],
+                "themes": ["decision", "surprise", "magic"],
+                "location": "riverbank",
+            },
+        },
+        {
+            "text": "Alice found herself falling down a very deep well. Either the well was very deep, or she fell very slowly, for she had plenty of time as she fell to look about her and to wonder what was going to happen next.",
+            "metadata": {
+                "book": book_title,
+                "chapter": 2,
+                "page": 15,
+                "characters": ["Alice"],
+                "themes": ["falling", "wonder", "transformation"],
+                "location": "rabbit hole",
+            },
+        },
+        {
+            "text": "Alice meets the Cheshire Cat, who tells her that everyone in Wonderland is mad, including Alice herself.",
+            "metadata": {
+                "book": book_title,
+                "chapter": 6,
+                "page": 85,
+                "characters": ["Alice", "Cheshire Cat"],
+                "themes": ["madness", "philosophy", "identity"],
+                "location": "Duchess's house",
+            },
+        },
+        {
+            "text": "At the Queen's croquet ground, Alice witnesses the absurd trial that reveals the arbitrary nature of Wonderland's justice system.",
+            "metadata": {
+                "book": book_title,
+                "chapter": 8,
+                "page": 120,
+                "characters": ["Alice", "Queen of Hearts", "King of Hearts"],
+                "themes": ["justice", "absurdity", "authority"],
+                "location": "Queen's court",
+            },
+        },
+        {
+            "text": "Alice realizes that Wonderland was all a dream, even the Rabbit, as she wakes up on the riverbank next to her sister.",
+            "metadata": {
+                "book": book_title,
+                "chapter": 12,
+                "page": 180,
+                "characters": ["Alice", "Sister", "Rabbit"],
+                "themes": ["revelation", "reality", "growth"],
+                "location": "riverbank",
+            },
+        },
+    ]
+
+    return sample_chunks
+
+
+def build_spoiler_free_index(book_chunks: list[dict[str, Any]], index_name: str) -> str:
+    """
+    Build a LEANN index with book chunks that include spoiler metadata.
+
+    Args:
+        book_chunks: List of book chunks with metadata
+        index_name: Name for the index
+
+    Returns:
+        Path to the built index
+    """
+    print(f"📚 Building spoiler-free book index: {index_name}")
+
+    # Initialize LEANN builder
+    builder = LeannBuilder(
+        backend_name="hnsw", embedding_model="text-embedding-3-small", embedding_mode="openai"
+    )
+
+    # Add each chunk with its metadata
+    for chunk in book_chunks:
+        builder.add_text(text=chunk["text"], metadata=chunk["metadata"])
+
+    # Build the index
+    index_path = f"{index_name}_book_index"
+    builder.build_index(index_path)
+
+    print(f"✅ Index built successfully: {index_path}")
+    return index_path
+
+
+def spoiler_free_search(
+    index_path: str,
+    query: str,
+    max_chapter: int,
+    character_filter: Optional[list[str]] = None,
+) -> list[dict[str, Any]]:
+    """
+    Perform a spoiler-free search on the book index.
+
+    Args:
+        index_path: Path to the LEANN index
+        query: Search query
+        max_chapter: Maximum chapter number to include
+        character_filter: Optional list of characters to focus on
+
+    Returns:
+        List of search results safe for the reader
+    """
+    print(f"🔍 Searching: '{query}' (up to chapter {max_chapter})")
+
+    searcher = LeannSearcher(index_path)
+
+    metadata_filters = {"chapter": {"<=": max_chapter}}
+
+    if character_filter:
+        metadata_filters["characters"] = {"contains": character_filter[0]}
+
+    results = searcher.search(query=query, top_k=10, metadata_filters=metadata_filters)
+
+    return results
+
+
+def demo_spoiler_free_rag():
+    """
+    Demonstrate the spoiler-free book RAG system.
+    """
+    print("🎭 Spoiler-Free Book RAG Demo")
+    print("=" * 40)
+
+    # Step 1: Prepare book data
+    book_title = "Alice's Adventures in Wonderland"
+    book_chunks = chunk_book_with_metadata(book_title)
+
+    print(f"📖 Loaded {len(book_chunks)} chunks from '{book_title}'")
+
+    # Step 2: Build the index (in practice, this would be done once)
+    try:
+        index_path = build_spoiler_free_index(book_chunks, "alice_wonderland")
+    except Exception as e:
+        print(f"❌ Failed to build index (likely missing dependencies): {e}")
+        print(
+            "💡 This demo shows the filtering logic - actual indexing requires LEANN dependencies"
+        )
+        return
+
+    # Step 3: Demonstrate various spoiler-free searches
+    search_scenarios = [
+        {
+            "description": "Reader who has only read Chapter 1",
+            "query": "What can you tell me about the rabbit?",
+            "max_chapter": 1,
+        },
+        {
+            "description": "Reader who has read up to Chapter 5",
+            "query": "Tell me about Alice's adventures",
+            "max_chapter": 5,
+        },
+        {
+            "description": "Reader who has read most of the book",
+            "query": "What does the Cheshire Cat represent?",
+            "max_chapter": 10,
+        },
+        {
+            "description": "Reader who has read the whole book",
+            "query": "What can you tell me about the rabbit?",
+            "max_chapter": 12,
+        },
+    ]
+
+    for scenario in search_scenarios:
+        print(f"\n📚 Scenario: {scenario['description']}")
+        print(f"   Query: {scenario['query']}")
+
+        try:
+            results = spoiler_free_search(
+                index_path=index_path,
+                query=scenario["query"],
+                max_chapter=scenario["max_chapter"],
+            )
+
+            print(f"   📄 Found {len(results)} results:")
+            for i, result in enumerate(results[:3], 1):  # Show top 3
+                chapter = result.metadata.get("chapter", "?")
+                location = result.metadata.get("location", "?")
+                print(f"      {i}. Chapter {chapter} ({location}): {result.text[:80]}...")
+
+        except Exception as e:
+            print(f"   ❌ Search failed: {e}")
+
+
+if __name__ == "__main__":
+    print("📚 LEANN Spoiler-Free Book RAG Example")
+    print("=====================================")
+
+    try:
+        demo_spoiler_free_rag()
+    except ImportError as e:
+        print(f"❌ Cannot run demo due to missing dependencies: {e}")
+    except Exception as e:
+        print(f"❌ Error running demo: {e}")

knowledge.leann.meta.json

@@ -1,32 +0,0 @@
-{
-  "version": "0.1.0",
-  "backend_name": "diskann",
-  "embedding_model": "sentence-transformers/all-mpnet-base-v2",
-  "num_chunks": 6,
-  "chunks": [
-    {
-      "text": "Python is a powerful programming language",
-      "metadata": {}
-    },
-    {
-      "text": "Machine learning transforms industries",
-      "metadata": {}
-    },
-    {
-      "text": "Neural networks process complex data",
-      "metadata": {}
-    },
-    {
-      "text": "Java is a powerful programming language",
-      "metadata": {}
-    },
-    {
-      "text": "C++ is a powerful programming language",
-      "metadata": {}
-    },
-    {
-      "text": "C# is a powerful programming language",
-      "metadata": {}
-    }
-  ]
-}

packages/leann-backend-diskann/CMakeLists.txt

@@ -1,8 +0,0 @@
-# packages/leann-backend-diskann/CMakeLists.txt (最终简化版)
-
-cmake_minimum_required(VERSION 3.20)
-project(leann_backend_diskann_wrapper)
-
-# 告诉 CMake 直接进入 DiskANN 子模块并执行它自己的 CMakeLists.txt
-# DiskANN 会自己处理所有事情，包括编译 Python 绑定
-add_subdirectory(src/third_party/DiskANN)

packages/leann-backend-diskann/__init__.py

@@ -0,0 +1 @@
+# This file makes the directory a Python package

packages/leann-backend-diskann/leann_backend_diskann/__init__.py

@@ -0,0 +1,7 @@
+from . import diskann_backend as diskann_backend
+from . import graph_partition
+
+# Export main classes and functions
+from .graph_partition import GraphPartitioner, partition_graph
+
+__all__ = ["GraphPartitioner", "diskann_backend", "graph_partition", "partition_graph"]

packages/leann-backend-diskann/leann_backend_diskann/diskann_backend.py

@@ -1,31 +1,77 @@
-import numpy as np
-import os
-import json
-import struct
-from pathlib import Path
-from typing import Dict
 import contextlib
-import threading
-import time
-import atexit
-import socket
-import subprocess
+import logging
+import os
+import struct
 import sys
+from pathlib import Path
+from typing import Any, Literal, Optional
 
-from leann.registry import register_backend
+import numpy as np
+import psutil
 from leann.interface import (
-    LeannBackendFactoryInterface,
     LeannBackendBuilderInterface,
-    LeannBackendSearcherInterface
+    LeannBackendFactoryInterface,
+    LeannBackendSearcherInterface,
 )
-from . import _diskannpy as diskannpy
+from leann.registry import register_backend
+from leann.searcher_base import BaseSearcher
 
-METRIC_MAP = {
+logger = logging.getLogger(__name__)
+
+
+@contextlib.contextmanager
+def suppress_cpp_output_if_needed():
+    """Suppress C++ stdout/stderr based on LEANN_LOG_LEVEL"""
+    # In CI we avoid fiddling with low-level file descriptors to prevent aborts
+    if os.getenv("CI") == "true":
+        yield
+        return
+
+    log_level = os.getenv("LEANN_LOG_LEVEL", "WARNING").upper()
+
+    # Only suppress if log level is WARNING or higher (ERROR, CRITICAL)
+    should_suppress = log_level in ["WARNING", "ERROR", "CRITICAL"]
+
+    if not should_suppress:
+        # Don't suppress, just yield
+        yield
+        return
+
+    # Save original file descriptors
+    stdout_fd = sys.stdout.fileno()
+    stderr_fd = sys.stderr.fileno()
+
+    # Save original stdout/stderr
+    stdout_dup = os.dup(stdout_fd)
+    stderr_dup = os.dup(stderr_fd)
+
+    try:
+        # Redirect to /dev/null
+        devnull = os.open(os.devnull, os.O_WRONLY)
+        os.dup2(devnull, stdout_fd)
+        os.dup2(devnull, stderr_fd)
+        os.close(devnull)
+
+        yield
+
+    finally:
+        # Restore original file descriptors
+        os.dup2(stdout_dup, stdout_fd)
+        os.dup2(stderr_dup, stderr_fd)
+        os.close(stdout_dup)
+        os.close(stderr_dup)
+
+
+def _get_diskann_metrics():
+    from . import _diskannpy as diskannpy  # type: ignore
+
+    return {
         "mips": diskannpy.Metric.INNER_PRODUCT,
         "l2": diskannpy.Metric.L2,
         "cosine": diskannpy.Metric.COSINE,
     }
 
+
 @contextlib.contextmanager
 def chdir(path):
     original_dir = os.getcwd()
@@ -35,102 +81,51 @@ def chdir(path):
     finally:
         os.chdir(original_dir)
 
-def _write_vectors_to_bin(data: np.ndarray, file_path: str):
+
+def _write_vectors_to_bin(data: np.ndarray, file_path: Path):
     num_vectors, dim = data.shape
-    with open(file_path, 'wb') as f:
-        f.write(struct.pack('I', num_vectors))
-        f.write(struct.pack('I', dim))
+    with open(file_path, "wb") as f:
+        f.write(struct.pack("I", num_vectors))
+        f.write(struct.pack("I", dim))
         f.write(data.tobytes())
 
-def _check_port(port: int) -> bool:
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-        return s.connect_ex(('localhost', port)) == 0
 
-class EmbeddingServerManager:
-    def __init__(self):
-        self.server_process = None
-        self.server_port = None
-        atexit.register(self.stop_server)
+def _calculate_smart_memory_config(data: np.ndarray) -> tuple[float, float]:
+    """
+    Calculate smart memory configuration for DiskANN based on data size and system specs.
 
-    def start_server(self, port=5555, model_name="sentence-transformers/all-mpnet-base-v2"):
-        if self.server_process and self.server_process.poll() is None:
-            print(f"INFO: Reusing existing server process for this session (PID {self.server_process.pid})")
-            return True
+    Args:
+        data: The embedding data array
 
-        # 检查端口是否已被其他无关进程占用
-        if _check_port(port):
-            print(f"WARNING: Port {port} is already in use. Assuming an external server is running and connecting to it.")
-            return True
+    Returns:
+        tuple: (search_memory_maximum, build_memory_maximum) in GB
+    """
+    num_vectors, dim = data.shape
 
-        print(f"INFO: Starting session-level embedding server as a background process...")
+    # Calculate embedding storage size
+    embedding_size_bytes = num_vectors * dim * 4  # float32 = 4 bytes
+    embedding_size_gb = embedding_size_bytes / (1024**3)
 
-        try:
-            command = [
-                sys.executable,
-                "-m", "packages.leann-backend-diskann.leann_backend_diskann.embedding_server",
-                "--zmq-port", str(port), 
-                "--model-name", model_name
-            ]
-            project_root = Path(__file__).parent.parent.parent.parent
-            print(f"INFO: Running command from project root: {project_root}")
-            self.server_process = subprocess.Popen(
-                command,
-                cwd=project_root,
-                # stdout=subprocess.PIPE,
-                # stderr=subprocess.PIPE,
-                text=True,
-                encoding='utf-8'
+    # search_memory_maximum: 1/10 of embedding size for optimal PQ compression
+    # This controls Product Quantization size - smaller means more compression
+    search_memory_gb = max(0.1, embedding_size_gb / 10)  # At least 100MB
+
+    # build_memory_maximum: Based on available system RAM for sharding control
+    # This controls how much memory DiskANN uses during index construction
+    available_memory_gb = psutil.virtual_memory().available / (1024**3)
+    total_memory_gb = psutil.virtual_memory().total / (1024**3)
+
+    # Use 50% of available memory, but at least 2GB and at most 75% of total
+    build_memory_gb = max(2.0, min(available_memory_gb * 0.5, total_memory_gb * 0.75))
+
+    logger.info(
+        f"Smart memory config - Data: {embedding_size_gb:.2f}GB, "
+        f"Search mem: {search_memory_gb:.2f}GB (PQ control), "
+        f"Build mem: {build_memory_gb:.2f}GB (sharding control)"
     )
-            self.server_port = port
-            print(f"INFO: Server process started with PID: {self.server_process.pid}")
 
-            max_wait, wait_interval = 30, 0.5
-            for _ in range(int(max_wait / wait_interval)):
-                if _check_port(port):
-                    print(f"✅ Embedding server is up and ready for this session.")
-                    log_thread = threading.Thread(target=self._log_monitor, daemon=True)
-                    log_thread.start()
-                    return True
-                if self.server_process.poll() is not None:
-                    print("❌ ERROR: Server process terminated unexpectedly during startup.")
-                    self._log_monitor()
-                    return False
-                time.sleep(wait_interval)
+    return search_memory_gb, build_memory_gb
 
-            print(f"❌ ERROR: Server process failed to start listening within {max_wait} seconds.")
-            self.stop_server()
-            return False
-                
-        except Exception as e:
-            print(f"❌ ERROR: Failed to start embedding server process: {e}")
-            return False
-
-    def _log_monitor(self):
-        if not self.server_process:
-            return
-        try:
-            if self.server_process.stdout:
-                for line in iter(self.server_process.stdout.readline, ''):
-                    print(f"[EmbeddingServer LOG]: {line.strip()}")
-                self.server_process.stdout.close()
-            if self.server_process.stderr:
-                for line in iter(self.server_process.stderr.readline, ''):
-                    print(f"[EmbeddingServer ERROR]: {line.strip()}")
-                self.server_process.stderr.close()
-        except Exception as e:
-            print(f"Log monitor error: {e}")
-
-    def stop_server(self):
-        if self.server_process and self.server_process.poll() is None:
-            print(f"INFO: Terminating session server process (PID: {self.server_process.pid})...")
-            self.server_process.terminate()
-            try:
-                self.server_process.wait(timeout=5)
-                print("INFO: Server process terminated.")
-            except subprocess.TimeoutExpired:
-                print("WARNING: Server process did not terminate gracefully, killing it.")
-                self.server_process.kill()
-        self.server_process = None
 
 @register_backend("diskann")
 class DiskannBackend(LeannBackendFactoryInterface):
@@ -140,134 +135,321 @@ class DiskannBackend(LeannBackendFactoryInterface):
 
     @staticmethod
     def searcher(index_path: str, **kwargs) -> LeannBackendSearcherInterface:
-        path = Path(index_path)
-        meta_path = path.parent / f"{path.name}.meta.json"
-        if not meta_path.exists():
-            raise FileNotFoundError(f"Leann metadata file not found at {meta_path}. Cannot infer vector dimension for searcher.")
-        
-        with open(meta_path, 'r') as f:
-            meta = json.load(f)
-
-        dimensions = meta.get("dimensions")
-        if not dimensions:
-            raise ValueError("Dimensions not found in Leann metadata. Please rebuild the index with a newer version of Leann.")
-        
-        kwargs['dimensions'] = dimensions
         return DiskannSearcher(index_path, **kwargs)
 
+
 class DiskannBuilder(LeannBackendBuilderInterface):
     def __init__(self, **kwargs):
         self.build_params = kwargs
 
-    def build(self, data: np.ndarray, index_path: str, **kwargs):
+    def _safe_cleanup_after_partition(self, index_dir: Path, index_prefix: str):
+        """
+        Safely cleanup files after partition.
+        In partition mode, C++ doesn't read _disk.index content,
+        so we can delete it if all derived files exist.
+        """
+        disk_index_file = index_dir / f"{index_prefix}_disk.index"
+        beam_search_file = index_dir / f"{index_prefix}_disk_beam_search.index"
+
+        # Required files that C++ partition mode needs
+        # Note: C++ generates these with _disk.index suffix
+        disk_suffix = "_disk.index"
+        required_files = [
+            f"{index_prefix}{disk_suffix}_medoids.bin",  # Critical: assert fails if missing
+            # Note: _centroids.bin is not created in single-shot build - C++ handles this automatically
+            f"{index_prefix}_pq_pivots.bin",  # PQ table
+            f"{index_prefix}_pq_compressed.bin",  # PQ compressed vectors
+        ]
+
+        # Check if all required files exist
+        missing_files = []
+        for filename in required_files:
+            file_path = index_dir / filename
+            if not file_path.exists():
+                missing_files.append(filename)
+
+        if missing_files:
+            logger.warning(
+                f"Cannot safely delete _disk.index - missing required files: {missing_files}"
+            )
+            logger.info("Keeping all original files for safety")
+            return
+
+        # Calculate space savings
+        space_saved = 0
+        files_to_delete = []
+
+        if disk_index_file.exists():
+            space_saved += disk_index_file.stat().st_size
+            files_to_delete.append(disk_index_file)
+
+        if beam_search_file.exists():
+            space_saved += beam_search_file.stat().st_size
+            files_to_delete.append(beam_search_file)
+
+        # Safe to delete!
+        for file_to_delete in files_to_delete:
+            try:
+                os.remove(file_to_delete)
+                logger.info(f"✅ Safely deleted: {file_to_delete.name}")
+            except Exception as e:
+                logger.warning(f"Failed to delete {file_to_delete.name}: {e}")
+
+        if space_saved > 0:
+            space_saved_mb = space_saved / (1024 * 1024)
+            logger.info(f"💾 Space saved: {space_saved_mb:.1f} MB")
+
+            # Show what files are kept
+            logger.info("📁 Kept essential files for partition mode:")
+            for filename in required_files:
+                file_path = index_dir / filename
+                if file_path.exists():
+                    size_mb = file_path.stat().st_size / (1024 * 1024)
+                    logger.info(f"  - {filename} ({size_mb:.1f} MB)")
+
+    def build(self, data: np.ndarray, ids: list[str], index_path: str, **kwargs):
         path = Path(index_path)
         index_dir = path.parent
         index_prefix = path.stem
-
         index_dir.mkdir(parents=True, exist_ok=True)
 
         if data.dtype != np.float32:
+            logger.warning(f"Converting data to float32, shape: {data.shape}")
             data = data.astype(np.float32)
-        if not data.flags['C_CONTIGUOUS']:
-            data = np.ascontiguousarray(data)
 
         data_filename = f"{index_prefix}_data.bin"
         _write_vectors_to_bin(data, index_dir / data_filename)
 
         build_kwargs = {**self.build_params, **kwargs}
-        metric_str = build_kwargs.get("distance_metric", "mips").lower()
-        metric_enum = METRIC_MAP.get(metric_str)
+
+        # Extract is_recompute from nested backend_kwargs if needed
+        is_recompute = build_kwargs.get("is_recompute", False)
+        if not is_recompute and "backend_kwargs" in build_kwargs:
+            is_recompute = build_kwargs["backend_kwargs"].get("is_recompute", False)
+
+        # Flatten all backend_kwargs parameters to top level for compatibility
+        if "backend_kwargs" in build_kwargs:
+            nested_params = build_kwargs.pop("backend_kwargs")
+            build_kwargs.update(nested_params)
+
+        metric_enum = _get_diskann_metrics().get(
+            build_kwargs.get("distance_metric", "mips").lower()
+        )
         if metric_enum is None:
-            raise ValueError(f"Unsupported distance_metric '{metric_str}'.")
+            raise ValueError(
+                f"Unsupported distance_metric '{build_kwargs.get('distance_metric', 'unknown')}'."
+            )
 
-        complexity = build_kwargs.get("complexity", 64)
-        graph_degree = build_kwargs.get("graph_degree", 32)
-        final_index_ram_limit = build_kwargs.get("search_memory_maximum", 4.0)
-        indexing_ram_budget = build_kwargs.get("build_memory_maximum", 8.0)
-        num_threads = build_kwargs.get("num_threads", 8)
-        pq_disk_bytes = build_kwargs.get("pq_disk_bytes", 0)
-        codebook_prefix = ""
-
-        print(f"INFO: Building DiskANN index for {data.shape[0]} vectors with metric {metric_enum}...")
+        # Calculate smart memory configuration if not explicitly provided
+        if (
+            "search_memory_maximum" not in build_kwargs
+            or "build_memory_maximum" not in build_kwargs
+        ):
+            smart_search_mem, smart_build_mem = _calculate_smart_memory_config(data)
+        else:
+            smart_search_mem = build_kwargs.get("search_memory_maximum", 4.0)
+            smart_build_mem = build_kwargs.get("build_memory_maximum", 8.0)
 
         try:
+            from . import _diskannpy as diskannpy  # type: ignore
+
             with chdir(index_dir):
                 diskannpy.build_disk_float_index(
                     metric_enum,
                     data_filename,
                     index_prefix,
-                    complexity,
-                    graph_degree,
-                    final_index_ram_limit,
-                    indexing_ram_budget,
-                    num_threads,
-                    pq_disk_bytes,
-                    codebook_prefix
+                    build_kwargs.get("complexity", 64),
+                    build_kwargs.get("graph_degree", 32),
+                    build_kwargs.get("search_memory_maximum", smart_search_mem),
+                    build_kwargs.get("build_memory_maximum", smart_build_mem),
+                    build_kwargs.get("num_threads", 8),
+                    build_kwargs.get("pq_disk_bytes", 0),
+                    "",
                 )
-            print(f"✅ DiskANN index built successfully at '{index_dir / index_prefix}'")
-        except Exception as e:
-            print(f"💥 ERROR: DiskANN index build failed. Exception: {e}")
-            raise
+
+            # Auto-partition if is_recompute is enabled
+            if build_kwargs.get("is_recompute", False):
+                logger.info("is_recompute=True, starting automatic graph partitioning...")
+                from .graph_partition import partition_graph
+
+                # Partition the index using absolute paths
+                # Convert to absolute paths to avoid issues with working directory changes
+                absolute_index_dir = Path(index_dir).resolve()
+                absolute_index_prefix_path = str(absolute_index_dir / index_prefix)
+                disk_graph_path, partition_bin_path = partition_graph(
+                    index_prefix_path=absolute_index_prefix_path,
+                    output_dir=str(absolute_index_dir),
+                    partition_prefix=index_prefix,
+                )
+
+                # Safe cleanup: In partition mode, C++ doesn't read _disk.index content
+                # but still needs the derived files (_medoids.bin, _centroids.bin, etc.)
+                self._safe_cleanup_after_partition(index_dir, index_prefix)
+
+                logger.info("✅ Graph partitioning completed successfully!")
+                logger.info(f"  - Disk graph: {disk_graph_path}")
+                logger.info(f"  - Partition file: {partition_bin_path}")
+
         finally:
             temp_data_file = index_dir / data_filename
             if temp_data_file.exists():
                 os.remove(temp_data_file)
+                logger.debug(f"Cleaned up temporary data file: {temp_data_file}")
 
-class DiskannSearcher(LeannBackendSearcherInterface):
+
+class DiskannSearcher(BaseSearcher):
     def __init__(self, index_path: str, **kwargs):
-        path = Path(index_path)
-        index_dir = path.parent
-        index_prefix = path.stem
-        metric_str = kwargs.get("distance_metric", "mips").lower()
-        metric_enum = METRIC_MAP.get(metric_str)
-        if metric_enum is None:
-            raise ValueError(f"Unsupported distance_metric '{metric_str}'.")
-        
-        num_threads = kwargs.get("num_threads", 8)
-        num_nodes_to_cache = kwargs.get("num_nodes_to_cache", 0)
-        dimensions = kwargs.get("dimensions")
-        if not dimensions:
-            raise ValueError("Vector dimension not provided to DiskannSearcher.")
-        
-        try:
-            full_index_prefix = str(index_dir / index_prefix)
-            self._index = diskannpy.StaticDiskFloatIndex(
-                metric_enum, full_index_prefix, num_threads, num_nodes_to_cache, 1, "", ""
+        super().__init__(
+            index_path,
+            backend_module_name="leann_backend_diskann.diskann_embedding_server",
+            **kwargs,
         )
-            self.num_threads = num_threads
-            self.embedding_server_manager = EmbeddingServerManager()
-            print("✅ DiskANN index loaded successfully.")
-        except Exception as e:
-            print(f"💥 ERROR: Failed to load DiskANN index. Exception: {e}")
-            raise
 
-    def search(self, query: np.ndarray, top_k: int, **kwargs) -> Dict[str, any]:
-        complexity = kwargs.get("complexity", 256)
-        beam_width = kwargs.get("beam_width", 4)
+        # Initialize DiskANN index with suppressed C++ output based on log level
+        with suppress_cpp_output_if_needed():
+            from . import _diskannpy as diskannpy  # type: ignore
 
-        USE_DEFERRED_FETCH = kwargs.get("USE_DEFERRED_FETCH", False)
-        skip_search_reorder = kwargs.get("skip_search_reorder", False)
-        recompute_beighbor_embeddings = kwargs.get("recompute_beighbor_embeddings", False)
-        dedup_node_dis = kwargs.get("dedup_node_dis", False)
-        prune_ratio = kwargs.get("prune_ratio", 0.0)
-        batch_recompute = kwargs.get("batch_recompute", False)
-        global_pruning = kwargs.get("global_pruning", False)
+            distance_metric = kwargs.get("distance_metric", "mips").lower()
+            metric_enum = _get_diskann_metrics().get(distance_metric)
+            if metric_enum is None:
+                raise ValueError(f"Unsupported distance_metric '{distance_metric}'.")
 
-        if recompute_beighbor_embeddings:
-            print(f"INFO: DiskANN ZMQ mode enabled - ensuring embedding server is running")
-            zmq_port = kwargs.get("zmq_port", 6666)
-            embedding_model = kwargs.get("embedding_model", "sentence-transformers/all-mpnet-base-v2")
+            self.num_threads = kwargs.get("num_threads", 8)
 
-            if not self.embedding_server_manager.start_server(zmq_port, embedding_model):
-                print(f"WARNING: Failed to start embedding server, falling back to PQ computation")
-                kwargs['recompute_beighbor_embeddings'] = False
+            # For DiskANN, we need to reinitialize the index when zmq_port changes
+            # Store the initialization parameters for later use
+            # Note: C++ load method expects the BASE path (without _disk.index suffix)
+            # C++ internally constructs: index_prefix + "_disk.index"
+            index_name = self.index_path.stem  # "simple_test.leann" -> "simple_test"
+            diskann_index_prefix = str(self.index_dir / index_name)  # /path/to/simple_test
+            full_index_prefix = diskann_index_prefix  # /path/to/simple_test (base path)
+
+            # Auto-detect partition files and set partition_prefix
+            partition_graph_file = self.index_dir / f"{index_name}_disk_graph.index"
+            partition_bin_file = self.index_dir / f"{index_name}_partition.bin"
+
+            partition_prefix = ""
+            if partition_graph_file.exists() and partition_bin_file.exists():
+                # C++ expects full path prefix, not just filename
+                partition_prefix = str(self.index_dir / index_name)  # /path/to/simple_test
+                logger.info(
+                    f"✅ Detected partition files, using partition_prefix='{partition_prefix}'"
+                )
+            else:
+                logger.debug("No partition files detected, using standard index files")
+
+            self._init_params = {
+                "metric_enum": metric_enum,
+                "full_index_prefix": full_index_prefix,
+                "num_threads": self.num_threads,
+                "num_nodes_to_cache": kwargs.get("num_nodes_to_cache", 0),
+                "cache_mechanism": 1,
+                "pq_prefix": "",
+                "partition_prefix": partition_prefix,
+            }
+
+            # Log partition configuration for debugging
+            if partition_prefix:
+                logger.info(
+                    f"✅ Detected partition files, using partition_prefix='{partition_prefix}'"
+                )
+            self._diskannpy = diskannpy
+            self._current_zmq_port = None
+            self._index = None
+            logger.debug("DiskANN searcher initialized (index will be loaded on first search)")
+
+    def _ensure_index_loaded(self, zmq_port: int):
+        """Ensure the index is loaded with the correct zmq_port."""
+        if self._index is None or self._current_zmq_port != zmq_port:
+            # Need to (re)load the index with the correct zmq_port
+            with suppress_cpp_output_if_needed():
+                if self._index is not None:
+                    logger.debug(f"Reloading DiskANN index with new zmq_port: {zmq_port}")
+                else:
+                    logger.debug(f"Loading DiskANN index with zmq_port: {zmq_port}")
+
+                self._index = self._diskannpy.StaticDiskFloatIndex(
+                    self._init_params["metric_enum"],
+                    self._init_params["full_index_prefix"],
+                    self._init_params["num_threads"],
+                    self._init_params["num_nodes_to_cache"],
+                    self._init_params["cache_mechanism"],
+                    zmq_port,
+                    self._init_params["pq_prefix"],
+                    self._init_params["partition_prefix"],
+                )
+                self._current_zmq_port = zmq_port
+
+    def search(
+        self,
+        query: np.ndarray,
+        top_k: int,
+        complexity: int = 64,
+        beam_width: int = 1,
+        prune_ratio: float = 0.0,
+        recompute_embeddings: bool = False,
+        pruning_strategy: Literal["global", "local", "proportional"] = "global",
+        zmq_port: Optional[int] = None,
+        batch_recompute: bool = False,
+        dedup_node_dis: bool = False,
+        **kwargs,
+    ) -> dict[str, Any]:
+        """
+        Search for nearest neighbors using DiskANN index.
+
+        Args:
+            query: Query vectors (B, D) where B is batch size, D is dimension
+            top_k: Number of nearest neighbors to return
+            complexity: Search complexity/candidate list size, higher = more accurate but slower
+            beam_width: Number of parallel IO requests per iteration
+            prune_ratio: Ratio of neighbors to prune via approximate distance (0.0-1.0)
+            recompute_embeddings: Whether to fetch fresh embeddings from server
+            pruning_strategy: PQ candidate selection strategy:
+                - "global": Use global pruning strategy (default)
+                - "local": Use local pruning strategy
+                - "proportional": Not supported in DiskANN, falls back to global
+            zmq_port: ZMQ port for embedding server communication. Must be provided if recompute_embeddings is True.
+            batch_recompute: Whether to batch neighbor recomputation (DiskANN-specific)
+            dedup_node_dis: Whether to cache and reuse distance computations (DiskANN-specific)
+            **kwargs: Additional DiskANN-specific parameters (for legacy compatibility)
+
+        Returns:
+            Dict with 'labels' (list of lists) and 'distances' (ndarray)
+        """
+        # Handle zmq_port compatibility: Ensure index is loaded with correct port
+        if recompute_embeddings:
+            if zmq_port is None:
+                raise ValueError("zmq_port must be provided if recompute_embeddings is True")
+            self._ensure_index_loaded(zmq_port)
+        else:
+            # If not recomputing, we still need an index, use a default port
+            if self._index is None:
+                self._ensure_index_loaded(6666)  # Default port when not recomputing
+
+        # DiskANN doesn't support "proportional" strategy
+        if pruning_strategy == "proportional":
+            raise NotImplementedError(
+                "DiskANN backend does not support 'proportional' pruning strategy. Use 'global' or 'local' instead."
+            )
 
         if query.dtype != np.float32:
             query = query.astype(np.float32)
-        if query.ndim == 1:
-            query = np.expand_dims(query, axis=0)
 
-        try:
+        # Map pruning_strategy to DiskANN's global_pruning parameter
+        if pruning_strategy == "local":
+            use_global_pruning = False
+        else:  # "global"
+            use_global_pruning = True
+
+        # Strategy:
+        # - Traversal always uses PQ distances
+        # - If recompute_embeddings=True, do a single final rerank via deferred fetch
+        #   (fetch embeddings for the final candidate set only)
+        # - Do not recompute neighbor distances along the path
+        use_deferred_fetch = True if recompute_embeddings else False
+        recompute_neighors = False  # Expected typo. For backward compatibility.
+
+        with suppress_cpp_output_if_needed():
             labels, distances = self._index.batch_search(
                 query,
                 query.shape[0],
@@ -275,21 +457,15 @@ class DiskannSearcher(LeannBackendSearcherInterface):
                 complexity,
                 beam_width,
                 self.num_threads,
-                USE_DEFERRED_FETCH,
-                skip_search_reorder,
-                recompute_beighbor_embeddings,
+                use_deferred_fetch,
+                kwargs.get("skip_search_reorder", False),
+                recompute_neighors,
                 dedup_node_dis,
                 prune_ratio,
                 batch_recompute,
-                global_pruning
+                use_global_pruning,
             )
-            return {"labels": labels, "distances": distances}
-        except Exception as e:
-            print(f"💥 ERROR: DiskANN search failed. Exception: {e}")
-            batch_size = query.shape[0]
-            return {"labels": np.full((batch_size, top_k), -1, dtype=np.int64), 
-                   "distances": np.full((batch_size, top_k), float('inf'), dtype=np.float32)}
 
-    def __del__(self):
-        if hasattr(self, 'embedding_server_manager'):
-            self.embedding_server_manager.stop_server()
+        string_labels = [[str(int_label) for int_label in batch_labels] for batch_labels in labels]
+
+        return {"labels": string_labels, "distances": distances}

packages/leann-backend-diskann/leann_backend_diskann/diskann_embedding_server.py

@@ -0,0 +1,472 @@
+"""
+DiskANN-specific embedding server
+"""
+
+import argparse
+import json
+import logging
+import os
+import sys
+import threading
+import time
+from pathlib import Path
+from typing import Optional
+
+import numpy as np
+import zmq
+
+# Set up logging based on environment variable
+LOG_LEVEL = os.getenv("LEANN_LOG_LEVEL", "WARNING").upper()
+logger = logging.getLogger(__name__)
+
+# Force set logger level (don't rely on basicConfig in subprocess)
+log_level = getattr(logging, LOG_LEVEL, logging.WARNING)
+logger.setLevel(log_level)
+
+# Ensure we have a handler if none exists
+if not logger.handlers:
+    handler = logging.StreamHandler()
+    formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+    logger.propagate = False
+
+
+def create_diskann_embedding_server(
+    passages_file: Optional[str] = None,
+    zmq_port: int = 5555,
+    model_name: str = "sentence-transformers/all-mpnet-base-v2",
+    embedding_mode: str = "sentence-transformers",
+    distance_metric: str = "l2",
+):
+    """
+    Create and start a ZMQ-based embedding server for DiskANN backend.
+    Uses ROUTER socket and protobuf communication as required by DiskANN C++ implementation.
+    """
+    logger.info(f"Starting DiskANN server on port {zmq_port} with model {model_name}")
+    logger.info(f"Using embedding mode: {embedding_mode}")
+
+    # Add leann-core to path for unified embedding computation
+    current_dir = Path(__file__).parent
+    leann_core_path = current_dir.parent.parent / "leann-core" / "src"
+    sys.path.insert(0, str(leann_core_path))
+
+    try:
+        from leann.api import PassageManager
+        from leann.embedding_compute import compute_embeddings
+
+        logger.info("Successfully imported unified embedding computation module")
+    except ImportError as e:
+        logger.error(f"Failed to import embedding computation module: {e}")
+        return
+    finally:
+        sys.path.pop(0)
+
+    # Check port availability
+    import socket
+
+    def check_port(port):
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            return s.connect_ex(("localhost", port)) == 0
+
+    if check_port(zmq_port):
+        logger.error(f"Port {zmq_port} is already in use")
+        return
+
+    # Only support metadata file, fail fast for everything else
+    if not passages_file or not passages_file.endswith(".meta.json"):
+        raise ValueError("Only metadata files (.meta.json) are supported")
+
+    # Load metadata to get passage sources
+    with open(passages_file) as f:
+        meta = json.load(f)
+
+    logger.info(f"Loading PassageManager with metadata_file_path: {passages_file}")
+    passages = PassageManager(meta["passage_sources"], metadata_file_path=passages_file)
+    logger.info(f"Loaded PassageManager with {len(passages)} passages from metadata")
+
+    # Import protobuf after ensuring the path is correct
+    try:
+        from . import embedding_pb2
+    except ImportError as e:
+        logger.error(f"Failed to import protobuf module: {e}")
+        return
+
+    def zmq_server_thread():
+        """ZMQ server thread using REP socket for universal compatibility"""
+        context = zmq.Context()
+        socket = context.socket(
+            zmq.REP
+        )  # REP socket for both BaseSearcher and DiskANN C++ REQ clients
+        socket.bind(f"tcp://*:{zmq_port}")
+        logger.info(f"DiskANN ZMQ REP server listening on port {zmq_port}")
+
+        socket.setsockopt(zmq.RCVTIMEO, 1000)
+        socket.setsockopt(zmq.SNDTIMEO, 1000)
+        socket.setsockopt(zmq.LINGER, 0)
+
+        while True:
+            try:
+                # REP socket receives single-part messages
+                message = socket.recv()
+
+                # Check for empty messages - REP socket requires response to every request
+                if len(message) == 0:
+                    logger.debug("Received empty message, sending empty response")
+                    socket.send(b"")  # REP socket must respond to every request
+                    continue
+
+                logger.debug(f"Received ZMQ request of size {len(message)} bytes")
+                logger.debug(f"Message preview: {message[:50]}")  # Show first 50 bytes
+
+                e2e_start = time.time()
+
+                # Try protobuf first (for DiskANN C++ node_ids requests - primary use case)
+                texts = []
+                node_ids = []
+                is_text_request = False
+
+                try:
+                    req_proto = embedding_pb2.NodeEmbeddingRequest()
+                    req_proto.ParseFromString(message)
+                    node_ids = list(req_proto.node_ids)
+
+                    if not node_ids:
+                        raise RuntimeError(
+                            f"PROTOBUF: Received empty node_ids! Message size: {len(message)}"
+                        )
+
+                    logger.info(
+                        f"✅ PROTOBUF: Node ID request for {len(node_ids)} node embeddings: {node_ids[:10]}"
+                    )
+                except Exception as protobuf_error:
+                    logger.debug(f"Protobuf parsing failed: {protobuf_error}")
+                    # Fallback to msgpack (for BaseSearcher direct text requests)
+                    try:
+                        import msgpack
+
+                        request = msgpack.unpackb(message)
+                        # For BaseSearcher compatibility, request is a list of texts directly
+                        if isinstance(request, list) and all(
+                            isinstance(item, str) for item in request
+                        ):
+                            texts = request
+                            is_text_request = True
+                            logger.info(f"✅ MSGPACK: Direct text request for {len(texts)} texts")
+                        else:
+                            raise ValueError("Not a valid msgpack text request")
+                    except Exception as msgpack_error:
+                        raise RuntimeError(
+                            f"Both protobuf and msgpack parsing failed! Protobuf: {protobuf_error}, Msgpack: {msgpack_error}"
+                        )
+
+                # Look up texts by node IDs (only if not direct text request)
+                if not is_text_request:
+                    for nid in node_ids:
+                        try:
+                            passage_data = passages.get_passage(str(nid))
+                            txt = passage_data["text"]
+                            if not txt:
+                                raise RuntimeError(f"FATAL: Empty text for passage ID {nid}")
+                            texts.append(txt)
+                        except KeyError as e:
+                            logger.error(f"Passage ID {nid} not found: {e}")
+                            raise e
+                        except Exception as e:
+                            logger.error(f"Exception looking up passage ID {nid}: {e}")
+                            raise
+
+                    # Debug logging
+                    logger.debug(f"Processing {len(texts)} texts")
+                    logger.debug(f"Text lengths: {[len(t) for t in texts[:5]]}")  # Show first 5
+
+                # Process embeddings using unified computation
+                embeddings = compute_embeddings(texts, model_name, mode=embedding_mode)
+                logger.info(
+                    f"Computed embeddings for {len(texts)} texts, shape: {embeddings.shape}"
+                )
+
+                # Prepare response based on request type
+                if is_text_request:
+                    # For BaseSearcher compatibility: return msgpack format
+                    import msgpack
+
+                    response_data = msgpack.packb(embeddings.tolist())
+                else:
+                    # For DiskANN C++ compatibility: return protobuf format
+                    resp_proto = embedding_pb2.NodeEmbeddingResponse()
+                    hidden_contiguous = np.ascontiguousarray(embeddings, dtype=np.float32)
+
+                    # Serialize embeddings data
+                    resp_proto.embeddings_data = hidden_contiguous.tobytes()
+                    resp_proto.dimensions.append(hidden_contiguous.shape[0])
+                    resp_proto.dimensions.append(hidden_contiguous.shape[1])
+
+                    response_data = resp_proto.SerializeToString()
+
+                # Send response back to the client
+                socket.send(response_data)
+
+                e2e_end = time.time()
+                logger.info(f"⏱️  ZMQ E2E time: {e2e_end - e2e_start:.6f}s")
+
+            except zmq.Again:
+                logger.debug("ZMQ socket timeout, continuing to listen")
+                continue
+            except Exception as e:
+                logger.error(f"Error in ZMQ server loop: {e}")
+                import traceback
+
+                traceback.print_exc()
+                raise
+
+    def zmq_server_thread_with_shutdown(shutdown_event):
+        """ZMQ server thread that respects shutdown signal.
+
+        This creates its own REP socket, binds to zmq_port, and periodically
+        checks shutdown_event using recv timeouts to exit cleanly.
+        """
+        logger.info("DiskANN ZMQ server thread started with shutdown support")
+
+        context = zmq.Context()
+        rep_socket = context.socket(zmq.REP)
+        rep_socket.bind(f"tcp://*:{zmq_port}")
+        logger.info(f"DiskANN ZMQ REP server listening on port {zmq_port}")
+
+        # Set receive timeout so we can check shutdown_event periodically
+        rep_socket.setsockopt(zmq.RCVTIMEO, 1000)  # 1 second timeout
+        rep_socket.setsockopt(zmq.SNDTIMEO, 1000)
+        rep_socket.setsockopt(zmq.LINGER, 0)
+
+        try:
+            while not shutdown_event.is_set():
+                try:
+                    e2e_start = time.time()
+                    # REP socket receives single-part messages
+                    message = rep_socket.recv()
+
+                    # Check for empty messages - REP socket requires response to every request
+                    if not message:
+                        logger.warning("Received empty message, sending empty response")
+                        rep_socket.send(b"")
+                        continue
+
+                    # Try protobuf first (same logic as original)
+                    texts = []
+                    is_text_request = False
+
+                    try:
+                        req_proto = embedding_pb2.NodeEmbeddingRequest()
+                        req_proto.ParseFromString(message)
+                        node_ids = list(req_proto.node_ids)
+
+                        # Look up texts by node IDs
+                        for nid in node_ids:
+                            try:
+                                passage_data = passages.get_passage(str(nid))
+                                txt = passage_data["text"]
+                                if not txt:
+                                    raise RuntimeError(f"FATAL: Empty text for passage ID {nid}")
+                                texts.append(txt)
+                            except KeyError:
+                                raise RuntimeError(f"FATAL: Passage with ID {nid} not found")
+
+                        logger.info(f"ZMQ received protobuf request for {len(node_ids)} node IDs")
+                    except Exception:
+                        # Fallback to msgpack for text requests
+                        try:
+                            import msgpack
+
+                            request = msgpack.unpackb(message)
+                            if isinstance(request, list) and all(
+                                isinstance(item, str) for item in request
+                            ):
+                                texts = request
+                                is_text_request = True
+                                logger.info(
+                                    f"ZMQ received msgpack text request for {len(texts)} texts"
+                                )
+                            else:
+                                raise ValueError("Not a valid msgpack text request")
+                        except Exception:
+                            logger.error("Both protobuf and msgpack parsing failed!")
+                            # Send error response
+                            resp_proto = embedding_pb2.NodeEmbeddingResponse()
+                            rep_socket.send(resp_proto.SerializeToString())
+                            continue
+
+                    # Process the request
+                    embeddings = compute_embeddings(texts, model_name, mode=embedding_mode)
+                    logger.info(f"Computed embeddings shape: {embeddings.shape}")
+
+                    # Validation
+                    if np.isnan(embeddings).any() or np.isinf(embeddings).any():
+                        logger.error("NaN or Inf detected in embeddings!")
+                        # Send error response
+                        if is_text_request:
+                            import msgpack
+
+                            response_data = msgpack.packb([])
+                        else:
+                            resp_proto = embedding_pb2.NodeEmbeddingResponse()
+                            response_data = resp_proto.SerializeToString()
+                        rep_socket.send(response_data)
+                        continue
+
+                    # Prepare response based on request type
+                    if is_text_request:
+                        # For direct text requests, return msgpack
+                        import msgpack
+
+                        response_data = msgpack.packb(embeddings.tolist())
+                    else:
+                        # For protobuf requests, return protobuf
+                        resp_proto = embedding_pb2.NodeEmbeddingResponse()
+                        hidden_contiguous = np.ascontiguousarray(embeddings, dtype=np.float32)
+
+                        resp_proto.embeddings_data = hidden_contiguous.tobytes()
+                        resp_proto.dimensions.append(hidden_contiguous.shape[0])
+                        resp_proto.dimensions.append(hidden_contiguous.shape[1])
+
+                        response_data = resp_proto.SerializeToString()
+
+                    # Send response back to the client
+                    rep_socket.send(response_data)
+
+                    e2e_end = time.time()
+                    logger.info(f"⏱️  ZMQ E2E time: {e2e_end - e2e_start:.6f}s")
+
+                except zmq.Again:
+                    # Timeout - check shutdown_event and continue
+                    continue
+                except Exception as e:
+                    if not shutdown_event.is_set():
+                        logger.error(f"Error in ZMQ server loop: {e}")
+                        try:
+                            # Send error response for REP socket
+                            resp_proto = embedding_pb2.NodeEmbeddingResponse()
+                            rep_socket.send(resp_proto.SerializeToString())
+                        except Exception:
+                            pass
+                    else:
+                        logger.info("Shutdown in progress, ignoring ZMQ error")
+                        break
+        finally:
+            try:
+                rep_socket.close(0)
+            except Exception:
+                pass
+            try:
+                context.term()
+            except Exception:
+                pass
+
+        logger.info("DiskANN ZMQ server thread exiting gracefully")
+
+    # Add shutdown coordination
+    shutdown_event = threading.Event()
+
+    def shutdown_zmq_server():
+        """Gracefully shutdown ZMQ server."""
+        logger.info("Initiating graceful shutdown...")
+        shutdown_event.set()
+
+        if zmq_thread.is_alive():
+            logger.info("Waiting for ZMQ thread to finish...")
+            zmq_thread.join(timeout=5)
+            if zmq_thread.is_alive():
+                logger.warning("ZMQ thread did not finish in time")
+
+        # Clean up ZMQ resources
+        try:
+            # Note: socket and context are cleaned up by thread exit
+            logger.info("ZMQ resources cleaned up")
+        except Exception as e:
+            logger.warning(f"Error cleaning ZMQ resources: {e}")
+
+        # Clean up other resources
+        try:
+            import gc
+
+            gc.collect()
+            logger.info("Additional resources cleaned up")
+        except Exception as e:
+            logger.warning(f"Error cleaning additional resources: {e}")
+
+        logger.info("Graceful shutdown completed")
+        sys.exit(0)
+
+    # Register signal handlers within this function scope
+    import signal
+
+    def signal_handler(sig, frame):
+        logger.info(f"Received signal {sig}, shutting down gracefully...")
+        shutdown_zmq_server()
+
+    signal.signal(signal.SIGTERM, signal_handler)
+    signal.signal(signal.SIGINT, signal_handler)
+
+    # Start ZMQ thread (NOT daemon!)
+    zmq_thread = threading.Thread(
+        target=lambda: zmq_server_thread_with_shutdown(shutdown_event),
+        daemon=False,  # Not daemon - we want to wait for it
+    )
+    zmq_thread.start()
+    logger.info(f"Started DiskANN ZMQ server thread on port {zmq_port}")
+
+    # Keep the main thread alive
+    try:
+        while not shutdown_event.is_set():
+            time.sleep(0.1)  # Check shutdown more frequently
+    except KeyboardInterrupt:
+        logger.info("DiskANN Server shutting down...")
+        shutdown_zmq_server()
+        return
+
+    # If we reach here, shutdown was triggered by signal
+    logger.info("Main loop exited, process should be shutting down")
+
+
+if __name__ == "__main__":
+    import sys
+
+    # Signal handlers are now registered within create_diskann_embedding_server
+
+    parser = argparse.ArgumentParser(description="DiskANN Embedding service")
+    parser.add_argument("--zmq-port", type=int, default=5555, help="ZMQ port to run on")
+    parser.add_argument(
+        "--passages-file",
+        type=str,
+        help="Metadata JSON file containing passage sources",
+    )
+    parser.add_argument(
+        "--model-name",
+        type=str,
+        default="sentence-transformers/all-mpnet-base-v2",
+        help="Embedding model name",
+    )
+    parser.add_argument(
+        "--embedding-mode",
+        type=str,
+        default="sentence-transformers",
+        choices=["sentence-transformers", "openai", "mlx", "ollama"],
+        help="Embedding backend mode",
+    )
+    parser.add_argument(
+        "--distance-metric",
+        type=str,
+        default="l2",
+        choices=["l2", "mips", "cosine"],
+        help="Distance metric for similarity computation",
+    )
+
+    args = parser.parse_args()
+
+    # Create and start the DiskANN embedding server
+    create_diskann_embedding_server(
+        passages_file=args.passages_file,
+        zmq_port=args.zmq_port,
+        model_name=args.model_name,
+        embedding_mode=args.embedding_mode,
+        distance_metric=args.distance_metric,
+    )

packages/leann-backend-diskann/leann_backend_diskann/embedding_pb2.py

@@ -1,24 +1,25 @@
-# -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: embedding.proto
+# ruff: noqa
 """Generated protocol buffer code."""
-from google.protobuf.internal import builder as _builder
+
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool
 from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+
 # @@protoc_insertion_point(imports)
 
 _sym_db = _symbol_database.Default()
 
 
-
-
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0f\x65mbedding.proto\x12\x0eprotoembedding\"(\n\x14NodeEmbeddingRequest\x12\x10\n\x08node_ids\x18\x01 \x03(\r\"Y\n\x15NodeEmbeddingResponse\x12\x17\n\x0f\x65mbeddings_data\x18\x01 \x01(\x0c\x12\x12\n\ndimensions\x18\x02 \x03(\x05\x12\x13\n\x0bmissing_ids\x18\x03 \x03(\rb\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
+    b'\n\x0f\x65mbedding.proto\x12\x0eprotoembedding"(\n\x14NodeEmbeddingRequest\x12\x10\n\x08node_ids\x18\x01 \x03(\r"Y\n\x15NodeEmbeddingResponse\x12\x17\n\x0f\x65mbeddings_data\x18\x01 \x01(\x0c\x12\x12\n\ndimensions\x18\x02 \x03(\x05\x12\x13\n\x0bmissing_ids\x18\x03 \x03(\rb\x06proto3'
+)
 
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
-_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'embedding_pb2', globals())
-if _descriptor._USE_C_DESCRIPTORS == False:
-
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "embedding_pb2", globals())
+if not _descriptor._USE_C_DESCRIPTORS:
     DESCRIPTOR._options = None
     _NODEEMBEDDINGREQUEST._serialized_start = 35
     _NODEEMBEDDINGREQUEST._serialized_end = 75

packages/leann-backend-diskann/leann_backend_diskann/embedding_server.py

@@ -1,397 +0,0 @@
-#!/usr/bin/env python3
-"""
-Embedding server for leann-backend-diskann - Fixed ZMQ REQ-REP pattern
-"""
-
-import pickle
-import argparse
-import threading
-import time
-
-from transformers import AutoTokenizer, AutoModel
-import os
-from contextlib import contextmanager
-import zmq
-import numpy as np
-
-RED = "\033[91m"
-RESET = "\033[0m"
-
-# 简化的文档存储 - 替代 LazyPassages
-class SimpleDocumentStore:
-    """简化的文档存储，支持任意ID"""
-    def __init__(self, documents: dict = None):
-        self.documents = documents or {}
-        # 默认演示文档
-        self.default_docs = {
-            0: "Python is a high-level, interpreted language known for simplicity.",
-            1: "Machine learning builds systems that learn from data.", 
-            2: "Data structures like arrays, lists, and graphs organize data.",
-        }
-    
-    def __getitem__(self, doc_id):
-        doc_id = int(doc_id)
-        
-        # 优先使用指定的文档
-        if doc_id in self.documents:
-            return {"text": self.documents[doc_id]}
-        
-        # 其次使用默认演示文档
-        if doc_id in self.default_docs:
-            return {"text": self.default_docs[doc_id]}
-        
-        # 对于任意其他ID，返回通用文档
-        fallback_docs = [
-            "This is a general document about technology and programming concepts.",
-            "This document discusses machine learning and artificial intelligence topics.", 
-            "This content covers data structures, algorithms, and computer science fundamentals.",
-            "This is a document about software engineering and development practices.",
-            "This content focuses on databases, data management, and information systems."
-        ]
-        
-        # 根据ID选择一个fallback文档
-        fallback_text = fallback_docs[doc_id % len(fallback_docs)]
-        return {"text": f"[ID:{doc_id}] {fallback_text}"}
-    
-    def __len__(self):
-        return len(self.documents) + len(self.default_docs)
-
-def create_embedding_server_thread(
-    zmq_port=5555,
-    model_name="sentence-transformers/all-mpnet-base-v2",
-    max_batch_size=128,
-):
-    """
-    在当前线程中创建并运行 embedding server
-    这个函数设计为在单独的线程中调用
-    """
-    print(f"INFO: Initializing embedding server thread on port {zmq_port}")
-    
-    try:
-        # 检查端口是否已被占用
-        import socket
-        def check_port(port):
-            with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-                return s.connect_ex(('localhost', port)) == 0
-
-        if check_port(zmq_port):
-            print(f"{RED}Port {zmq_port} is already in use{RESET}")
-            return
-
-        # 初始化模型
-        tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)
-        import torch
-
-        # 选择设备
-        mps_available = hasattr(torch.backends, 'mps') and torch.backends.mps.is_available()
-        cuda_available = torch.cuda.is_available()
-        
-        if cuda_available:
-            device = torch.device("cuda")
-            print("INFO: Using CUDA device")
-        elif mps_available:
-            device = torch.device("mps")
-            print("INFO: Using MPS device (Apple Silicon)")
-        else:
-            device = torch.device("cpu")
-            print("INFO: Using CPU device")
-        
-        # 加载模型
-        print(f"INFO: Loading model {model_name}")
-        model = AutoModel.from_pretrained(model_name).to(device).eval()
-
-        # 优化模型
-        if cuda_available or mps_available:
-            try:
-                model = model.half()
-                model = torch.compile(model)
-                print(f"INFO: Using FP16 precision with model: {model_name}")
-            except Exception as e:
-                print(f"WARNING: Model optimization failed: {e}")
-
-        # 默认演示文档
-        demo_documents = {
-            0: "Python is a high-level, interpreted language known for simplicity.",
-            1: "Machine learning builds systems that learn from data.", 
-            2: "Data structures like arrays, lists, and graphs organize data.",
-        }
-
-        passages = SimpleDocumentStore(demo_documents)
-        print(f"INFO: Loaded {len(passages)} demo documents")
-
-        class DeviceTimer:
-            """设备计时器"""
-            def __init__(self, name="", device=device):
-                self.name = name
-                self.device = device
-                self.start_time = 0
-                self.end_time = 0
-                
-                if cuda_available:
-                    self.start_event = torch.cuda.Event(enable_timing=True)
-                    self.end_event = torch.cuda.Event(enable_timing=True)
-                else:
-                    self.start_event = None
-                    self.end_event = None
-
-            @contextmanager
-            def timing(self):
-                self.start()
-                yield
-                self.end()
-
-            def start(self):
-                if cuda_available:
-                    torch.cuda.synchronize()
-                    self.start_event.record()
-                else:
-                    if self.device.type == "mps":
-                        torch.mps.synchronize()
-                    self.start_time = time.time()
-
-            def end(self):
-                if cuda_available:
-                    self.end_event.record()
-                    torch.cuda.synchronize()
-                else:
-                    if self.device.type == "mps":
-                        torch.mps.synchronize()
-                    self.end_time = time.time()
-
-            def elapsed_time(self):
-                if cuda_available:
-                    return self.start_event.elapsed_time(self.end_event) / 1000.0
-                else:
-                    return self.end_time - self.start_time
-
-            def print_elapsed(self):
-                print(f"Time taken for {self.name}: {self.elapsed_time():.6f} seconds")
-
-        def process_batch(texts_batch, ids_batch, missing_ids):
-            """处理文本批次"""
-            batch_size = len(texts_batch)
-            print(f"INFO: Processing batch of size {batch_size}")
-
-            tokenize_timer = DeviceTimer("tokenization (batch)", device)
-            to_device_timer = DeviceTimer("transfer to device (batch)", device)
-            embed_timer = DeviceTimer("embedding (batch)", device)
-            pool_timer = DeviceTimer("mean pooling (batch)", device)
-
-            with tokenize_timer.timing():
-                encoded_batch = tokenizer.batch_encode_plus(
-                    texts_batch,
-                    padding="max_length",
-                    truncation=True,
-                    max_length=256,
-                    return_tensors="pt",
-                    return_token_type_ids=False,
-                )
-            tokenize_timer.print_elapsed()
-
-            seq_length = encoded_batch["input_ids"].size(1)
-            print(f"Batch size: {batch_size}, Sequence length: {seq_length}")
-
-            with to_device_timer.timing():
-                enc = {k: v.to(device) for k, v in encoded_batch.items()}
-            to_device_timer.print_elapsed()
-
-            with torch.no_grad():
-                with embed_timer.timing():
-                    out = model(enc["input_ids"], enc["attention_mask"])
-                embed_timer.print_elapsed()
-
-                with pool_timer.timing():
-                    hidden_states = out.last_hidden_state if hasattr(out, "last_hidden_state") else out
-                    mask_expanded = enc["attention_mask"].unsqueeze(-1).expand(hidden_states.size()).float()
-                    sum_embeddings = torch.sum(hidden_states * mask_expanded, 1)
-                    sum_mask = torch.clamp(mask_expanded.sum(1), min=1e-9)
-                    batch_embeddings = sum_embeddings / sum_mask
-                pool_timer.print_elapsed()
-
-            return batch_embeddings.cpu().numpy()
-
-        # ZMQ server 主循环 - 修改为REP套接字
-        context = zmq.Context()
-        socket = context.socket(zmq.ROUTER)  # 改为REP套接字
-        socket.bind(f"tcp://127.0.0.1:{zmq_port}")
-        print(f"INFO: ZMQ ROUTER server listening on port {zmq_port}")
-
-        # 设置超时
-        socket.setsockopt(zmq.RCVTIMEO, 5000)  # 5秒接收超时
-        socket.setsockopt(zmq.SNDTIMEO, 300000)  # 300秒发送超时
-
-        from . import embedding_pb2
-
-        print(f"INFO: Embedding server ready to serve requests")
-
-        while True:
-            try:
-                parts = socket.recv_multipart()
-
-                # --- 恢复稳健的消息格式判断 ---
-                # 必须检查 parts 的长度，避免 IndexError
-                if len(parts) >= 3:
-                    identity = parts[0]
-                    # empty = parts[1]  # 中间的空帧我们通常不关心
-                    message = parts[2]
-                elif len(parts) == 2:
-                    # 也能处理没有空帧的情况
-                    identity = parts[0]
-                    message = parts[1]
-                else:
-                    # 如果收到格式错误的消息，打印警告并忽略它，而不是崩溃
-                    print(f"WARNING: Received unexpected message format with {len(parts)} parts. Ignoring.")
-                    continue
-                print(f"INFO: Received ZMQ request from client {identity.hex()[:8]}, size {len(message)} bytes")
-
-                e2e_start = time.time()
-                lookup_timer = DeviceTimer("text lookup", device)
-
-                # 解析请求
-                req_proto = embedding_pb2.NodeEmbeddingRequest()
-                req_proto.ParseFromString(message)
-                node_ids = req_proto.node_ids
-                print(f"INFO: Request for {len(node_ids)} node embeddings: {list(node_ids)}")
-
-                # 添加调试信息
-                if len(node_ids) > 0:
-                    print(f"DEBUG: Node ID range: {min(node_ids)} to {max(node_ids)}")
-                
-                # 查找文本
-                texts = []
-                missing_ids = []
-                with lookup_timer.timing():
-                    for nid in node_ids:
-                        txtinfo = passages[nid]
-                        txt = txtinfo["text"]
-                        texts.append(txt)
-                lookup_timer.print_elapsed()
-
-                if missing_ids:
-                    print(f"WARNING: Missing passages for IDs: {missing_ids}")
-
-                # 处理批次
-                total_size = len(texts)
-                print(f"INFO: Total batch size: {total_size}, max_batch_size: {max_batch_size}")
-                
-                all_embeddings = []
-                
-                if total_size > max_batch_size:
-                    print(f"INFO: Splitting batch of size {total_size} into chunks of {max_batch_size}")
-                    for i in range(0, total_size, max_batch_size):
-                        end_idx = min(i + max_batch_size, total_size)
-                        print(f"INFO: Processing chunk {i//max_batch_size + 1}/{(total_size + max_batch_size - 1)//max_batch_size}: items {i} to {end_idx-1}")
-                        
-                        chunk_texts = texts[i:end_idx]
-                        chunk_ids = node_ids[i:end_idx]
-                        
-                        embeddings_chunk = process_batch(chunk_texts, chunk_ids, missing_ids)
-                        all_embeddings.append(embeddings_chunk)
-                        
-                        if cuda_available:
-                            torch.cuda.empty_cache()
-                        elif device.type == "mps":
-                            torch.mps.empty_cache()
-                            
-                    hidden = np.vstack(all_embeddings)
-                    print(f"INFO: Combined embeddings shape: {hidden.shape}")
-                else:
-                    hidden = process_batch(texts, node_ids, missing_ids)
-
-                # 序列化响应
-                ser_start = time.time()
-
-                resp_proto = embedding_pb2.NodeEmbeddingResponse()
-                hidden_contiguous = np.ascontiguousarray(hidden, dtype=np.float32)
-                resp_proto.embeddings_data = hidden_contiguous.tobytes()
-                resp_proto.dimensions.append(hidden_contiguous.shape[0])
-                resp_proto.dimensions.append(hidden_contiguous.shape[1])
-                resp_proto.missing_ids.extend(missing_ids)
-
-                response_data = resp_proto.SerializeToString()
-                
-                # REP 套接字发送单个响应
-                socket.send_multipart([identity, b'', response_data])
-
-                ser_end = time.time()
-
-                print(f"INFO: Serialize time: {ser_end - ser_start:.6f} seconds")
-
-                if device.type == "cuda":
-                    torch.cuda.synchronize()
-                elif device.type == "mps":
-                    torch.mps.synchronize()
-                e2e_end = time.time()
-                print(f"INFO: ZMQ E2E time: {e2e_end - e2e_start:.6f} seconds")
-
-            except zmq.Again:
-                print("INFO: ZMQ socket timeout, continuing to listen")
-                # REP套接字不需要重新创建，只需要继续监听
-                continue
-            except Exception as e:
-                print(f"ERROR: Error in ZMQ server: {e}")
-                try:
-                    # 发送空响应以维持REQ-REP状态
-                    empty_resp = embedding_pb2.NodeEmbeddingResponse()
-                    socket.send(empty_resp.SerializeToString())
-                except:
-                    # 如果发送失败，重新创建socket
-                    socket.close()
-                    socket = context.socket(zmq.REP)
-                    socket.bind(f"tcp://127.0.0.1:{zmq_port}")
-                    socket.setsockopt(zmq.RCVTIMEO, 5000)
-                    socket.setsockopt(zmq.SNDTIMEO, 300000)
-                    print("INFO: ZMQ socket recreated after error")
-
-    except Exception as e:
-        print(f"ERROR: Failed to start embedding server: {e}")
-        raise
-
-
-# 保持原有的 create_embedding_server 函数不变，只添加线程化版本
-def create_embedding_server(
-    domain="demo",
-    load_passages=True,
-    load_embeddings=False,
-    use_fp16=True,
-    use_int8=False,
-    use_cuda_graphs=False,
-    zmq_port=5555,
-    max_batch_size=128,
-    lazy_load_passages=False,
-    model_name="sentence-transformers/all-mpnet-base-v2",
-):
-    """
-    原有的 create_embedding_server 函数保持不变
-    这个是阻塞版本，用于直接运行
-    """
-    create_embedding_server_thread(zmq_port, model_name, max_batch_size)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Embedding service")
-    parser.add_argument("--zmq-port", type=int, default=5555, help="ZMQ port to run on")
-    parser.add_argument("--domain", type=str, default="demo", help="Domain name")
-    parser.add_argument("--load-passages", action="store_true", default=True)
-    parser.add_argument("--load-embeddings", action="store_true", default=False)
-    parser.add_argument("--use-fp16", action="store_true", default=False)
-    parser.add_argument("--use-int8", action="store_true", default=False)
-    parser.add_argument("--use-cuda-graphs", action="store_true", default=False)
-    parser.add_argument("--max-batch-size", type=int, default=128, help="Maximum batch size before splitting")
-    parser.add_argument("--lazy-load-passages", action="store_true", default=True)
-    parser.add_argument("--model-name", type=str, default="sentence-transformers/all-mpnet-base-v2", 
-                        help="Embedding model name")
-    args = parser.parse_args()
-
-    create_embedding_server(
-        domain=args.domain,
-        load_passages=args.load_passages,
-        load_embeddings=args.load_embeddings,
-        use_fp16=args.use_fp16,
-        use_int8=args.use_int8,
-        use_cuda_graphs=args.use_cuda_graphs,
-        zmq_port=args.zmq_port,
-        max_batch_size=args.max_batch_size,
-        lazy_load_passages=args.lazy_load_passages,
-        model_name=args.model_name,
-    )

packages/leann-backend-diskann/leann_backend_diskann/graph_partition.py

@@ -0,0 +1,299 @@
+#!/usr/bin/env python3
+"""
+Graph Partition Module for LEANN DiskANN Backend
+
+This module provides Python bindings for the graph partition functionality
+of DiskANN, allowing users to partition disk-based indices for better
+performance.
+"""
+
+import os
+import shutil
+import subprocess
+import tempfile
+from pathlib import Path
+from typing import Optional
+
+
+class GraphPartitioner:
+    """
+    A Python interface for DiskANN's graph partition functionality.
+
+    This class provides methods to partition disk-based indices for improved
+    search performance and memory efficiency.
+    """
+
+    def __init__(self, build_type: str = "release"):
+        """
+        Initialize the GraphPartitioner.
+
+        Args:
+            build_type: Build type for the executables ("debug" or "release")
+        """
+        self.build_type = build_type
+        self._ensure_executables()
+
+    def _get_executable_path(self, name: str) -> str:
+        """Get the path to a graph partition executable."""
+        # Get the directory where this Python module is located
+        module_dir = Path(__file__).parent
+        # Navigate to the graph_partition directory
+        graph_partition_dir = module_dir.parent / "third_party" / "DiskANN" / "graph_partition"
+        executable_path = graph_partition_dir / "build" / self.build_type / "graph_partition" / name
+
+        if not executable_path.exists():
+            raise FileNotFoundError(f"Executable {name} not found at {executable_path}")
+
+        return str(executable_path)
+
+    def _ensure_executables(self):
+        """Ensure that the required executables are built."""
+        try:
+            self._get_executable_path("partitioner")
+            self._get_executable_path("index_relayout")
+        except FileNotFoundError:
+            # Try to build the executables automatically
+            print("Executables not found, attempting to build them...")
+            self._build_executables()
+
+    def _build_executables(self):
+        """Build the required executables."""
+        graph_partition_dir = (
+            Path(__file__).parent.parent / "third_party" / "DiskANN" / "graph_partition"
+        )
+        original_dir = os.getcwd()
+
+        try:
+            os.chdir(graph_partition_dir)
+
+            # Clean any existing build
+            if (graph_partition_dir / "build").exists():
+                shutil.rmtree(graph_partition_dir / "build")
+
+            # Run the build script
+            cmd = ["./build.sh", self.build_type, "split_graph", "/tmp/dummy"]
+            subprocess.run(cmd, capture_output=True, text=True, cwd=graph_partition_dir)
+
+            # Check if executables were created
+            partitioner_path = self._get_executable_path("partitioner")
+            relayout_path = self._get_executable_path("index_relayout")
+
+            print(f"✅ Built partitioner: {partitioner_path}")
+            print(f"✅ Built index_relayout: {relayout_path}")
+
+        except Exception as e:
+            raise RuntimeError(f"Failed to build executables: {e}")
+        finally:
+            os.chdir(original_dir)
+
+    def partition_graph(
+        self,
+        index_prefix_path: str,
+        output_dir: Optional[str] = None,
+        partition_prefix: Optional[str] = None,
+        **kwargs,
+    ) -> tuple[str, str]:
+        """
+        Partition a disk-based index for improved performance.
+
+        Args:
+            index_prefix_path: Path to the index prefix (e.g., "/path/to/index")
+            output_dir: Output directory for results (defaults to parent of index_prefix_path)
+            partition_prefix: Prefix for output files (defaults to basename of index_prefix_path)
+            **kwargs: Additional parameters for graph partitioning:
+                - gp_times: Number of LDG partition iterations (default: 10)
+                - lock_nums: Number of lock nodes (default: 10)
+                - cut: Cut adjacency list degree (default: 100)
+                - scale_factor: Scale factor (default: 1)
+                - data_type: Data type (default: "float")
+                - thread_nums: Number of threads (default: 10)
+
+        Returns:
+            Tuple of (disk_graph_index_path, partition_bin_path)
+
+        Raises:
+            RuntimeError: If the partitioning process fails
+        """
+        # Set default parameters
+        params = {
+            "gp_times": 10,
+            "lock_nums": 10,
+            "cut": 100,
+            "scale_factor": 1,
+            "data_type": "float",
+            "thread_nums": 10,
+            **kwargs,
+        }
+
+        # Determine output directory
+        if output_dir is None:
+            output_dir = str(Path(index_prefix_path).parent)
+
+        # Create output directory if it doesn't exist
+        Path(output_dir).mkdir(parents=True, exist_ok=True)
+
+        # Determine partition prefix
+        if partition_prefix is None:
+            partition_prefix = Path(index_prefix_path).name
+
+        # Get executable paths
+        partitioner_path = self._get_executable_path("partitioner")
+        relayout_path = self._get_executable_path("index_relayout")
+
+        # Create temporary directory for processing
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Change to the graph_partition directory for temporary files
+            graph_partition_dir = (
+                Path(__file__).parent.parent / "third_party" / "DiskANN" / "graph_partition"
+            )
+            original_dir = os.getcwd()
+
+            try:
+                os.chdir(graph_partition_dir)
+
+                # Create temporary data directory
+                temp_data_dir = Path(temp_dir) / "data"
+                temp_data_dir.mkdir(parents=True, exist_ok=True)
+
+                # Set up paths for temporary files
+                graph_path = temp_data_dir / "starling" / "_M_R_L_B" / "GRAPH"
+                graph_gp_path = (
+                    graph_path
+                    / f"GP_TIMES_{params['gp_times']}_LOCK_{params['lock_nums']}_GP_USE_FREQ0_CUT{params['cut']}_SCALE{params['scale_factor']}"
+                )
+                graph_gp_path.mkdir(parents=True, exist_ok=True)
+
+                # Find input index file
+                old_index_file = f"{index_prefix_path}_disk_beam_search.index"
+                if not os.path.exists(old_index_file):
+                    old_index_file = f"{index_prefix_path}_disk.index"
+
+                if not os.path.exists(old_index_file):
+                    raise RuntimeError(f"Index file not found: {old_index_file}")
+
+                # Run partitioner
+                gp_file_path = graph_gp_path / "_part.bin"
+                partitioner_cmd = [
+                    partitioner_path,
+                    "--index_file",
+                    old_index_file,
+                    "--data_type",
+                    params["data_type"],
+                    "--gp_file",
+                    str(gp_file_path),
+                    "-T",
+                    str(params["thread_nums"]),
+                    "--ldg_times",
+                    str(params["gp_times"]),
+                    "--scale",
+                    str(params["scale_factor"]),
+                    "--mode",
+                    "1",
+                ]
+
+                print(f"Running partitioner: {' '.join(partitioner_cmd)}")
+                result = subprocess.run(
+                    partitioner_cmd, capture_output=True, text=True, cwd=graph_partition_dir
+                )
+
+                if result.returncode != 0:
+                    raise RuntimeError(
+                        f"Partitioner failed with return code {result.returncode}.\n"
+                        f"stdout: {result.stdout}\n"
+                        f"stderr: {result.stderr}"
+                    )
+
+                # Run relayout
+                part_tmp_index = graph_gp_path / "_part_tmp.index"
+                relayout_cmd = [
+                    relayout_path,
+                    old_index_file,
+                    str(gp_file_path),
+                    params["data_type"],
+                    "1",
+                ]
+
+                print(f"Running relayout: {' '.join(relayout_cmd)}")
+                result = subprocess.run(
+                    relayout_cmd, capture_output=True, text=True, cwd=graph_partition_dir
+                )
+
+                if result.returncode != 0:
+                    raise RuntimeError(
+                        f"Relayout failed with return code {result.returncode}.\n"
+                        f"stdout: {result.stdout}\n"
+                        f"stderr: {result.stderr}"
+                    )
+
+                # Copy results to output directory
+                disk_graph_path = Path(output_dir) / f"{partition_prefix}_disk_graph.index"
+                partition_bin_path = Path(output_dir) / f"{partition_prefix}_partition.bin"
+
+                shutil.copy2(part_tmp_index, disk_graph_path)
+                shutil.copy2(gp_file_path, partition_bin_path)
+
+                print(f"Results copied to: {output_dir}")
+                return str(disk_graph_path), str(partition_bin_path)
+
+            finally:
+                os.chdir(original_dir)
+
+    def get_partition_info(self, partition_bin_path: str) -> dict:
+        """
+        Get information about a partition file.
+
+        Args:
+            partition_bin_path: Path to the partition binary file
+
+        Returns:
+            Dictionary containing partition information
+        """
+        if not os.path.exists(partition_bin_path):
+            raise FileNotFoundError(f"Partition file not found: {partition_bin_path}")
+
+        # For now, return basic file information
+        # In the future, this could parse the binary file for detailed info
+        stat = os.stat(partition_bin_path)
+        return {
+            "file_size": stat.st_size,
+            "file_path": partition_bin_path,
+            "modified_time": stat.st_mtime,
+        }
+
+
+def partition_graph(
+    index_prefix_path: str,
+    output_dir: Optional[str] = None,
+    partition_prefix: Optional[str] = None,
+    build_type: str = "release",
+    **kwargs,
+) -> tuple[str, str]:
+    """
+    Convenience function to partition a graph index.
+
+    Args:
+        index_prefix_path: Path to the index prefix
+        output_dir: Output directory (defaults to parent of index_prefix_path)
+        partition_prefix: Prefix for output files (defaults to basename of index_prefix_path)
+        build_type: Build type for executables ("debug" or "release")
+        **kwargs: Additional parameters for graph partitioning
+
+    Returns:
+        Tuple of (disk_graph_index_path, partition_bin_path)
+    """
+    partitioner = GraphPartitioner(build_type=build_type)
+    return partitioner.partition_graph(index_prefix_path, output_dir, partition_prefix, **kwargs)
+
+
+# Example usage:
+if __name__ == "__main__":
+    # Example: partition an index
+    try:
+        disk_graph_path, partition_bin_path = partition_graph(
+            "/path/to/your/index_prefix", gp_times=10, lock_nums=10, cut=100
+        )
+        print("Partitioning completed successfully!")
+        print(f"Disk graph index: {disk_graph_path}")
+        print(f"Partition binary: {partition_bin_path}")
+    except Exception as e:
+        print(f"Partitioning failed: {e}")

packages/leann-backend-diskann/pyproject.toml

@@ -4,13 +4,18 @@ build-backend = "scikit_build_core.build"
 
 [project]
 name = "leann-backend-diskann"
-version = "0.1.0"
-dependencies = ["leann-core==0.1.0", "numpy"]
+version = "0.3.2"
+dependencies = ["leann-core==0.3.2", "numpy", "protobuf>=3.19.0"]
 
 [tool.scikit-build]
-# 关键：简化的 CMake 路径
+# Key: simplified CMake path
 cmake.source-dir = "third_party/DiskANN"
-# 关键：Python 包在根目录，路径完全匹配
+# Key: Python package in root directory, paths match exactly
 wheel.packages = ["leann_backend_diskann"]
-# 使用默认的 redirect 模式
+# Use default redirect mode
 editable.mode = "redirect"
+cmake.build-type = "Release"
+build.verbose = true
+build.tool-args = ["-j8"]
+# Let CMake find packages via Homebrew prefix
+cmake.define = {CMAKE_PREFIX_PATH = {env = "CMAKE_PREFIX_PATH"}, OpenMP_ROOT = {env = "OpenMP_ROOT"}}

packages/leann-backend-hnsw/CMakeLists.txt

@@ -1,7 +1,47 @@
-# 最终简化版
 cmake_minimum_required(VERSION 3.24)
 project(leann_backend_hnsw_wrapper)
+set(CMAKE_C_COMPILER_WORKS 1)
+set(CMAKE_CXX_COMPILER_WORKS 1)
 
+# Set OpenMP path for macOS
+if(APPLE)
+    # Detect Homebrew installation path (Apple Silicon vs Intel)
+    if(EXISTS "/opt/homebrew/opt/libomp")
+        set(HOMEBREW_PREFIX "/opt/homebrew")
+    elseif(EXISTS "/usr/local/opt/libomp")
+        set(HOMEBREW_PREFIX "/usr/local")
+    else()
+        message(FATAL_ERROR "Could not find libomp installation. Please install with: brew install libomp")
+    endif()
+
+    set(OpenMP_C_FLAGS "-Xpreprocessor -fopenmp -I${HOMEBREW_PREFIX}/opt/libomp/include")
+    set(OpenMP_CXX_FLAGS "-Xpreprocessor -fopenmp -I${HOMEBREW_PREFIX}/opt/libomp/include")
+    set(OpenMP_C_LIB_NAMES "omp")
+    set(OpenMP_CXX_LIB_NAMES "omp")
+    set(OpenMP_omp_LIBRARY "${HOMEBREW_PREFIX}/opt/libomp/lib/libomp.dylib")
+
+    # Force use of system libc++ to avoid version mismatch
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -stdlib=libc++")
+    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -stdlib=libc++")
+
+    # Set minimum macOS version for better compatibility
+    set(CMAKE_OSX_DEPLOYMENT_TARGET "11.0" CACHE STRING "Minimum macOS version")
+endif()
+
+# Use system ZeroMQ instead of building from source
+find_package(PkgConfig REQUIRED)
+pkg_check_modules(ZMQ REQUIRED libzmq)
+
+# Add cppzmq headers
+include_directories(third_party/cppzmq)
+
+# Configure msgpack-c - disable boost dependency
+set(MSGPACK_USE_BOOST OFF CACHE BOOL "" FORCE)
+add_compile_definitions(MSGPACK_NO_BOOST)
+include_directories(third_party/msgpack-c/include)
+
+# Faiss configuration - streamlined build
 set(FAISS_ENABLE_PYTHON ON CACHE BOOL "" FORCE)
 set(FAISS_ENABLE_GPU OFF CACHE BOOL "" FORCE)
 set(FAISS_ENABLE_EXTRAS OFF CACHE BOOL "" FORCE)
@@ -9,4 +49,24 @@ set(BUILD_TESTING OFF CACHE BOOL "" FORCE)
 set(FAISS_ENABLE_C_API OFF CACHE BOOL "" FORCE)
 set(FAISS_OPT_LEVEL "generic" CACHE STRING "" FORCE)
 
+# Disable additional SIMD versions to speed up compilation
+set(FAISS_ENABLE_AVX2 OFF CACHE BOOL "" FORCE)
+set(FAISS_ENABLE_AVX512 OFF CACHE BOOL "" FORCE)
+
+# Additional optimization options from INSTALL.md
+set(CMAKE_BUILD_TYPE "Release" CACHE STRING "" FORCE)
+set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)  # Static library is faster to build
+
+# Avoid building demos and benchmarks
+set(BUILD_DEMOS OFF CACHE BOOL "" FORCE)
+set(BUILD_BENCHS OFF CACHE BOOL "" FORCE)
+
+# NEW: Tell Faiss to only build the generic version
+set(FAISS_BUILD_GENERIC ON CACHE BOOL "" FORCE)
+set(FAISS_BUILD_AVX2 OFF CACHE BOOL "" FORCE)
+set(FAISS_BUILD_AVX512 OFF CACHE BOOL "" FORCE)
+
+# IMPORTANT: Disable building AVX versions to speed up compilation
+set(FAISS_BUILD_AVX_VERSIONS OFF CACHE BOOL "" FORCE)
+
 add_subdirectory(third_party/faiss)

packages/leann-backend-hnsw/leann_backend_hnsw/__init__.py

@@ -1 +1 @@
-from . import hnsw_backend
+from . import hnsw_backend as hnsw_backend

packages/leann-backend-hnsw/leann_backend_hnsw/convert_to_csr.py

@@ -1,69 +1,98 @@
-import struct
-import sys
-import numpy as np
-import os
 import argparse
 import gc  # Import garbage collector interface
+import logging
+import os
+import struct
+import sys
 import time
+
+import numpy as np
+
+# Set up logging to avoid print buffer issues
+logger = logging.getLogger(__name__)
+LOG_LEVEL = os.getenv("LEANN_LOG_LEVEL", "WARNING").upper()
+log_level = getattr(logging, LOG_LEVEL, logging.WARNING)
+logger.setLevel(log_level)
+
 # --- FourCCs (add more if needed) ---
-INDEX_HNSW_FLAT_FOURCC = int.from_bytes(b'IHNf', 'little')
+INDEX_HNSW_FLAT_FOURCC = int.from_bytes(b"IHNf", "little")
 # Add other HNSW fourccs if you expect different storage types inside HNSW
 # INDEX_HNSW_PQ_FOURCC = int.from_bytes(b'IHNp', 'little')
 # INDEX_HNSW_SQ_FOURCC = int.from_bytes(b'IHNs', 'little')
 # INDEX_HNSW_CAGRA_FOURCC = int.from_bytes(b'IHNc', 'little') # Example
 
 EXPECTED_HNSW_FOURCCS = {INDEX_HNSW_FLAT_FOURCC}  # Modify if needed
-NULL_INDEX_FOURCC = int.from_bytes(b'null', 'little')
+NULL_INDEX_FOURCC = int.from_bytes(b"null", "little")
 
 # --- Helper functions for reading/writing binary data ---
 
+
 def read_struct(f, fmt):
     """Reads data according to the struct format."""
     size = struct.calcsize(fmt)
     data = f.read(size)
     if len(data) != size:
-        raise EOFError(f"File ended unexpectedly reading struct fmt '{fmt}'. Expected {size} bytes, got {len(data)}.")
+        raise EOFError(
+            f"File ended unexpectedly reading struct fmt '{fmt}'. Expected {size} bytes, got {len(data)}."
+        )
     return struct.unpack(fmt, data)[0]
 
+
 def read_vector_raw(f, element_fmt_char):
     """Reads a vector (size followed by data), returns count and raw bytes."""
     count = -1  # Initialize count
     total_bytes = -1  # Initialize total_bytes
     try:
-        count = read_struct(f, '<Q') # size_t usually 64-bit unsigned
+        count = read_struct(f, "<Q")  # size_t usually 64-bit unsigned
         element_size = struct.calcsize(element_fmt_char)
         # --- FIX for MemoryError: Check for unreasonably large count ---
         max_reasonable_count = 10 * (10**9)  # ~10 billion elements limit
         if count > max_reasonable_count or count < 0:
-            raise MemoryError(f"Vector count {count} seems unreasonably large, possibly due to file corruption or incorrect format read.")
+            raise MemoryError(
+                f"Vector count {count} seems unreasonably large, possibly due to file corruption or incorrect format read."
+            )
 
         total_bytes = count * element_size
         # --- FIX for MemoryError: Check for huge byte size before allocation ---
         max_reasonable_bytes = 50 * (1024**3)  # ~50 GB limit
         if total_bytes > max_reasonable_bytes or total_bytes < 0:  # Check for overflow
-             raise MemoryError(f"Attempting to read {total_bytes} bytes ({count} elements * {element_size} bytes/element), which exceeds the safety limit. File might be corrupted or format mismatch.")
+            raise MemoryError(
+                f"Attempting to read {total_bytes} bytes ({count} elements * {element_size} bytes/element), which exceeds the safety limit. File might be corrupted or format mismatch."
+            )
 
         data_bytes = f.read(total_bytes)
 
         if len(data_bytes) != total_bytes:
-             raise EOFError(f"File ended unexpectedly reading vector data. Expected {total_bytes} bytes, got {len(data_bytes)}.")
+            raise EOFError(
+                f"File ended unexpectedly reading vector data. Expected {total_bytes} bytes, got {len(data_bytes)}."
+            )
         return count, data_bytes
     except (MemoryError, OverflowError) as e:
         # Add context to the error message
-         print(f"\nError during raw vector read (element_fmt='{element_fmt_char}', count={count}, total_bytes={total_bytes}): {e}", file=sys.stderr)
+        print(
+            f"\nError during raw vector read (element_fmt='{element_fmt_char}', count={count}, total_bytes={total_bytes}): {e}",
+            file=sys.stderr,
+        )
         raise e  # Re-raise the original error type
 
+
 def read_numpy_vector(f, np_dtype, struct_fmt_char):
     """Reads a vector into a NumPy array."""
     count = -1  # Initialize count for robust error handling
-    print(f"  Reading vector (dtype={np_dtype}, fmt='{struct_fmt_char}')... ", end='', flush=True)
+    print(
+        f"  Reading vector (dtype={np_dtype}, fmt='{struct_fmt_char}')... ",
+        end="",
+        flush=True,
+    )
     try:
         count, data_bytes = read_vector_raw(f, struct_fmt_char)
         print(f"Count={count}, Bytes={len(data_bytes)}")
         if count > 0 and len(data_bytes) > 0:
             arr = np.frombuffer(data_bytes, dtype=np_dtype)
             if arr.size != count:
-                raise ValueError(f"Inconsistent array size after reading. Expected {count}, got {arr.size}")
+                raise ValueError(
+                    f"Inconsistent array size after reading. Expected {count}, got {arr.size}"
+                )
             return arr
         elif count == 0:
             return np.array([], dtype=np_dtype)
@@ -71,17 +100,23 @@ def read_numpy_vector(f, np_dtype, struct_fmt_char):
             raise ValueError("Read zero bytes but count > 0.")
     except MemoryError as e:
         # Now count should be defined (or -1 if error was in read_struct)
-        print(f"\nMemoryError creating NumPy array (dtype={np_dtype}, count={count}). {e}", file=sys.stderr)
+        print(
+            f"\nMemoryError creating NumPy array (dtype={np_dtype}, count={count}). {e}",
+            file=sys.stderr,
+        )
         raise e
     except Exception as e:  # Catch other potential errors like ValueError
-        print(f"\nError reading numpy vector (dtype={np_dtype}, fmt='{struct_fmt_char}', count={count}): {e}", file=sys.stderr)
+        print(
+            f"\nError reading numpy vector (dtype={np_dtype}, fmt='{struct_fmt_char}', count={count}): {e}",
+            file=sys.stderr,
+        )
         raise e
 
 
 def write_numpy_vector(f, arr, struct_fmt_char):
     """Writes a NumPy array as a vector (size followed by data)."""
     count = arr.size
-    f.write(struct.pack('<Q', count))
+    f.write(struct.pack("<Q", count))
     try:
         expected_dtype = np.dtype(struct_fmt_char)
         if arr.dtype != expected_dtype:
@@ -91,21 +126,28 @@ def write_numpy_vector(f, arr, struct_fmt_char):
         f.write(data_to_write)
         del data_to_write  # Hint GC
     except MemoryError as e:
-         print(f"\nMemoryError converting NumPy array to bytes for writing (size={count}, dtype={arr.dtype}). {e}", file=sys.stderr)
+        print(
+            f"\nMemoryError converting NumPy array to bytes for writing (size={count}, dtype={arr.dtype}). {e}",
+            file=sys.stderr,
+        )
         raise e
 
+
 def write_list_vector(f, lst, struct_fmt_char):
     """Writes a Python list as a vector iteratively."""
     count = len(lst)
-    f.write(struct.pack('<Q', count))
-    fmt = '<' + struct_fmt_char
+    f.write(struct.pack("<Q", count))
+    fmt = "<" + struct_fmt_char
     chunk_size = 1024 * 1024
     element_size = struct.calcsize(fmt)
     # Allocate buffer outside the loop if possible, or handle MemoryError during allocation
     try:
         buffer = bytearray(chunk_size * element_size)
     except MemoryError:
-        print(f"MemoryError: Cannot allocate buffer for writing list vector chunk (size {chunk_size * element_size} bytes).", file=sys.stderr)
+        print(
+            f"MemoryError: Cannot allocate buffer for writing list vector chunk (size {chunk_size * element_size} bytes).",
+            file=sys.stderr,
+        )
         raise
     buffer_count = 0
 
@@ -120,61 +162,75 @@ def write_list_vector(f, lst, struct_fmt_char):
                 buffer_count = 0
 
         except struct.error as e:
-            print(f"\nStruct packing error for item {item} at index {i} with format '{fmt}'. {e}", file=sys.stderr)
+            print(
+                f"\nStruct packing error for item {item} at index {i} with format '{fmt}'. {e}",
+                file=sys.stderr,
+            )
             raise e
 
 
 def get_cum_neighbors(cum_nneighbor_per_level_np, level):
     """Helper to get cumulative neighbors count, matching C++ logic."""
-    if level < 0: return 0
+    if level < 0:
+        return 0
     if level < len(cum_nneighbor_per_level_np):
         return cum_nneighbor_per_level_np[level]
     else:
         return cum_nneighbor_per_level_np[-1] if len(cum_nneighbor_per_level_np) > 0 else 0
 
-def write_compact_format(f_out, original_hnsw_data, assign_probas_np, cum_nneighbor_per_level_np, 
-                        levels_np, compact_level_ptr, compact_node_offsets_np, 
-                        compact_neighbors_data, storage_fourcc, storage_data):
+
+def write_compact_format(
+    f_out,
+    original_hnsw_data,
+    assign_probas_np,
+    cum_nneighbor_per_level_np,
+    levels_np,
+    compact_level_ptr,
+    compact_node_offsets_np,
+    compact_neighbors_data,
+    storage_fourcc,
+    storage_data,
+):
     """Write HNSW data in compact format following C++ read order exactly."""
     # Write IndexHNSW Header
-    f_out.write(struct.pack('<I', original_hnsw_data['index_fourcc']))
-    f_out.write(struct.pack('<i', original_hnsw_data['d']))
-    f_out.write(struct.pack('<q', original_hnsw_data['ntotal']))
-    f_out.write(struct.pack('<q', original_hnsw_data['dummy1']))
-    f_out.write(struct.pack('<q', original_hnsw_data['dummy2']))
-    f_out.write(struct.pack('<?', original_hnsw_data['is_trained']))
-    f_out.write(struct.pack('<i', original_hnsw_data['metric_type']))
-    if original_hnsw_data['metric_type'] > 1:
-         f_out.write(struct.pack('<f', original_hnsw_data['metric_arg']))
+    f_out.write(struct.pack("<I", original_hnsw_data["index_fourcc"]))
+    f_out.write(struct.pack("<i", original_hnsw_data["d"]))
+    f_out.write(struct.pack("<q", original_hnsw_data["ntotal"]))
+    f_out.write(struct.pack("<q", original_hnsw_data["dummy1"]))
+    f_out.write(struct.pack("<q", original_hnsw_data["dummy2"]))
+    f_out.write(struct.pack("<?", original_hnsw_data["is_trained"]))
+    f_out.write(struct.pack("<i", original_hnsw_data["metric_type"]))
+    if original_hnsw_data["metric_type"] > 1:
+        f_out.write(struct.pack("<f", original_hnsw_data["metric_arg"]))
 
     # Write HNSW struct parts (standard order)
-    write_numpy_vector(f_out, assign_probas_np, 'd')
-    write_numpy_vector(f_out, cum_nneighbor_per_level_np, 'i')
-    write_numpy_vector(f_out, levels_np, 'i')
+    write_numpy_vector(f_out, assign_probas_np, "d")
+    write_numpy_vector(f_out, cum_nneighbor_per_level_np, "i")
+    write_numpy_vector(f_out, levels_np, "i")
 
     # Write compact format flag
-    f_out.write(struct.pack('<?', True)) # storage_is_compact = True
+    f_out.write(struct.pack("<?", True))  # storage_is_compact = True
 
     # Write compact data in CORRECT C++ read order: level_ptr, node_offsets FIRST
     if isinstance(compact_level_ptr, np.ndarray):
-        write_numpy_vector(f_out, compact_level_ptr, 'Q')
+        write_numpy_vector(f_out, compact_level_ptr, "Q")
     else:
-        write_list_vector(f_out, compact_level_ptr, 'Q')
+        write_list_vector(f_out, compact_level_ptr, "Q")
 
-    write_numpy_vector(f_out, compact_node_offsets_np, 'Q')
+    write_numpy_vector(f_out, compact_node_offsets_np, "Q")
 
     # Write HNSW scalar parameters
-    f_out.write(struct.pack('<i', original_hnsw_data['entry_point']))
-    f_out.write(struct.pack('<i', original_hnsw_data['max_level']))
-    f_out.write(struct.pack('<i', original_hnsw_data['efConstruction']))
-    f_out.write(struct.pack('<i', original_hnsw_data['efSearch']))
-    f_out.write(struct.pack('<i', original_hnsw_data['dummy_upper_beam']))
+    f_out.write(struct.pack("<i", original_hnsw_data["entry_point"]))
+    f_out.write(struct.pack("<i", original_hnsw_data["max_level"]))
+    f_out.write(struct.pack("<i", original_hnsw_data["efConstruction"]))
+    f_out.write(struct.pack("<i", original_hnsw_data["efSearch"]))
+    f_out.write(struct.pack("<i", original_hnsw_data["dummy_upper_beam"]))
 
     # Write storage fourcc (this determines how to read what follows)
-    f_out.write(struct.pack('<I', storage_fourcc))
+    f_out.write(struct.pack("<I", storage_fourcc))
 
     # Write compact neighbors data AFTER storage fourcc
-    write_list_vector(f_out, compact_neighbors_data, 'i')
+    write_list_vector(f_out, compact_neighbors_data, "i")
 
     # Write storage data if not NULL (only after neighbors)
     if storage_fourcc != NULL_INDEX_FOURCC and storage_data:
@@ -183,6 +239,7 @@ def write_compact_format(f_out, original_hnsw_data, assign_probas_np, cum_nneigh
 
 # --- Main Conversion Logic ---
 
+
 def convert_hnsw_graph_to_csr(input_filename, output_filename, prune_embeddings=True):
     """
     Converts an HNSW graph file to the CSR format.
@@ -193,94 +250,120 @@ def convert_hnsw_graph_to_csr(input_filename, output_filename, prune_embeddings=
         output_filename: Output CSR index file
         prune_embeddings: Whether to prune embedding storage (write NULL storage marker)
     """
+    # Keep prints simple; rely on CI runner to flush output as needed
+
     print(f"Starting conversion: {input_filename} -> {output_filename}")
     start_time = time.time()
     original_hnsw_data = {}
     neighbors_np = None  # Initialize to allow check in finally block
     try:
-        with open(input_filename, 'rb') as f_in, open(output_filename, 'wb') as f_out:
-
+        with open(input_filename, "rb") as f_in, open(output_filename, "wb") as f_out:
             # --- Read IndexHNSW FourCC and Header ---
             print(f"[{time.time() - start_time:.2f}s] Reading Index HNSW header...")
             # ... (Keep the header reading logic as before) ...
-            hnsw_index_fourcc = read_struct(f_in, '<I')
+            hnsw_index_fourcc = read_struct(f_in, "<I")
             if hnsw_index_fourcc not in EXPECTED_HNSW_FOURCCS:
-                 print(f"Error: Expected HNSW Index FourCC ({list(EXPECTED_HNSW_FOURCCS)}), got {hnsw_index_fourcc:08x}.", file=sys.stderr)
+                print(
+                    f"Error: Expected HNSW Index FourCC ({list(EXPECTED_HNSW_FOURCCS)}), got {hnsw_index_fourcc:08x}.",
+                    file=sys.stderr,
+                )
                 return False
-            original_hnsw_data['index_fourcc'] = hnsw_index_fourcc
-            original_hnsw_data['d'] = read_struct(f_in, '<i')
-            original_hnsw_data['ntotal'] = read_struct(f_in, '<q')
-            original_hnsw_data['dummy1'] = read_struct(f_in, '<q')
-            original_hnsw_data['dummy2'] = read_struct(f_in, '<q')
-            original_hnsw_data['is_trained'] = read_struct(f_in, '?')
-            original_hnsw_data['metric_type'] = read_struct(f_in, '<i')
-            original_hnsw_data['metric_arg'] = 0.0
-            if original_hnsw_data['metric_type'] > 1:
-                 original_hnsw_data['metric_arg'] = read_struct(f_in, '<f')
-            print(f"[{time.time() - start_time:.2f}s]   Header read: d={original_hnsw_data['d']}, ntotal={original_hnsw_data['ntotal']}")
-
+            original_hnsw_data["index_fourcc"] = hnsw_index_fourcc
+            original_hnsw_data["d"] = read_struct(f_in, "<i")
+            original_hnsw_data["ntotal"] = read_struct(f_in, "<q")
+            original_hnsw_data["dummy1"] = read_struct(f_in, "<q")
+            original_hnsw_data["dummy2"] = read_struct(f_in, "<q")
+            original_hnsw_data["is_trained"] = read_struct(f_in, "?")
+            original_hnsw_data["metric_type"] = read_struct(f_in, "<i")
+            original_hnsw_data["metric_arg"] = 0.0
+            if original_hnsw_data["metric_type"] > 1:
+                original_hnsw_data["metric_arg"] = read_struct(f_in, "<f")
+            print(
+                f"[{time.time() - start_time:.2f}s]   Header read: d={original_hnsw_data['d']}, ntotal={original_hnsw_data['ntotal']}"
+            )
 
             # --- Read original HNSW struct data ---
             print(f"[{time.time() - start_time:.2f}s] Reading HNSW struct vectors...")
-            assign_probas_np = read_numpy_vector(f_in, np.float64, 'd')
-            print(f"[{time.time() - start_time:.2f}s]   Read assign_probas ({assign_probas_np.size})")
+            assign_probas_np = read_numpy_vector(f_in, np.float64, "d")
+            print(
+                f"[{time.time() - start_time:.2f}s]   Read assign_probas ({assign_probas_np.size})"
+            )
             gc.collect()
 
-            cum_nneighbor_per_level_np = read_numpy_vector(f_in, np.int32, 'i')
-            print(f"[{time.time() - start_time:.2f}s]   Read cum_nneighbor_per_level ({cum_nneighbor_per_level_np.size})")
+            cum_nneighbor_per_level_np = read_numpy_vector(f_in, np.int32, "i")
+            print(
+                f"[{time.time() - start_time:.2f}s]   Read cum_nneighbor_per_level ({cum_nneighbor_per_level_np.size})"
+            )
             gc.collect()
 
-            levels_np = read_numpy_vector(f_in, np.int32, 'i')
+            levels_np = read_numpy_vector(f_in, np.int32, "i")
             print(f"[{time.time() - start_time:.2f}s]   Read levels ({levels_np.size})")
             gc.collect()
 
             ntotal = len(levels_np)
-            if ntotal != original_hnsw_data['ntotal']:
-                 print(f"Warning: ntotal mismatch! Header says {original_hnsw_data['ntotal']}, levels vector size is {ntotal}. Using levels vector size.", file=sys.stderr)
-                 original_hnsw_data['ntotal'] = ntotal
+            if ntotal != original_hnsw_data["ntotal"]:
+                print(
+                    f"Warning: ntotal mismatch! Header says {original_hnsw_data['ntotal']}, levels vector size is {ntotal}. Using levels vector size.",
+                    file=sys.stderr,
+                )
+                original_hnsw_data["ntotal"] = ntotal
 
             # --- Check for compact format flag ---
             print(f"[{time.time() - start_time:.2f}s]   Probing for compact storage flag...")
             pos_before_compact = f_in.tell()
             try:
-                is_compact_flag = read_struct(f_in, '<?')
+                is_compact_flag = read_struct(f_in, "<?")
                 print(f"[{time.time() - start_time:.2f}s]   Found compact flag: {is_compact_flag}")
 
                 if is_compact_flag:
                     # Input is already in compact format - read compact data
-                    print(f"[{time.time() - start_time:.2f}s]   Input is already in compact format, reading compact data...")
+                    print(
+                        f"[{time.time() - start_time:.2f}s]   Input is already in compact format, reading compact data..."
+                    )
 
-                    compact_level_ptr = read_numpy_vector(f_in, np.uint64, 'Q')
-                    print(f"[{time.time() - start_time:.2f}s]   Read compact_level_ptr ({compact_level_ptr.size})")
+                    compact_level_ptr = read_numpy_vector(f_in, np.uint64, "Q")
+                    print(
+                        f"[{time.time() - start_time:.2f}s]   Read compact_level_ptr ({compact_level_ptr.size})"
+                    )
 
-                    compact_node_offsets_np = read_numpy_vector(f_in, np.uint64, 'Q')
-                    print(f"[{time.time() - start_time:.2f}s]   Read compact_node_offsets ({compact_node_offsets_np.size})")
+                    compact_node_offsets_np = read_numpy_vector(f_in, np.uint64, "Q")
+                    print(
+                        f"[{time.time() - start_time:.2f}s]   Read compact_node_offsets ({compact_node_offsets_np.size})"
+                    )
 
                     # Read scalar parameters
-                    original_hnsw_data['entry_point'] = read_struct(f_in, '<i')
-                    original_hnsw_data['max_level'] = read_struct(f_in, '<i')
-                    original_hnsw_data['efConstruction'] = read_struct(f_in, '<i')
-                    original_hnsw_data['efSearch'] = read_struct(f_in, '<i')
-                    original_hnsw_data['dummy_upper_beam'] = read_struct(f_in, '<i')
-                    print(f"[{time.time() - start_time:.2f}s]   Read scalar params (ep={original_hnsw_data['entry_point']}, max_lvl={original_hnsw_data['max_level']})")
+                    original_hnsw_data["entry_point"] = read_struct(f_in, "<i")
+                    original_hnsw_data["max_level"] = read_struct(f_in, "<i")
+                    original_hnsw_data["efConstruction"] = read_struct(f_in, "<i")
+                    original_hnsw_data["efSearch"] = read_struct(f_in, "<i")
+                    original_hnsw_data["dummy_upper_beam"] = read_struct(f_in, "<i")
+                    print(
+                        f"[{time.time() - start_time:.2f}s]   Read scalar params (ep={original_hnsw_data['entry_point']}, max_lvl={original_hnsw_data['max_level']})"
+                    )
 
                     # Read storage fourcc
-                    storage_fourcc = read_struct(f_in, '<I')
-                    print(f"[{time.time() - start_time:.2f}s]   Found storage fourcc: {storage_fourcc:08x}")
+                    storage_fourcc = read_struct(f_in, "<I")
+                    print(
+                        f"[{time.time() - start_time:.2f}s]   Found storage fourcc: {storage_fourcc:08x}"
+                    )
 
                     if prune_embeddings and storage_fourcc != NULL_INDEX_FOURCC:
                         # Read compact neighbors data
-                        compact_neighbors_data_np = read_numpy_vector(f_in, np.int32, 'i')
-                        print(f"[{time.time() - start_time:.2f}s]   Read compact neighbors data ({compact_neighbors_data_np.size})")
+                        compact_neighbors_data_np = read_numpy_vector(f_in, np.int32, "i")
+                        print(
+                            f"[{time.time() - start_time:.2f}s]   Read compact neighbors data ({compact_neighbors_data_np.size})"
+                        )
                         compact_neighbors_data = compact_neighbors_data_np.tolist()
                         del compact_neighbors_data_np
 
                         # Skip storage data and write with NULL marker
-                        print(f"[{time.time() - start_time:.2f}s]   Pruning embeddings: Writing NULL storage marker.")
+                        print(
+                            f"[{time.time() - start_time:.2f}s]   Pruning embeddings: Writing NULL storage marker."
+                        )
                         storage_fourcc = NULL_INDEX_FOURCC
                     elif not prune_embeddings:
                         # Read and preserve compact neighbors and storage
-                        compact_neighbors_data_np = read_numpy_vector(f_in, np.int32, 'i')
+                        compact_neighbors_data_np = read_numpy_vector(f_in, np.int32, "i")
                         compact_neighbors_data = compact_neighbors_data_np.tolist()
                         del compact_neighbors_data_np
 
@@ -288,16 +371,25 @@ def convert_hnsw_graph_to_csr(input_filename, output_filename, prune_embeddings=
                         storage_data = f_in.read()
                     else:
                         # Already pruned (NULL storage)
-                        compact_neighbors_data_np = read_numpy_vector(f_in, np.int32, 'i')
+                        compact_neighbors_data_np = read_numpy_vector(f_in, np.int32, "i")
                         compact_neighbors_data = compact_neighbors_data_np.tolist()
                         del compact_neighbors_data_np
-                        storage_data = b''
+                        storage_data = b""
 
                     # Write the updated compact format
                     print(f"[{time.time() - start_time:.2f}s] Writing updated compact format...")
-                    write_compact_format(f_out, original_hnsw_data, assign_probas_np, cum_nneighbor_per_level_np, 
-                                       levels_np, compact_level_ptr, compact_node_offsets_np, 
-                                       compact_neighbors_data, storage_fourcc, storage_data if not prune_embeddings else b'')
+                    write_compact_format(
+                        f_out,
+                        original_hnsw_data,
+                        assign_probas_np,
+                        cum_nneighbor_per_level_np,
+                        levels_np,
+                        compact_level_ptr,
+                        compact_node_offsets_np,
+                        compact_neighbors_data,
+                        storage_fourcc,
+                        storage_data if not prune_embeddings else b"",
+                    )
 
                     print(f"[{time.time() - start_time:.2f}s] Conversion complete.")
                     return True
@@ -305,63 +397,86 @@ def convert_hnsw_graph_to_csr(input_filename, output_filename, prune_embeddings=
                 else:
                     # is_compact=False, rewind and read original format
                     f_in.seek(pos_before_compact)
-                    print(f"[{time.time() - start_time:.2f}s]   Compact flag is False, reading original format...")
+                    print(
+                        f"[{time.time() - start_time:.2f}s]   Compact flag is False, reading original format..."
+                    )
 
             except EOFError:
                 # No compact flag found, assume original format
                 f_in.seek(pos_before_compact)
-                print(f"[{time.time() - start_time:.2f}s]   No compact flag found, assuming original format...")
+                print(
+                    f"[{time.time() - start_time:.2f}s]   No compact flag found, assuming original format..."
+                )
 
             # --- Handle potential extra byte in original format (like C++ code) ---
-            print(f"[{time.time() - start_time:.2f}s]   Probing for potential extra byte before non-compact offsets...")
+            print(
+                f"[{time.time() - start_time:.2f}s]   Probing for potential extra byte before non-compact offsets..."
+            )
             pos_before_probe = f_in.tell()
             try:
-                suspected_flag = read_struct(f_in, '<B')  # Read 1 byte
+                suspected_flag = read_struct(f_in, "<B")  # Read 1 byte
                 if suspected_flag == 0x00:
-                    print(f"[{time.time() - start_time:.2f}s]   Found and consumed an unexpected 0x00 byte.")
+                    print(
+                        f"[{time.time() - start_time:.2f}s]   Found and consumed an unexpected 0x00 byte."
+                    )
                 elif suspected_flag == 0x01:
-                    print(f"[{time.time() - start_time:.2f}s]   ERROR: Found 0x01 but is_compact should be False")
+                    print(
+                        f"[{time.time() - start_time:.2f}s]   ERROR: Found 0x01 but is_compact should be False"
+                    )
                     raise ValueError("Inconsistent compact flag state")
                 else:
                     # Rewind - this byte is part of offsets data
                     f_in.seek(pos_before_probe)
-                    print(f"[{time.time() - start_time:.2f}s]   Rewound to original position (byte was 0x{suspected_flag:02x})")
+                    print(
+                        f"[{time.time() - start_time:.2f}s]   Rewound to original position (byte was 0x{suspected_flag:02x})"
+                    )
             except EOFError:
                 f_in.seek(pos_before_probe)
-                print(f"[{time.time() - start_time:.2f}s]   No extra byte found (EOF), proceeding with offsets read")
+                print(
+                    f"[{time.time() - start_time:.2f}s]   No extra byte found (EOF), proceeding with offsets read"
+                )
 
             # --- Read original format data ---
-            offsets_np = read_numpy_vector(f_in, np.uint64, 'Q')
+            offsets_np = read_numpy_vector(f_in, np.uint64, "Q")
             print(f"[{time.time() - start_time:.2f}s]   Read offsets ({offsets_np.size})")
             if len(offsets_np) != ntotal + 1:
-                 raise ValueError(f"Inconsistent offsets size: len(levels)={ntotal} but len(offsets)={len(offsets_np)}")
+                raise ValueError(
+                    f"Inconsistent offsets size: len(levels)={ntotal} but len(offsets)={len(offsets_np)}"
+                )
             gc.collect()
 
             print(f"[{time.time() - start_time:.2f}s]   Attempting to read neighbors vector...")
-            neighbors_np = read_numpy_vector(f_in, np.int32, 'i')
+            neighbors_np = read_numpy_vector(f_in, np.int32, "i")
             print(f"[{time.time() - start_time:.2f}s]   Read neighbors ({neighbors_np.size})")
             expected_neighbors_size = offsets_np[-1] if ntotal > 0 else 0
             if neighbors_np.size != expected_neighbors_size:
-                 print(f"Warning: neighbors vector size mismatch. Expected {expected_neighbors_size} based on offsets, got {neighbors_np.size}.")
+                print(
+                    f"Warning: neighbors vector size mismatch. Expected {expected_neighbors_size} based on offsets, got {neighbors_np.size}."
+                )
             gc.collect()
 
-            original_hnsw_data['entry_point'] = read_struct(f_in, '<i')
-            original_hnsw_data['max_level'] = read_struct(f_in, '<i')
-            original_hnsw_data['efConstruction'] = read_struct(f_in, '<i')
-            original_hnsw_data['efSearch'] = read_struct(f_in, '<i')
-            original_hnsw_data['dummy_upper_beam'] = read_struct(f_in, '<i')
-            print(f"[{time.time() - start_time:.2f}s]   Read scalar params (ep={original_hnsw_data['entry_point']}, max_lvl={original_hnsw_data['max_level']})")
+            original_hnsw_data["entry_point"] = read_struct(f_in, "<i")
+            original_hnsw_data["max_level"] = read_struct(f_in, "<i")
+            original_hnsw_data["efConstruction"] = read_struct(f_in, "<i")
+            original_hnsw_data["efSearch"] = read_struct(f_in, "<i")
+            original_hnsw_data["dummy_upper_beam"] = read_struct(f_in, "<i")
+            print(
+                f"[{time.time() - start_time:.2f}s]   Read scalar params (ep={original_hnsw_data['entry_point']}, max_lvl={original_hnsw_data['max_level']})"
+            )
 
             print(f"[{time.time() - start_time:.2f}s] Checking for storage data...")
             storage_fourcc = None
             try:
-                storage_fourcc = read_struct(f_in, '<I')
-                print(f"[{time.time() - start_time:.2f}s]   Found storage fourcc: {storage_fourcc:08x}.")
+                storage_fourcc = read_struct(f_in, "<I")
+                print(
+                    f"[{time.time() - start_time:.2f}s]   Found storage fourcc: {storage_fourcc:08x}."
+                )
             except EOFError:
                 print(f"[{time.time() - start_time:.2f}s]   No storage data found (EOF).")
             except Exception as e:
-                 print(f"[{time.time() - start_time:.2f}s]   Error reading potential storage data: {e}")
-
+                print(
+                    f"[{time.time() - start_time:.2f}s]   Error reading potential storage data: {e}"
+                )
 
             # --- Perform Conversion ---
             print(f"[{time.time() - start_time:.2f}s] Converting to CSR format...")
@@ -380,10 +495,14 @@ def convert_hnsw_graph_to_csr(input_filename, output_filename, prune_embeddings=
                 if i > 0 and i % (ntotal // 100 or 1) == 0:  # Log progress roughly every 1%
                     progress = (i / ntotal) * 100
                     elapsed = time.time() - start_time
-                    print(f"\r[{elapsed:.2f}s]   Converting node {i}/{ntotal} ({progress:.1f}%)...", end="")
+                    print(
+                        f"\r[{elapsed:.2f}s]   Converting node {i}/{ntotal} ({progress:.1f}%)...",
+                        end="",
+                    )
 
                 node_max_level = levels_np[i] - 1
-                if node_max_level < -1: node_max_level = -1
+                if node_max_level < -1:
+                    node_max_level = -1
 
                 node_ptr_start_index = current_level_ptr_idx
                 compact_node_offsets_np[i] = node_ptr_start_index
@@ -394,8 +513,12 @@ def convert_hnsw_graph_to_csr(input_filename, output_filename, prune_embeddings=
                 for level in range(node_max_level + 1):
                     compact_level_ptr.append(current_data_idx)
 
-                    begin_orig_np = original_offset_start + get_cum_neighbors(cum_nneighbor_per_level_np, level)
-                    end_orig_np = original_offset_start + get_cum_neighbors(cum_nneighbor_per_level_np, level + 1)
+                    begin_orig_np = original_offset_start + get_cum_neighbors(
+                        cum_nneighbor_per_level_np, level
+                    )
+                    end_orig_np = original_offset_start + get_cum_neighbors(
+                        cum_nneighbor_per_level_np, level + 1
+                    )
 
                     begin_orig = int(begin_orig_np)
                     end_orig = int(end_orig_np)
@@ -413,71 +536,116 @@ def convert_hnsw_graph_to_csr(input_filename, output_filename, prune_embeddings=
 
                         if num_valid > 0:
                             # Append valid neighbors
-                            compact_neighbors_data.extend(level_neighbors_slice[valid_neighbors_mask])
+                            compact_neighbors_data.extend(
+                                level_neighbors_slice[valid_neighbors_mask]
+                            )
                             current_data_idx += num_valid
                             total_valid_neighbors_counted += num_valid
 
-
                 compact_level_ptr.append(current_data_idx)
                 current_level_ptr_idx += num_pointers_expected
 
             compact_node_offsets_np[ntotal] = current_level_ptr_idx
-            print(f"\r[{time.time() - start_time:.2f}s]   Conversion loop finished.                        ") # Clear progress line
+            print(
+                f"\r[{time.time() - start_time:.2f}s]   Conversion loop finished.                        "
+            )  # Clear progress line
 
             # --- Validation Checks ---
             print(f"[{time.time() - start_time:.2f}s] Running validation checks...")
             valid_check_passed = True
             # Check 1: Total valid neighbors count
-            print(f"    Checking total valid neighbor count...")
+            print("    Checking total valid neighbor count...")
             expected_valid_count = np.sum(neighbors_np >= 0)
             if total_valid_neighbors_counted != len(compact_neighbors_data):
-                 print(f"Error: Mismatch between counted valid neighbors ({total_valid_neighbors_counted}) and final compact_data size ({len(compact_neighbors_data)})!", file=sys.stderr)
+                print(
+                    f"Error: Mismatch between counted valid neighbors ({total_valid_neighbors_counted}) and final compact_data size ({len(compact_neighbors_data)})!",
+                    file=sys.stderr,
+                )
                 valid_check_passed = False
             if expected_valid_count != len(compact_neighbors_data):
-                 print(f"Error: Mismatch between NumPy count of valid neighbors ({expected_valid_count}) and final compact_data size ({len(compact_neighbors_data)})!", file=sys.stderr)
+                print(
+                    f"Error: Mismatch between NumPy count of valid neighbors ({expected_valid_count}) and final compact_data size ({len(compact_neighbors_data)})!",
+                    file=sys.stderr,
+                )
                 valid_check_passed = False
             else:
                 print(f"    OK: Total valid neighbors = {len(compact_neighbors_data)}")
 
             # Check 2: Final pointer indices consistency
-            print(f"    Checking final pointer indices...")
+            print("    Checking final pointer indices...")
             if compact_node_offsets_np[ntotal] != len(compact_level_ptr):
-                 print(f"Error: Final node offset ({compact_node_offsets_np[ntotal]}) doesn't match level_ptr size ({len(compact_level_ptr)})!", file=sys.stderr)
+                print(
+                    f"Error: Final node offset ({compact_node_offsets_np[ntotal]}) doesn't match level_ptr size ({len(compact_level_ptr)})!",
+                    file=sys.stderr,
+                )
                 valid_check_passed = False
-            if (len(compact_level_ptr) > 0 and compact_level_ptr[-1] != len(compact_neighbors_data)) or \
-               (len(compact_level_ptr) == 0 and len(compact_neighbors_data) != 0):
+            if (
+                len(compact_level_ptr) > 0 and compact_level_ptr[-1] != len(compact_neighbors_data)
+            ) or (len(compact_level_ptr) == 0 and len(compact_neighbors_data) != 0):
                 last_ptr = compact_level_ptr[-1] if len(compact_level_ptr) > 0 else -1
-                 print(f"Error: Last level pointer ({last_ptr}) doesn't match compact_data size ({len(compact_neighbors_data)})!", file=sys.stderr)
+                print(
+                    f"Error: Last level pointer ({last_ptr}) doesn't match compact_data size ({len(compact_neighbors_data)})!",
+                    file=sys.stderr,
+                )
                 valid_check_passed = False
             else:
-                 print(f"    OK: Final pointers match data size.")
+                print("    OK: Final pointers match data size.")
 
             if not valid_check_passed:
-                print("Error: Validation checks failed. Output file might be incorrect.", file=sys.stderr)
+                print(
+                    "Error: Validation checks failed. Output file might be incorrect.",
+                    file=sys.stderr,
+                )
                 # Optional: Exit here if validation fails
                 # return False
 
             # --- Explicitly delete large intermediate arrays ---
-            print(f"[{time.time() - start_time:.2f}s] Deleting original neighbors and offsets arrays...")
+            print(
+                f"[{time.time() - start_time:.2f}s] Deleting original neighbors and offsets arrays..."
+            )
             del neighbors_np
             del offsets_np
             gc.collect()
 
-            print(f"    CSR Stats: |data|={len(compact_neighbors_data)}, |level_ptr|={len(compact_level_ptr)}")
+            print(
+                f"    CSR Stats: |data|={len(compact_neighbors_data)}, |level_ptr|={len(compact_level_ptr)}"
+            )
 
             # --- Write CSR HNSW graph data using unified function ---
-            print(f"[{time.time() - start_time:.2f}s] Writing CSR HNSW graph data in FAISS-compatible order...")
+            print(
+                f"[{time.time() - start_time:.2f}s] Writing CSR HNSW graph data in FAISS-compatible order..."
+            )
 
-            # Determine storage fourcc based on prune_embeddings
-            output_storage_fourcc = NULL_INDEX_FOURCC if prune_embeddings else (storage_fourcc if 'storage_fourcc' in locals() else NULL_INDEX_FOURCC)
+            # Determine storage fourcc and data based on prune_embeddings
             if prune_embeddings:
-                print(f"   Pruning embeddings: Writing NULL storage marker.")
-            storage_data = b''
+                print("   Pruning embeddings: Writing NULL storage marker.")
+                output_storage_fourcc = NULL_INDEX_FOURCC
+                storage_data = b""
+            else:
+                # Keep embeddings - read and preserve original storage data
+                if storage_fourcc and storage_fourcc != NULL_INDEX_FOURCC:
+                    print("   Preserving embeddings: Reading original storage data...")
+                    storage_data = f_in.read()  # Read remaining storage data
+                    output_storage_fourcc = storage_fourcc
+                    print(f"   Read {len(storage_data)} bytes of storage data")
+                else:
+                    print("   No embeddings found in original file (NULL storage)")
+                    output_storage_fourcc = NULL_INDEX_FOURCC
+                    storage_data = b""
 
             # Use the unified write function
-            write_compact_format(f_out, original_hnsw_data, assign_probas_np, cum_nneighbor_per_level_np, 
-                               levels_np, compact_level_ptr, compact_node_offsets_np, 
-                               compact_neighbors_data, output_storage_fourcc, storage_data if not prune_embeddings else b'')
+            write_compact_format(
+                f_out,
+                original_hnsw_data,
+                assign_probas_np,
+                cum_nneighbor_per_level_np,
+                levels_np,
+                compact_level_ptr,
+                compact_node_offsets_np,
+                compact_neighbors_data,
+                output_storage_fourcc,
+                storage_data,
+            )
 
             # Clean up memory
             del assign_probas_np, cum_nneighbor_per_level_np, levels_np
@@ -492,40 +660,66 @@ def convert_hnsw_graph_to_csr(input_filename, output_filename, prune_embeddings=
         print(f"Error: Input file not found: {input_filename}", file=sys.stderr)
         return False
     except MemoryError as e:
-         print(f"\nFatal MemoryError during conversion: {e}. Insufficient RAM.", file=sys.stderr)
+        print(
+            f"\nFatal MemoryError during conversion: {e}. Insufficient RAM.",
+            file=sys.stderr,
+        )
         # Clean up potentially partially written output file?
-         try: os.remove(output_filename)
-         except OSError: pass
+        try:
+            os.remove(output_filename)
+        except OSError:
+            pass
         return False
     except EOFError as e:
-        print(f"Error: Reached end of file unexpectedly reading {input_filename}. {e}", file=sys.stderr)
-        try: os.remove(output_filename)
-        except OSError: pass
+        print(
+            f"Error: Reached end of file unexpectedly reading {input_filename}. {e}",
+            file=sys.stderr,
+        )
+        try:
+            os.remove(output_filename)
+        except OSError:
+            pass
         return False
     except Exception as e:
         print(f"An unexpected error occurred during conversion: {e}", file=sys.stderr)
         import traceback
+
         traceback.print_exc()
         try:
             os.remove(output_filename)
-        except OSError: pass
+        except OSError:
+            pass
         return False
     # Ensure neighbors_np is deleted even if an error occurs after its allocation
     finally:
-        if 'neighbors_np' in locals() and neighbors_np is not None:
+        try:
+            if "neighbors_np" in locals() and neighbors_np is not None:
                 del neighbors_np
                 gc.collect()
+        except NameError:
+            pass
 
 
 # --- Script Execution ---
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Convert a Faiss IndexHNSWFlat file to a CSR-based HNSW graph file.")
+    parser = argparse.ArgumentParser(
+        description="Convert a Faiss IndexHNSWFlat file to a CSR-based HNSW graph file."
+    )
     parser.add_argument("input_index_file", help="Path to the input IndexHNSWFlat file")
-    parser.add_argument("output_csr_graph_file", help="Path to write the output CSR HNSW graph file")
-    parser.add_argument("--prune-embeddings", action="store_true", default=True, 
-                       help="Prune embedding storage (write NULL storage marker)")
-    parser.add_argument("--keep-embeddings", action="store_true", 
-                       help="Keep embedding storage (overrides --prune-embeddings)")
+    parser.add_argument(
+        "output_csr_graph_file", help="Path to write the output CSR HNSW graph file"
+    )
+    parser.add_argument(
+        "--prune-embeddings",
+        action="store_true",
+        default=True,
+        help="Prune embedding storage (write NULL storage marker)",
+    )
+    parser.add_argument(
+        "--keep-embeddings",
+        action="store_true",
+        help="Keep embedding storage (overrides --prune-embeddings)",
+    )
 
     args = parser.parse_args()
 
@@ -534,10 +728,12 @@ if __name__ == "__main__":
         sys.exit(1)
 
     if os.path.abspath(args.input_index_file) == os.path.abspath(args.output_csr_graph_file):
-         print(f"Error: Input and output filenames cannot be the same.", file=sys.stderr)
+        print("Error: Input and output filenames cannot be the same.", file=sys.stderr)
         sys.exit(1)
 
     prune_embeddings = args.prune_embeddings and not args.keep_embeddings
-    success = convert_hnsw_graph_to_csr(args.input_index_file, args.output_csr_graph_file, prune_embeddings)
+    success = convert_hnsw_graph_to_csr(
+        args.input_index_file, args.output_csr_graph_file, prune_embeddings
+    )
     if not success:
         sys.exit(1)

packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_backend.py

@@ -1,145 +1,39 @@
-import numpy as np
+import logging
 import os
-import json
-import struct
-from pathlib import Path
-from typing import Dict, Any
-import contextlib
-import threading
+import shutil
 import time
-import atexit
-import socket
-import subprocess
-import sys
+from pathlib import Path
+from typing import Any, Literal, Optional
+
+import numpy as np
+from leann.interface import (
+    LeannBackendBuilderInterface,
+    LeannBackendFactoryInterface,
+    LeannBackendSearcherInterface,
+)
+from leann.registry import register_backend
+from leann.searcher_base import BaseSearcher
 
 from .convert_to_csr import convert_hnsw_graph_to_csr
 
-from leann.registry import register_backend
-from leann.interface import (
-    LeannBackendFactoryInterface,
-    LeannBackendBuilderInterface,
-    LeannBackendSearcherInterface
-)
+logger = logging.getLogger(__name__)
+
 
 def get_metric_map():
-    from . import faiss
+    from . import faiss  # type: ignore
+
     return {
         "mips": faiss.METRIC_INNER_PRODUCT,
         "l2": faiss.METRIC_L2,
         "cosine": faiss.METRIC_INNER_PRODUCT,
     }
 
-def _check_port(port: int) -> bool:
-    """Check if a port is in use"""
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-        return s.connect_ex(('localhost', port)) == 0
 
-class HNSWEmbeddingServerManager:
-    """
-    HNSW-specific embedding server manager that handles the lifecycle of the embedding server process.
-    Mirrors the DiskANN EmbeddingServerManager architecture.
-    """
-    def __init__(self):
-        self.server_process = None
-        self.server_port = None
-        atexit.register(self.stop_server)
+def normalize_l2(data: np.ndarray) -> np.ndarray:
+    norms = np.linalg.norm(data, axis=1, keepdims=True)
+    norms[norms == 0] = 1  # Avoid division by zero
+    return data / norms
 
-    def start_server(self, port=5556, model_name="sentence-transformers/all-mpnet-base-v2", passages_file=None, distance_metric="mips"):
-        """
-        Start the HNSW embedding server process.
-        
-        Args:
-            port: ZMQ port for the server
-            model_name: Name of the embedding model to use
-            passages_file: Optional path to passages JSON file
-            distance_metric: The distance metric to use
-        """
-        if self.server_process and self.server_process.poll() is None:
-            print(f"INFO: Reusing existing HNSW server process for this session (PID {self.server_process.pid})")
-            return True
-            
-        # Check if port is already in use
-        if _check_port(port):
-            print(f"WARNING: Port {port} is already in use. Assuming an external HNSW server is running and connecting to it.")
-            return True
-        
-        print(f"INFO: Starting session-level HNSW embedding server as a background process...")
-        
-        try:
-            command = [
-                sys.executable,
-                "-m", "leann_backend_hnsw.hnsw_embedding_server",
-                "--zmq-port", str(port), 
-                "--model-name", model_name,
-                "--distance-metric", distance_metric
-            ]
-            
-            if passages_file:
-                command.extend(["--passages-file", str(passages_file)])
-            
-            project_root = Path(__file__).parent.parent.parent.parent
-            print(f"INFO: Running HNSW command from project root: {project_root}")
-            
-            self.server_process = subprocess.Popen(
-                command,
-                cwd=project_root,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
-                text=True,
-                encoding='utf-8'
-            )
-            self.server_port = port
-            print(f"INFO: HNSW server process started with PID: {self.server_process.pid}")
-
-            max_wait, wait_interval = 30, 0.5
-            for _ in range(int(max_wait / wait_interval)):
-                if _check_port(port):
-                    print(f"✅ HNSW embedding server is up and ready for this session.")
-                    log_thread = threading.Thread(target=self._log_monitor, daemon=True)
-                    log_thread.start()
-                    return True
-                if self.server_process.poll() is not None:
-                    print("❌ ERROR: HNSW server process terminated unexpectedly during startup.")
-                    self._log_monitor()
-                    return False
-                time.sleep(wait_interval)
-            
-            print(f"❌ ERROR: HNSW server process failed to start listening within {max_wait} seconds.")
-            self.stop_server()
-            return False
-                
-        except Exception as e:
-            print(f"❌ ERROR: Failed to start HNSW embedding server process: {e}")
-            return False
-
-    def _log_monitor(self):
-        """Monitor server logs"""
-        if not self.server_process:
-            return
-        try:
-            if self.server_process.stdout:
-                for line in iter(self.server_process.stdout.readline, ''):
-                    print(f"[HNSWEmbeddingServer LOG]: {line.strip()}")
-                self.server_process.stdout.close()
-            if self.server_process.stderr:
-                for line in iter(self.server_process.stderr.readline, ''):
-                    print(f"[HNSWEmbeddingServer ERROR]: {line.strip()}")
-                self.server_process.stderr.close()
-        except Exception as e:
-            print(f"HNSW Log monitor error: {e}")
-
-    def stop_server(self):
-        """Stop the HNSW embedding server process"""
-        if self.server_process and self.server_process.poll() is None:
-            print(f"INFO: Terminating HNSW session server process (PID: {self.server_process.pid})...")
-            self.server_process.terminate()
-            try:
-                self.server_process.wait(timeout=5)
-                print("INFO: HNSW server process terminated.")
-            except subprocess.TimeoutExpired:
-                print("WARNING: HNSW server process did not terminate gracefully, killing it.")
-                self.server_process.kill()
-        self.server_process = None
 
 @register_backend("hnsw")
 class HNSWBackend(LeannBackendFactoryInterface):
@@ -149,372 +43,211 @@ class HNSWBackend(LeannBackendFactoryInterface):
 
     @staticmethod
     def searcher(index_path: str, **kwargs) -> LeannBackendSearcherInterface:
-        path = Path(index_path)
-        meta_path = path.parent / f"{path.name}.meta.json"
-        if not meta_path.exists():
-            raise FileNotFoundError(f"Leann metadata file not found at {meta_path}. Cannot infer vector dimension for searcher.")
-        
-        with open(meta_path, 'r') as f:
-            meta = json.load(f)
-
-        dimensions = meta.get("dimensions")
-        if not dimensions:
-            raise ValueError("Dimensions not found in Leann metadata. Please rebuild the index with a newer version of Leann.")
-        
-        kwargs['dimensions'] = dimensions
         return HNSWSearcher(index_path, **kwargs)
 
+
 class HNSWBuilder(LeannBackendBuilderInterface):
     def __init__(self, **kwargs):
         self.build_params = kwargs.copy()
-        
-        # --- Configuration defaults with standardized names ---
         self.is_compact = self.build_params.setdefault("is_compact", True)
         self.is_recompute = self.build_params.setdefault("is_recompute", True)
-        
-        # --- Additional Options ---
-        self.is_skip_neighbors = self.build_params.setdefault("is_skip_neighbors", False) 
-        self.disk_cache_ratio = self.build_params.setdefault("disk_cache_ratio", 0.0)
-        self.external_storage_path = self.build_params.get("external_storage_path", None)
-        
-        # --- Standard HNSW parameters ---
         self.M = self.build_params.setdefault("M", 32)
         self.efConstruction = self.build_params.setdefault("efConstruction", 200)
         self.distance_metric = self.build_params.setdefault("distance_metric", "mips")
         self.dimensions = self.build_params.get("dimensions")
+        if not self.is_recompute and self.is_compact:
+            # Auto-correct: non-recompute requires non-compact storage for HNSW
+            logger.warning(
+                "is_recompute=False requires non-compact HNSW. Forcing is_compact=False."
+            )
+            self.is_compact = False
+            self.build_params["is_compact"] = False
 
-        if self.is_skip_neighbors and not self.is_compact:
-            raise ValueError("is_skip_neighbors can only be used with is_compact=True")
-        
-        if self.is_recompute and not self.is_compact:
-            raise ValueError("is_recompute requires is_compact=True for efficiency")
-
-    def build(self, data: np.ndarray, index_path: str, **kwargs):
-        """Build HNSW index using FAISS"""
-        from . import faiss
+    def build(self, data: np.ndarray, ids: list[str], index_path: str, **kwargs):
+        from . import faiss  # type: ignore
 
         path = Path(index_path)
         index_dir = path.parent
         index_prefix = path.stem
-
         index_dir.mkdir(parents=True, exist_ok=True)
 
         if data.dtype != np.float32:
+            logger.warning(f"Converting data to float32, shape: {data.shape}")
             data = data.astype(np.float32)
-        if not data.flags['C_CONTIGUOUS']:
-            data = np.ascontiguousarray(data)
 
-        metric_str = self.distance_metric.lower()
-        metric_enum = get_metric_map().get(metric_str)
+        metric_enum = get_metric_map().get(self.distance_metric.lower())
         if metric_enum is None:
-            raise ValueError(f"Unsupported distance_metric '{metric_str}'.")
+            raise ValueError(f"Unsupported distance_metric '{self.distance_metric}'.")
 
-        M = self.M
-        efConstruction = self.efConstruction
-        dim = self.dimensions
-        if not dim:
-            dim = data.shape[1]
+        dim = self.dimensions or data.shape[1]
+        index = faiss.IndexHNSWFlat(dim, self.M, metric_enum)
+        index.hnsw.efConstruction = self.efConstruction
 
-        print(f"INFO: Building HNSW index for {data.shape[0]} vectors with metric {metric_enum}...")
-        
-        try:
-            index = faiss.IndexHNSWFlat(dim, M, metric_enum)
-            index.hnsw.efConstruction = efConstruction
-            
-            if metric_str == "cosine":
-                faiss.normalize_L2(data)
+        if self.distance_metric.lower() == "cosine":
+            data = normalize_l2(data)
 
         index.add(data.shape[0], faiss.swig_ptr(data))
-            
         index_file = index_dir / f"{index_prefix}.index"
         faiss.write_index(index, str(index_file))
 
-            print(f"✅ HNSW index built successfully at '{index_file}'")
-
         if self.is_compact:
             self._convert_to_csr(index_file)
 
-            if self.is_recompute:
-                self._generate_passages_file(index_dir, index_prefix, **kwargs)
-            
-        except Exception as e:
-            print(f"💥 ERROR: HNSW index build failed. Exception: {e}")
-            raise
-
     def _convert_to_csr(self, index_file: Path):
         """Convert built index to CSR format"""
-        try:
         mode_str = "CSR-pruned" if self.is_recompute else "CSR-standard"
-            print(f"INFO: Converting HNSW index to {mode_str} format...")
+        logger.info(f"INFO: Converting HNSW index to {mode_str} format...")
 
         csr_temp_file = index_file.with_suffix(".csr.tmp")
 
         success = convert_hnsw_graph_to_csr(
-                str(index_file), 
-                str(csr_temp_file),
-                prune_embeddings=self.is_recompute
+            str(index_file), str(csr_temp_file), prune_embeddings=self.is_recompute
         )
 
         if success:
-                print("✅ CSR conversion successful.")
-                import shutil
+            logger.info("✅ CSR conversion successful.")
+            # index_file_old = index_file.with_suffix(".old")
+            # shutil.move(str(index_file), str(index_file_old))
             shutil.move(str(csr_temp_file), str(index_file))
-                print(f"INFO: Replaced original index with {mode_str} version at '{index_file}'")
+            logger.info(f"INFO: Replaced original index with {mode_str} version at '{index_file}'")
         else:
             # Clean up and fail fast
             if csr_temp_file.exists():
                 os.remove(csr_temp_file)
             raise RuntimeError("CSR conversion failed - cannot proceed with compact format")
 
-        except Exception as e:
-            print(f"💥 ERROR: CSR conversion failed. Exception: {e}")
-            raise
-
-    def _generate_passages_file(self, index_dir: Path, index_prefix: str, **kwargs):
-        """Generate passages file for recompute mode"""
-        try:
-            chunks = kwargs.get('chunks', [])
-            if not chunks:
-                print("INFO: No chunks data provided, skipping passages file generation")
-                return
-            
-            # Generate node_id to text mapping
-            passages_data = {}
-            for node_id, chunk in enumerate(chunks):
-                passages_data[str(node_id)] = chunk["text"]
-            
-            # Save passages file
-            passages_file = index_dir / f"{index_prefix}.passages.json"
-            with open(passages_file, 'w', encoding='utf-8') as f:
-                json.dump(passages_data, f, ensure_ascii=False, indent=2)
-            
-            print(f"✅ Generated passages file for recompute mode at '{passages_file}' ({len(passages_data)} passages)")
-            
-        except Exception as e:
-            print(f"💥 ERROR: Failed to generate passages file. Exception: {e}")
-            # Don't raise - this is not critical for index building
-            pass
-
-class HNSWSearcher(LeannBackendSearcherInterface):
-    def _get_index_storage_status(self, index_file: Path) -> tuple[bool, bool]:
-        """
-        Robustly determines the index's storage status by parsing the file.
-        
-        Returns:
-            A tuple (is_compact, is_pruned).
-        """
-        if not index_file.exists():
-            return False, False
-        
-        with open(index_file, 'rb') as f:
-            try:
-                def read_struct(fmt):
-                    size = struct.calcsize(fmt)
-                    data = f.read(size)
-                    if len(data) != size:
-                        raise EOFError(f"File ended unexpectedly reading struct fmt '{fmt}'.")
-                    return struct.unpack(fmt, data)[0]
-
-                def skip_vector(element_size):
-                    count = read_struct('<Q')
-                    f.seek(count * element_size, 1)
-
-                # 1. Read up to the compact flag
-                read_struct('<I'); read_struct('<i'); read_struct('<q'); 
-                read_struct('<q'); read_struct('<q'); read_struct('<?')
-                metric_type = read_struct('<i')
-                if metric_type > 1: read_struct('<f')
-                skip_vector(8); skip_vector(4); skip_vector(4)
-                
-                # 2. Check if there's a compact flag byte
-                # Try to read the compact flag, but handle both old and new formats
-                pos_before_compact = f.tell()
-                try:
-                    is_compact = read_struct('<?')
-                    print(f"INFO: Detected is_compact flag as: {is_compact}")
-                except (EOFError, struct.error):
-                    # Old format without compact flag - assume non-compact
-                    f.seek(pos_before_compact)
-                    is_compact = False
-                    print(f"INFO: No compact flag found, assuming is_compact=False")
-
-                # 3. Read storage FourCC to determine if pruned
-                is_pruned = False
-                try:
-                    if is_compact:
-                        # For compact, we need to skip pointers and scalars to get to the storage FourCC
-                        skip_vector(8) # level_ptr
-                        skip_vector(8) # node_offsets
-                        read_struct('<i'); read_struct('<i'); read_struct('<i');
-                        read_struct('<i'); read_struct('<i')
-                        storage_fourcc = read_struct('<I')
-                    else:
-                        # For non-compact, we need to read the flag probe, then skip offsets and neighbors
-                        pos_before_probe = f.tell()
-                        flag_byte = f.read(1)
-                        if not (flag_byte and flag_byte == b'\x00'):
-                            f.seek(pos_before_probe)
-                        skip_vector(8); skip_vector(4) # offsets, neighbors
-                        read_struct('<i'); read_struct('<i'); read_struct('<i');
-                        read_struct('<i'); read_struct('<i')
-                        # Now we are at the storage. The entire rest is storage blob.
-                        storage_fourcc = struct.unpack('<I', f.read(4))[0]
-                        
-                    NULL_INDEX_FOURCC = int.from_bytes(b'null', 'little')
-                    if storage_fourcc == NULL_INDEX_FOURCC:
-                        is_pruned = True
-                except (EOFError, struct.error):
-                    # Cannot determine pruning status, assume not pruned
-                    pass
-                
-                print(f"INFO: Detected is_pruned as: {is_pruned}")
-                return is_compact, is_pruned
-
-            except (EOFError, struct.error) as e:
-                print(f"WARNING: Could not parse index file to detect format: {e}. Assuming standard, not pruned.")
-                return False, False
 
+class HNSWSearcher(BaseSearcher):
     def __init__(self, index_path: str, **kwargs):
-        from . import faiss
-        path = Path(index_path)
-        index_dir = path.parent
-        index_prefix = path.stem
+        super().__init__(
+            index_path,
+            backend_module_name="leann_backend_hnsw.hnsw_embedding_server",
+            **kwargs,
+        )
+        from . import faiss  # type: ignore
 
-        # Store configuration and paths for later use
-        self.config = kwargs.copy()
-        self.config["index_path"] = index_path
-        self.index_dir = index_dir
-        self.index_prefix = index_prefix
-        
-        metric_str = self.config.get("distance_metric", "mips").lower()
-        metric_enum = get_metric_map().get(metric_str)
+        self.distance_metric = (
+            self.meta.get("backend_kwargs", {}).get("distance_metric", "mips").lower()
+        )
+        metric_enum = get_metric_map().get(self.distance_metric)
         if metric_enum is None:
-            raise ValueError(f"Unsupported distance_metric '{metric_str}'.")
+            raise ValueError(f"Unsupported distance_metric '{self.distance_metric}'.")
 
-        dimensions = self.config.get("dimensions")
-        if not dimensions:
-            raise ValueError("Vector dimension not provided to HNSWSearcher.")
+        self.is_compact, self.is_pruned = (
+            self.meta.get("is_compact", True),
+            self.meta.get("is_pruned", True),
+        )
 
-        index_file = index_dir / f"{index_prefix}.index"
+        index_file = self.index_dir / f"{self.index_path.stem}.index"
         if not index_file.exists():
             raise FileNotFoundError(f"HNSW index file not found at {index_file}")
 
-        self.is_compact, self.is_pruned = self._get_index_storage_status(index_file)
-        
-        # Validate configuration constraints
-        if not self.is_compact and self.config.get("is_skip_neighbors", False):
-            raise ValueError("is_skip_neighbors can only be used with is_compact=True")
-        
-        if self.config.get("is_recompute", False) and self.config.get("external_storage_path"):
-            raise ValueError("Cannot use both is_recompute and external_storage_path simultaneously")
-            
         hnsw_config = faiss.HNSWIndexConfig()
         hnsw_config.is_compact = self.is_compact
-        
-        # Apply additional configuration options with strict validation
-        hnsw_config.is_skip_neighbors = self.config.get("is_skip_neighbors", False)
-        hnsw_config.is_recompute = self.is_pruned or self.config.get("is_recompute", False)
-        hnsw_config.disk_cache_ratio = self.config.get("disk_cache_ratio", 0.0)
-        hnsw_config.external_storage_path = self.config.get("external_storage_path")
-        hnsw_config.zmq_port = self.config.get("zmq_port", 5557)
-        
-        if self.is_pruned and not hnsw_config.is_recompute:
-            raise RuntimeError("Index is pruned (embeddings removed) but recompute is disabled. This is impossible - recompute must be enabled for pruned indices.")
-        
-        print(f"INFO: Loading index with is_compact={self.is_compact}, is_pruned={self.is_pruned}")
-        print(f"INFO: Config - skip_neighbors={hnsw_config.is_skip_neighbors}, recompute={hnsw_config.is_recompute}")
+        hnsw_config.is_recompute = (
+            self.is_pruned
+        )  # In C++ code, it's called is_recompute, but it's only for loading IIUC.
 
         self._index = faiss.read_index(str(index_file), faiss.IO_FLAG_MMAP, hnsw_config)
 
-        if self.is_compact:
-            print("✅ Compact CSR format HNSW index loaded successfully.")
-        else:
-            print("✅ Standard HNSW index loaded successfully.")
+    def search(
+        self,
+        query: np.ndarray,
+        top_k: int,
+        zmq_port: Optional[int] = None,
+        complexity: int = 64,
+        beam_width: int = 1,
+        prune_ratio: float = 0.0,
+        recompute_embeddings: bool = True,
+        pruning_strategy: Literal["global", "local", "proportional"] = "global",
+        batch_size: int = 0,
+        **kwargs,
+    ) -> dict[str, Any]:
+        """
+        Search for nearest neighbors using HNSW index.
 
-        self.metric_str = metric_str
-        self.embedding_server_manager = HNSWEmbeddingServerManager()
+        Args:
+            query: Query vectors (B, D) where B is batch size, D is dimension
+            top_k: Number of nearest neighbors to return
+            complexity: Search complexity/efSearch, higher = more accurate but slower
+            beam_width: Number of parallel search paths/beam_size
+            prune_ratio: Ratio of neighbors to prune via PQ (0.0-1.0)
+            recompute_embeddings: Whether to fetch fresh embeddings from server
+            pruning_strategy: PQ candidate selection strategy:
+                - "global": Use global PQ queue size for selection (default)
+                - "local": Local pruning, sort and select best candidates
+                - "proportional": Base selection on new neighbor count ratio
+            zmq_port: ZMQ port for embedding server communication. Must be provided if recompute_embeddings is True.
+            batch_size: Neighbor processing batch size, 0=disabled (HNSW-specific)
+            **kwargs: Additional HNSW-specific parameters (for legacy compatibility)
 
-    def _get_index_file(self, index_dir: Path, index_prefix: str) -> Path:
-        """Get the appropriate index file path based on format"""
-        # We always use the same filename now, format is detected internally
-        return index_dir / f"{index_prefix}.index"
+        Returns:
+            Dict with 'labels' (list of lists) and 'distances' (ndarray)
+        """
+        from . import faiss  # type: ignore
 
-    def search(self, query: np.ndarray, top_k: int, **kwargs) -> Dict[str, Any]:
-        """Search using HNSW index with optional recompute functionality"""
-        from . import faiss
-        # Merge config with search-time kwargs
-        search_config = self.config.copy()
-        search_config.update(kwargs)
-        
-        ef = search_config.get("ef", 200)  # Size of the dynamic candidate list for search
-        
-        # Recompute parameters
-        zmq_port = search_config.get("zmq_port", 5557)
-        embedding_model = search_config.get("embedding_model", "sentence-transformers/all-mpnet-base-v2")
-        passages_file = search_config.get("passages_file", None)
-        
-        # For recompute mode, try to find the passages file automatically
-        if self.is_pruned and not passages_file:
-            potential_passages_file = self.index_dir / f"{self.index_prefix}.passages.json"
-            print(f"DEBUG: Checking for passages file at: {potential_passages_file}")
-            if potential_passages_file.exists():
-                passages_file = str(potential_passages_file)
-                print(f"INFO: Found passages file for recompute mode: {passages_file}")
-            else:
-                print(f"WARNING: No passages file found for recompute mode at {potential_passages_file}")
-        
-        # If index is pruned (embeddings removed), we MUST start embedding server for recompute
-        if self.is_pruned:
-            print(f"INFO: Index is pruned - starting embedding server for recompute")
-            
-            # CRITICAL: Check passages file exists - fail fast if not
-            if not passages_file:
-                raise RuntimeError(f"FATAL: Index is pruned but no passages file found. Cannot proceed with recompute mode.")
-            
-            # Check if server is already running first
-            if _check_port(zmq_port):
-                print(f"INFO: Embedding server already running on port {zmq_port}")
-            else:
-                if not self.embedding_server_manager.start_server(zmq_port, embedding_model, passages_file, self.metric_str):
-                    raise RuntimeError(f"Failed to start HNSW embedding server on port {zmq_port}")
-                
-                # Give server extra time to fully initialize
-                print(f"INFO: Waiting for embedding server to fully initialize...")
-                time.sleep(3)
-                
-                # Final verification
-                if not _check_port(zmq_port):
-                    raise RuntimeError(f"Embedding server failed to start listening on port {zmq_port}")
-        else:
-            print(f"INFO: Index has embeddings stored - no recompute needed")
+        if not recompute_embeddings and self.is_pruned:
+            raise RuntimeError(
+                "Recompute is required for pruned/compact HNSW index. "
+                "Re-run search with --recompute, or rebuild with --no-recompute and --no-compact."
+            )
+        if recompute_embeddings:
+            if zmq_port is None:
+                raise ValueError("zmq_port must be provided if recompute_embeddings is True")
 
         if query.dtype != np.float32:
             query = query.astype(np.float32)
-        if query.ndim == 1:
-            query = np.expand_dims(query, axis=0)
+        if self.distance_metric == "cosine":
+            query = normalize_l2(query)
 
-        # Normalize query if using cosine similarity
-        if self.metric_str == "cosine":
-            faiss.normalize_L2(query)
+        params = faiss.SearchParametersHNSW()
+        if zmq_port is not None:
+            params.zmq_port = zmq_port  # C++ code won't use this if recompute_embeddings is False
+        params.efSearch = complexity
+        params.beam_size = beam_width
 
-        try:
-            # Set search parameter
-            self._index.hnsw.efSearch = ef
+        # For OpenAI embeddings with cosine distance, disable relative distance check
+        # This prevents early termination when all scores are in a narrow range
+        embedding_model = self.meta.get("embedding_model", "").lower()
+        if self.distance_metric == "cosine" and any(
+            openai_model in embedding_model for openai_model in ["text-embedding", "openai"]
+        ):
+            params.check_relative_distance = False
+        else:
+            params.check_relative_distance = True
 
-            # Prepare output arrays for the older FAISS SWIG API
-            batch_size = query.shape[0]
-            distances = np.empty((batch_size, top_k), dtype=np.float32)
-            labels = np.empty((batch_size, top_k), dtype=np.int64)
+        # PQ pruning: direct mapping to HNSW's pq_pruning_ratio
+        params.pq_pruning_ratio = prune_ratio
 
-            # Use standard FAISS search - recompute is handled internally by FAISS
-            self._index.search(query.shape[0], faiss.swig_ptr(query), top_k, faiss.swig_ptr(distances), faiss.swig_ptr(labels))
+        # Map pruning_strategy to HNSW parameters
+        if pruning_strategy == "local":
+            params.local_prune = True
+            params.send_neigh_times_ratio = 0.0
+        elif pruning_strategy == "proportional":
+            params.local_prune = False
+            params.send_neigh_times_ratio = 1.0  # Any value > 1e-6 triggers proportional mode
+        else:  # "global"
+            params.local_prune = False
+            params.send_neigh_times_ratio = 0.0
 
-            return {"labels": labels, "distances": distances}
+        # HNSW-specific batch processing parameter
+        params.batch_size = batch_size
 
-        except Exception as e:
-            print(f"💥 ERROR: HNSW search failed. Exception: {e}")
-            raise
+        batch_size_query = query.shape[0]
+        distances = np.empty((batch_size_query, top_k), dtype=np.float32)
+        labels = np.empty((batch_size_query, top_k), dtype=np.int64)
 
-    def __del__(self):
-        if hasattr(self, 'embedding_server_manager'):
-            self.embedding_server_manager.stop_server()
+        search_time = time.time()
+        self._index.search(
+            query.shape[0],
+            faiss.swig_ptr(query),
+            top_k,
+            faiss.swig_ptr(distances),
+            faiss.swig_ptr(labels),
+            params,
+        )
+        search_time = time.time() - search_time
+        logger.info(f"  Search time in HNSWSearcher.search() backend: {search_time} seconds")
+        string_labels = [[str(int_label) for int_label in batch_labels] for batch_labels in labels]
+
+        return {"labels": string_labels, "distances": distances}

packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_embedding_server.py

@@ -1,596 +1,428 @@
-#!/usr/bin/env python3
 """
-HNSW-specific embedding server with removed config.py dependencies
-Based on DiskANN embedding server architecture
+HNSW-specific embedding server
 """
 
-import pickle
 import argparse
+import json
+import logging
+import os
+import sys
 import threading
 import time
-from transformers import AutoTokenizer, AutoModel
-import os
-from contextlib import contextmanager
-import zmq
-import numpy as np
-import msgpack
-import json
 from pathlib import Path
-from typing import Dict, Any, Optional, Union
+from typing import Optional
 
-RED = "\033[91m"
-RESET = "\033[0m"
+import msgpack
+import numpy as np
+import zmq
 
-def is_similarity_metric():
-    """
-    Check if the metric type is similarity-based (like inner product).
-    0 = L2 (distance metric), 1 = Inner Product (similarity metric)
-    """
-    return True  # 1 is METRIC_INNER_PRODUCT in FAISS
+# Set up logging based on environment variable
+LOG_LEVEL = os.getenv("LEANN_LOG_LEVEL", "WARNING").upper()
+logger = logging.getLogger(__name__)
 
-# Function for E5-style average pooling
-import torch
-from torch import Tensor
-import torch.nn.functional as F
+# Force set logger level (don't rely on basicConfig in subprocess)
+log_level = getattr(logging, LOG_LEVEL, logging.WARNING)
+logger.setLevel(log_level)
 
-def e5_average_pool(last_hidden_states: Tensor, attention_mask: Tensor) -> Tensor:
-    last_hidden = last_hidden_states.masked_fill(~attention_mask[..., None].bool(), 0.0)
-    return last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None]
+# Ensure we have a handler if none exists
+if not logger.handlers:
+    handler = logging.StreamHandler()
+    formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+    logger.propagate = False
 
-class SimplePassageLoader:
-    """
-    Simple passage loader that replaces config.py dependencies
-    """
-    def __init__(self, passages_data: Optional[Dict[str, Any]] = None):
-        self.passages_data = passages_data or {}
-    
-    def __getitem__(self, passage_id: Union[str, int]) -> Dict[str, str]:
-        """Get passage by ID"""
-        str_id = str(passage_id)
-        if str_id in self.passages_data:
-            return {"text": self.passages_data[str_id]}
-        else:
-            # Return empty text for missing passages
-            return {"text": ""}
-    
-    def __len__(self) -> int:
-        return len(self.passages_data)
-
-def load_passages_from_file(passages_file: str) -> SimplePassageLoader:
-    """
-    Load passages from a JSON file
-    Expected format: {"passage_id": "passage_text", ...}
-    """
-    if not os.path.exists(passages_file):
-        print(f"Warning: Passages file {passages_file} not found. Using empty loader.")
-        return SimplePassageLoader()
-    
-    try:
-        with open(passages_file, 'r', encoding='utf-8') as f:
-            passages_data = json.load(f)
-        print(f"Loaded {len(passages_data)} passages from {passages_file}")
-        return SimplePassageLoader(passages_data)
-    except Exception as e:
-        print(f"Error loading passages from {passages_file}: {e}")
-        return SimplePassageLoader()
 
 def create_hnsw_embedding_server(
     passages_file: Optional[str] = None,
-    passages_data: Optional[Dict[str, str]] = None,
-    embeddings_file: Optional[str] = None,
-    use_fp16: bool = True,
-    use_int8: bool = False,
-    use_cuda_graphs: bool = False,
     zmq_port: int = 5555,
-    max_batch_size: int = 128,
     model_name: str = "sentence-transformers/all-mpnet-base-v2",
-    custom_max_length_param: Optional[int] = None,
     distance_metric: str = "mips",
+    embedding_mode: str = "sentence-transformers",
 ):
     """
     Create and start a ZMQ-based embedding server for HNSW backend.
-    
-    Args:
-        passages_file: Path to JSON file containing passage ID -> text mapping
-        passages_data: Direct passage data dict (alternative to passages_file)
-        embeddings_file: Path to pre-computed embeddings file (optional)
-        use_fp16: Whether to use FP16 precision
-        use_int8: Whether to use INT8 quantization
-        use_cuda_graphs: Whether to use CUDA graphs
-        zmq_port: ZMQ port to bind to
-        max_batch_size: Maximum batch size for processing
-        model_name: Transformer model name
-        custom_max_length_param: Custom max sequence length
-        distance_metric: The distance metric to use
+    Simplified version using unified embedding computation module.
     """
-    print(f"Loading tokenizer for {model_name}...")
-    tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)
-    print(f"Tokenizer loaded successfully!")
+    logger.info(f"Starting HNSW server on port {zmq_port} with model {model_name}")
+    logger.info(f"Using embedding mode: {embedding_mode}")
 
-    # Device setup
-    mps_available = hasattr(torch.backends, 'mps') and torch.backends.mps.is_available()
-    cuda_available = torch.cuda.is_available()
+    # Add leann-core to path for unified embedding computation
+    current_dir = Path(__file__).parent
+    leann_core_path = current_dir.parent.parent / "leann-core" / "src"
+    sys.path.insert(0, str(leann_core_path))
 
-    print(f"MPS available: {mps_available}")
-    print(f"CUDA available: {cuda_available}")
+    try:
+        from leann.api import PassageManager
+        from leann.embedding_compute import compute_embeddings
 
-    if cuda_available:
-        device = torch.device("cuda")
-        print("Using CUDA device")
-    elif mps_available:
-        device = torch.device("mps")
-        print("Using MPS device (Apple Silicon)")
-    else:
-        device = torch.device("cpu")
-        print("Using CPU device (no GPU acceleration available)")
-    
-    # Load model to the appropriate device
-    print(f"Starting HNSW server on port {zmq_port} with model {model_name}")
-    print(f"Loading model {model_name}... (this may take a while if downloading)")
-    model = AutoModel.from_pretrained(model_name).to(device).eval()
-    print(f"Model {model_name} loaded successfully!")
+        logger.info("Successfully imported unified embedding computation module")
+    except ImportError as e:
+        logger.error(f"Failed to import embedding computation module: {e}")
+        return
+    finally:
+        sys.path.pop(0)
 
     # Check port availability
     import socket
+
     def check_port(port):
         with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-            return s.connect_ex(('localhost', port)) == 0
+            return s.connect_ex(("localhost", port)) == 0
 
     if check_port(zmq_port):
-        print(f"{RED}Port {zmq_port} is already in use{RESET}")
+        logger.error(f"Port {zmq_port} is already in use")
         return
 
-    # Apply model optimizations (similar to DiskANN version)
-    if use_fp16 and (cuda_available or mps_available):
-        model = model.half()
-        model = torch.compile(model)
-        print(f"Using FP16 precision with model: {model_name}")
-    elif use_int8:
-        print("- Using TorchAO for Int8 dynamic activation and Int8 weight quantization")
-        from torchao.quantization import quantize_, Int8DynamicActivationInt8WeightConfig
-        quantize_(model, Int8DynamicActivationInt8WeightConfig())
-        model = torch.compile(model)
-        model.eval()
-        print("- Model successfully quantized and compiled")
+    # Only support metadata file, fail fast for everything else
+    if not passages_file or not passages_file.endswith(".meta.json"):
+        raise ValueError("Only metadata files (.meta.json) are supported")
 
-    # Load passages
-    if passages_data:
-        passages = SimplePassageLoader(passages_data)
-        print(f"Using provided passages data: {len(passages)} passages")
-    elif passages_file:
-        passages = load_passages_from_file(passages_file)
-    else:
-        passages = SimplePassageLoader()
-        print("No passages provided, using empty loader")
+    # Load metadata to get passage sources
+    with open(passages_file) as f:
+        meta = json.load(f)
 
-    # Load embeddings if provided
-    _embeddings = None
-    if embeddings_file and os.path.exists(embeddings_file):
+    # Let PassageManager handle path resolution uniformly. It supports fallback order:
+    # 1) path/index_path; 2) *_relative; 3) standard siblings next to meta
+    passages = PassageManager(meta["passage_sources"], metadata_file_path=passages_file)
+    # Dimension from metadata for shaping responses
     try:
-            with open(embeddings_file, "rb") as f:
-                _embeddings = pickle.load(f)
-            print(f"Loaded embeddings from {embeddings_file}")
+        embedding_dim: int = int(meta.get("dimensions", 0))
+    except Exception:
+        embedding_dim = 0
+    logger.info(f"Loaded PassageManager with {len(passages)} passages from metadata")
+
+    # (legacy ZMQ thread removed; using shutdown-capable server only)
+
+    def zmq_server_thread_with_shutdown(shutdown_event):
+        """ZMQ server thread that respects shutdown signal.
+
+        Creates its own REP socket bound to zmq_port and polls with timeouts
+        to allow graceful shutdown.
+        """
+        logger.info("ZMQ server thread started with shutdown support")
+
+        context = zmq.Context()
+        rep_socket = context.socket(zmq.REP)
+        rep_socket.bind(f"tcp://*:{zmq_port}")
+        logger.info(f"HNSW ZMQ REP server listening on port {zmq_port}")
+        rep_socket.setsockopt(zmq.RCVTIMEO, 1000)
+        # Keep sends from blocking during shutdown; fail fast and drop on close
+        rep_socket.setsockopt(zmq.SNDTIMEO, 1000)
+        rep_socket.setsockopt(zmq.LINGER, 0)
+
+        # Track last request type/length for shape-correct fallbacks
+        last_request_type = "unknown"  # 'text' | 'distance' | 'embedding' | 'unknown'
+        last_request_length = 0
+
+        try:
+            while not shutdown_event.is_set():
+                try:
+                    e2e_start = time.time()
+                    logger.debug("🔍 Waiting for ZMQ message...")
+                    request_bytes = rep_socket.recv()
+
+                    # Rest of the processing logic (same as original)
+                    request = msgpack.unpackb(request_bytes)
+
+                    if len(request) == 1 and request[0] == "__QUERY_MODEL__":
+                        response_bytes = msgpack.packb([model_name])
+                        rep_socket.send(response_bytes)
+                        continue
+
+                    # Handle direct text embedding request
+                    if (
+                        isinstance(request, list)
+                        and request
+                        and all(isinstance(item, str) for item in request)
+                    ):
+                        last_request_type = "text"
+                        last_request_length = len(request)
+                        embeddings = compute_embeddings(request, model_name, mode=embedding_mode)
+                        rep_socket.send(msgpack.packb(embeddings.tolist()))
+                        e2e_end = time.time()
+                        logger.info(f"⏱️  Text embedding E2E time: {e2e_end - e2e_start:.6f}s")
+                        continue
+
+                    # Handle distance calculation request: [[ids], [query_vector]]
+                    if (
+                        isinstance(request, list)
+                        and len(request) == 2
+                        and isinstance(request[0], list)
+                        and isinstance(request[1], list)
+                    ):
+                        node_ids = request[0]
+                        # Handle nested [[ids]] shape defensively
+                        if len(node_ids) == 1 and isinstance(node_ids[0], list):
+                            node_ids = node_ids[0]
+                        query_vector = np.array(request[1], dtype=np.float32)
+                        last_request_type = "distance"
+                        last_request_length = len(node_ids)
+
+                        logger.debug("Distance calculation request received")
+                        logger.debug(f"    Node IDs: {node_ids}")
+                        logger.debug(f"    Query vector dim: {len(query_vector)}")
+
+                        # Gather texts for found ids
+                        texts: list[str] = []
+                        found_indices: list[int] = []
+                        for idx, nid in enumerate(node_ids):
+                            try:
+                                passage_data = passages.get_passage(str(nid))
+                                txt = passage_data.get("text", "")
+                                if isinstance(txt, str) and len(txt) > 0:
+                                    texts.append(txt)
+                                    found_indices.append(idx)
+                                else:
+                                    logger.error(f"Empty text for passage ID {nid}")
+                            except KeyError:
+                                logger.error(f"Passage ID {nid} not found")
                             except Exception as e:
-            print(f"Error loading embeddings: {e}")
+                                logger.error(f"Exception looking up passage ID {nid}: {e}")
 
-    class DeviceTimer:
-        """Device event-based timer for accurate timing."""
-        def __init__(self, name="", device=device):
-            self.name = name
-            self.device = device
-            self.start_time = 0
-            self.end_time = 0
+                        # Prepare full-length response with large sentinel values
+                        large_distance = 1e9
+                        response_distances = [large_distance] * len(node_ids)
 
-            if cuda_available:
-                self.start_event = torch.cuda.Event(enable_timing=True)
-                self.end_event = torch.cuda.Event(enable_timing=True)
+                        if texts:
+                            try:
+                                embeddings = compute_embeddings(
+                                    texts, model_name, mode=embedding_mode
+                                )
+                                logger.info(
+                                    f"Computed embeddings for {len(texts)} texts, shape: {embeddings.shape}"
+                                )
+                                if distance_metric == "l2":
+                                    partial = np.sum(
+                                        np.square(embeddings - query_vector.reshape(1, -1)), axis=1
+                                    )
+                                else:  # mips or cosine
+                                    partial = -np.dot(embeddings, query_vector)
+
+                                for pos, dval in zip(found_indices, partial.flatten().tolist()):
+                                    response_distances[pos] = float(dval)
+                            except Exception as e:
+                                logger.error(f"Distance computation error, using sentinels: {e}")
+
+                        # Send response in expected shape [[distances]]
+                        rep_socket.send(msgpack.packb([response_distances], use_single_float=True))
+                        e2e_end = time.time()
+                        logger.info(f"⏱️  Distance calculation E2E time: {e2e_end - e2e_start:.6f}s")
+                        continue
+
+                    # Fallback: treat as embedding-by-id request
+                    if (
+                        isinstance(request, list)
+                        and len(request) == 1
+                        and isinstance(request[0], list)
+                    ):
+                        node_ids = request[0]
+                    elif isinstance(request, list):
+                        node_ids = request
                     else:
-                self.start_event = None
-                self.end_event = None
+                        node_ids = []
+                    last_request_type = "embedding"
+                    last_request_length = len(node_ids)
+                    logger.info(f"ZMQ received {len(node_ids)} node IDs for embedding fetch")
 
-        @contextmanager
-        def timing(self):
-            self.start()
-            yield
-            self.end()
-
-        def start(self):
-            if cuda_available:
-                torch.cuda.synchronize()
-                self.start_event.record()
+                    # Preallocate zero-filled flat data for robustness
+                    if embedding_dim <= 0:
+                        dims = [0, 0]
+                        flat_data: list[float] = []
                     else:
-                if self.device.type == "mps":
-                    torch.mps.synchronize()
-                self.start_time = time.time()
+                        dims = [len(node_ids), embedding_dim]
+                        flat_data = [0.0] * (dims[0] * dims[1])
 
-        def end(self):
-            if cuda_available:
-                self.end_event.record()
-                torch.cuda.synchronize()
+                    # Collect texts for found ids
+                    texts: list[str] = []
+                    found_indices: list[int] = []
+                    for idx, nid in enumerate(node_ids):
+                        try:
+                            passage_data = passages.get_passage(str(nid))
+                            txt = passage_data.get("text", "")
+                            if isinstance(txt, str) and len(txt) > 0:
+                                texts.append(txt)
+                                found_indices.append(idx)
                             else:
-                if self.device.type == "mps":
-                    torch.mps.synchronize()
-                self.end_time = time.time()
+                                logger.error(f"Empty text for passage ID {nid}")
+                        except KeyError:
+                            logger.error(f"Passage with ID {nid} not found")
+                        except Exception as e:
+                            logger.error(f"Exception looking up passage ID {nid}: {e}")
 
-        def elapsed_time(self):
-            if cuda_available:
-                return self.start_event.elapsed_time(self.end_event) / 1000.0
-            else:
-                return self.end_time - self.start_time
-
-        def print_elapsed(self):
-            return  # Disabled for now
-
-    def process_batch(texts_batch, ids_batch, missing_ids):
-        """Process a batch of texts and return embeddings"""
-        _is_e5_model = "e5" in model_name.lower()
-        _is_bge_model = "bge" in model_name.lower()
-        batch_size = len(texts_batch)
-        
-        # E5 model preprocessing
-        if _is_e5_model:
-            processed_texts_batch = [f"passage: {text}" for text in texts_batch]
-        else:
-            processed_texts_batch = texts_batch
-        
-        # Set max length
-        if _is_e5_model:
-            current_max_length = custom_max_length_param if custom_max_length_param is not None else 512
-        else:
-            current_max_length = custom_max_length_param if custom_max_length_param is not None else 256
-        
-        tokenize_timer = DeviceTimer("tokenization (batch)", device)
-        to_device_timer = DeviceTimer("transfer to device (batch)", device)
-        embed_timer = DeviceTimer("embedding (batch)", device)
-        pool_timer = DeviceTimer("pooling (batch)", device)
-        norm_timer = DeviceTimer("normalization (batch)", device)
-        
-        with tokenize_timer.timing():
-            encoded_batch = tokenizer(
-                processed_texts_batch,
-                padding="max_length",
-                truncation=True,
-                max_length=current_max_length,
-                return_tensors="pt",
-                return_token_type_ids=False,
+                    if texts:
+                        try:
+                            embeddings = compute_embeddings(texts, model_name, mode=embedding_mode)
+                            logger.info(
+                                f"Computed embeddings for {len(texts)} texts, shape: {embeddings.shape}"
                             )
 
-        seq_length = encoded_batch["input_ids"].size(1)
-        
-        with to_device_timer.timing():
-            enc = {k: v.to(device) for k, v in encoded_batch.items()}
-        
-        with torch.no_grad():
-            with embed_timer.timing():
-                out = model(enc["input_ids"], enc["attention_mask"])
-            
-            with pool_timer.timing():
-                if _is_bge_model:
-                    pooled_embeddings = out.last_hidden_state[:, 0]
-                elif not hasattr(out, 'last_hidden_state'):
-                    if isinstance(out, torch.Tensor) and len(out.shape) == 2:
-                        pooled_embeddings = out
+                            if np.isnan(embeddings).any() or np.isinf(embeddings).any():
+                                logger.error(
+                                    f"NaN or Inf detected in embeddings! Requested IDs: {node_ids[:5]}..."
+                                )
+                                dims = [0, embedding_dim]
+                                flat_data = []
                             else:
-                        print(f"{RED}ERROR: Cannot determine how to pool. Output shape: {out.shape if isinstance(out, torch.Tensor) else 'N/A'}{RESET}")
-                        hidden_dim = getattr(model.config, 'hidden_size', 384 if _is_e5_model else 768)
-                        pooled_embeddings = torch.zeros((batch_size, hidden_dim), device=device, dtype=enc["input_ids"].dtype if hasattr(enc["input_ids"], "dtype") else torch.float32)
-                elif _is_e5_model:
-                    pooled_embeddings = e5_average_pool(out.last_hidden_state, enc['attention_mask'])
-                else:
-                    hidden_states = out.last_hidden_state
-                    mask_expanded = enc["attention_mask"].unsqueeze(-1).expand(hidden_states.size()).float()
-                    sum_embeddings = torch.sum(hidden_states * mask_expanded, 1)
-                    sum_mask = torch.clamp(mask_expanded.sum(1), min=1e-9)
-                    pooled_embeddings = sum_embeddings / sum_mask
-            
-            final_embeddings = pooled_embeddings
-            if _is_e5_model or _is_bge_model:
-                with norm_timer.timing():
-                    final_embeddings = F.normalize(pooled_embeddings, p=2, dim=1)
-        
-        if torch.isnan(final_embeddings).any() or torch.isinf(final_embeddings).any():
-            print(f"{RED}!!! In process_batch: NaN or Inf detected in final_embeddings! "
-                  f"Model: {model_name}, E5: {_is_e5_model}. IDs (sample): {ids_batch[:5]}...{RESET}")
-            dim_size = final_embeddings.shape[-1]
-            error_output = torch.zeros((batch_size, dim_size), device='cpu', dtype=torch.float32).numpy()
-            print(f"{RED}Returning zero embeddings of shape ({batch_size}, {dim_size}) due to NaN/Inf.{RESET}")
-            return error_output
-        
-        return final_embeddings.cpu().numpy()
-
-    def client_warmup(zmq_port):
-        """Perform client-side warmup"""
-        time.sleep(2)
-        print(f"Performing client-side warmup with model {model_name}...")
-        sample_ids = ["1", "2", "3", "4", "5"]
-        
-        try:
-            context = zmq.Context()
-            socket = context.socket(zmq.REQ)
-            socket.connect(f"tcp://localhost:{zmq_port}")
-            socket.setsockopt(zmq.RCVTIMEO, 30000)
-            socket.setsockopt(zmq.SNDTIMEO, 30000)
-
-            try: 
-                ids_to_send = [int(x) for x in sample_ids]
-            except ValueError: 
-                ids_to_send = []
-
-            if not ids_to_send: 
-                print("Skipping warmup send.")
-                return
-
-            request_payload = [ids_to_send]
-            request_bytes = msgpack.packb(request_payload)
-
-            for i in range(3):
-                print(f"Sending warmup request {i+1}/3 via ZMQ (MessagePack)...")
-                socket.send(request_bytes)
-                response_bytes = socket.recv()
-
-                response_payload = msgpack.unpackb(response_bytes)
-                dimensions = response_payload[0]
-                embeddings_count = dimensions[0] if dimensions and len(dimensions) > 0 else 0
-                print(f"Warmup request {i+1}/3 successful, received {embeddings_count} embeddings")
-                time.sleep(0.1)
-
-            print("Client-side MessagePack ZMQ warmup complete")
-            socket.close()
-            context.term()
-        except Exception as e:
-            print(f"Error during MessagePack ZMQ warmup: {e}")
-
-    def zmq_server_thread():
-        """ZMQ server thread"""
-        context = zmq.Context()
-        socket = context.socket(zmq.REP)
-        socket.bind(f"tcp://*:{zmq_port}")
-        print(f"HNSW ZMQ server listening on port {zmq_port}")
-
-        socket.setsockopt(zmq.RCVTIMEO, 300000)
-        socket.setsockopt(zmq.SNDTIMEO, 300000)
-
-        while True:
-            try:
-                message_bytes = socket.recv()
-                print(f"Received ZMQ request of size {len(message_bytes)} bytes")
-
-                e2e_start = time.time()
-                lookup_timer = DeviceTimer("text lookup", device)
-
-                try:
-                    request_payload = msgpack.unpackb(message_bytes)
-                    
-                    # Handle distance calculation requests
-                    if isinstance(request_payload, list) and len(request_payload) == 2 and isinstance(request_payload[0], list) and isinstance(request_payload[1], list):
-                        node_ids = request_payload[0]
-                        query_vector = np.array(request_payload[1], dtype=np.float32)
-                        
-                        print(f"Request for distance calculation: {len(node_ids)} nodes, query vector dim: {len(query_vector)}")
-                        
-                        # Get embeddings for node IDs
-                        texts = []
-                        missing_ids = []
-                        with lookup_timer.timing():
-                            for nid in node_ids:
-                                try:
-                                    txtinfo = passages[nid]
-                                    if txtinfo is None or txtinfo["text"] == "":
-                                        raise RuntimeError(f"FATAL: Passage with ID {nid} not found - failing fast")
-                                    else:
-                                        txt = txtinfo["text"]
-                                except (KeyError, IndexError):
-                                    raise RuntimeError(f"FATAL: Passage with ID {nid} not found - failing fast")
-                                texts.append(txt)
-                        lookup_timer.print_elapsed()
-                        
-                        # Process embeddings in chunks if needed
-                        all_node_embeddings = []
-                        total_size = len(texts)
-                        
-                        if total_size > max_batch_size:
-                            for i in range(0, total_size, max_batch_size):
-                                end_idx = min(i + max_batch_size, total_size)
-                                chunk_texts = texts[i:end_idx]
-                                chunk_ids = node_ids[i:end_idx]
-                                
-                                embeddings_chunk = process_batch(chunk_texts, chunk_ids, missing_ids)
-                                all_node_embeddings.append(embeddings_chunk)
-                                
-                                if cuda_available:
-                                    torch.cuda.empty_cache()
-                                elif device.type == "mps":
-                                    torch.mps.empty_cache()
-                            
-                            node_embeddings = np.vstack(all_node_embeddings)
-                        else:
-                            node_embeddings = process_batch(texts, node_ids, missing_ids)
-                        
-                        # Calculate distances
-                        query_tensor = torch.tensor(query_vector, device=device).float()
-                        node_embeddings_tensor = torch.tensor(node_embeddings, device=device).float()
-                        
-                        calc_timer = DeviceTimer("distance calculation", device)
-                        with calc_timer.timing():
-                            with torch.no_grad():
-                                if distance_metric == "l2":
-                                    node_embeddings_np = node_embeddings_tensor.cpu().numpy().astype(np.float32)
-                                    query_np = query_tensor.cpu().numpy().astype(np.float32)
-                                    distances = np.sum(np.square(node_embeddings_np - query_np.reshape(1, -1)), axis=1)
-                                else: # mips or cosine
-                                    node_embeddings_np = node_embeddings_tensor.cpu().numpy()
-                                    query_np = query_tensor.cpu().numpy()
-                                    distances = -np.dot(node_embeddings_np, query_np)
-                        calc_timer.print_elapsed()
-                        
-                        try:
-                            response_payload = distances.flatten().tolist()
-                            response_bytes = msgpack.packb([response_payload], use_single_float=True)
-                            print(f"Sending distance response with {len(distances)} distances")
-                        except Exception as pack_error:
-                            print(f"Error packing MessagePack distance response: {pack_error}")
-                            response_bytes = msgpack.packb([[]])
-                        
-                        socket.send(response_bytes)
-                        
-                        if device.type == "cuda":
-                            torch.cuda.synchronize()
-                        elif device.type == "mps":
-                            torch.mps.synchronize()
-                        e2e_end = time.time()
-                        print(f"Distance calculation E2E time: {e2e_end - e2e_start:.6f} seconds")
-                        continue
-                    
-                    # Standard embedding request
-                    if not isinstance(request_payload, list) or len(request_payload) != 1 or not isinstance(request_payload[0], list):
-                        print(f"Error: Invalid MessagePack request format. Expected [[ids...]], got: {type(request_payload)}")
-                        socket.send(msgpack.packb([[], []]))
-                        continue
-                    
-                    node_ids = request_payload[0]
-                    print(f"Request for {len(node_ids)} node embeddings")
-                    
-                except Exception as unpack_error:
-                    print(f"Error unpacking MessagePack request: {unpack_error}")
-                    socket.send(msgpack.packb([[], []]))
-                    continue
-
-                # Look up texts by node IDs
-                texts = []
-                missing_ids = []
-                with lookup_timer.timing():
-                    for nid in node_ids:
-                        try:
-                            txtinfo = passages[nid]
-                            if txtinfo is None or txtinfo["text"] == "":
-                                raise RuntimeError(f"FATAL: Passage with ID {nid} not found - failing fast")
-                            else:
-                                txt = txtinfo["text"]
-                        except (KeyError, IndexError):
-                            raise RuntimeError(f"FATAL: Passage with ID {nid} not found - failing fast")
-                        texts.append(txt)
-                lookup_timer.print_elapsed()
-
-                if missing_ids:
-                    print(f"Missing passages for IDs: {missing_ids}")
-
-                # Process in chunks
-                total_size = len(texts)
-                print(f"Total batch size: {total_size}, max_batch_size: {max_batch_size}")
-                
-                all_embeddings = []
-                
-                if total_size > max_batch_size:
-                    print(f"Splitting batch of size {total_size} into chunks of {max_batch_size}")
-                    for i in range(0, total_size, max_batch_size):
-                        end_idx = min(i + max_batch_size, total_size)
-                        print(f"Processing chunk {i//max_batch_size + 1}/{(total_size + max_batch_size - 1)//max_batch_size}: items {i} to {end_idx-1}")
-                        
-                        chunk_texts = texts[i:end_idx]
-                        chunk_ids = node_ids[i:end_idx]
-                        
-                        embeddings_chunk = process_batch(chunk_texts, chunk_ids, missing_ids)
-                        all_embeddings.append(embeddings_chunk)
-                        
-                        if cuda_available:
-                            torch.cuda.empty_cache()
-                        elif device.type == "mps":
-                            torch.mps.empty_cache()
-                            
-                    hidden = np.vstack(all_embeddings)
-                    print(f"Combined embeddings shape: {hidden.shape}")
-                else:
-                    hidden = process_batch(texts, node_ids, missing_ids)
-
-                # Serialization and response
-                ser_start = time.time()
-
-                print(f"DEBUG zmq_server_thread: Final 'hidden' array | Shape: {hidden.shape} | Dtype: {hidden.dtype} | Has NaN/Inf: {np.isnan(hidden).any() or np.isinf(hidden).any()}")
-                if np.isnan(hidden).any() or np.isinf(hidden).any():
-                    print(f"{RED}!!! ERROR: NaN or Inf detected in final 'hidden' numpy array BEFORE sending! "
-                          f"Requested IDs (sample): {node_ids[:5]}...{RESET}")
-                    assert False
-
-                try:
-                    hidden_contiguous_f32 = np.ascontiguousarray(hidden, dtype=np.float32)
-                    response_payload = [
-                        list(hidden_contiguous_f32.shape),
-                        hidden_contiguous_f32.flatten().tolist()
+                                emb_f32 = np.ascontiguousarray(embeddings, dtype=np.float32)
+                                flat = emb_f32.flatten().tolist()
+                                for j, pos in enumerate(found_indices):
+                                    start = pos * embedding_dim
+                                    end = start + embedding_dim
+                                    if end <= len(flat_data):
+                                        flat_data[start:end] = flat[
+                                            j * embedding_dim : (j + 1) * embedding_dim
                                         ]
+                        except Exception as e:
+                            logger.error(f"Embedding computation error, returning zeros: {e}")
+
+                    response_payload = [dims, flat_data]
                     response_bytes = msgpack.packb(response_payload, use_single_float=True)
-                except Exception as pack_error:
-                     print(f"Error packing MessagePack response: {pack_error}")
-                     response_bytes = msgpack.packb([[], []])
 
-                socket.send(response_bytes)
-                ser_end = time.time()
-
-                print(f"Serialize time: {ser_end - ser_start:.6f} seconds")
-
-                if device.type == "cuda":
-                    torch.cuda.synchronize()
-                elif device.type == "mps":
-                    torch.mps.synchronize()
+                    rep_socket.send(response_bytes)
                     e2e_end = time.time()
-                print(f"ZMQ E2E time: {e2e_end - e2e_start:.6f} seconds")
+                    logger.info(f"⏱️  ZMQ E2E time: {e2e_end - e2e_start:.6f}s")
 
                 except zmq.Again:
-                print("ZMQ socket timeout, continuing to listen")
+                    # Timeout - check shutdown_event and continue
                     continue
                 except Exception as e:
-                print(f"Error in ZMQ server loop: {e}")
-                import traceback
-                traceback.print_exc()
+                    if not shutdown_event.is_set():
+                        logger.error(f"Error in ZMQ server loop: {e}")
+                        # Shape-correct fallback
                         try:
-                    socket.send(msgpack.packb([[], []]))
-                except:
+                            if last_request_type == "distance":
+                                large_distance = 1e9
+                                fallback_len = max(0, int(last_request_length))
+                                safe = [[large_distance] * fallback_len]
+                            elif last_request_type == "embedding":
+                                bsz = max(0, int(last_request_length))
+                                dim = max(0, int(embedding_dim))
+                                safe = (
+                                    [[bsz, dim], [0.0] * (bsz * dim)] if dim > 0 else [[0, 0], []]
+                                )
+                            elif last_request_type == "text":
+                                safe = []  # direct text embeddings expectation is a flat list
+                            else:
+                                safe = [[0, int(embedding_dim) if embedding_dim > 0 else 0], []]
+                            rep_socket.send(msgpack.packb(safe, use_single_float=True))
+                        except Exception:
+                            pass
+                    else:
+                        logger.info("Shutdown in progress, ignoring ZMQ error")
+                        break
+        finally:
+            try:
+                rep_socket.close(0)
+            except Exception:
+                pass
+            try:
+                context.term()
+            except Exception:
                 pass
 
-    # Start warmup and server threads
-    if len(passages) > 0:
-        warmup_thread = threading.Thread(target=client_warmup, args=(zmq_port,))
-        warmup_thread.daemon = True
-        warmup_thread.start()
+        logger.info("ZMQ server thread exiting gracefully")
 
-    zmq_thread = threading.Thread(target=zmq_server_thread, daemon=True)
+    # Add shutdown coordination
+    shutdown_event = threading.Event()
+
+    def shutdown_zmq_server():
+        """Gracefully shutdown ZMQ server."""
+        logger.info("Initiating graceful shutdown...")
+        shutdown_event.set()
+
+        if zmq_thread.is_alive():
+            logger.info("Waiting for ZMQ thread to finish...")
+            zmq_thread.join(timeout=5)
+            if zmq_thread.is_alive():
+                logger.warning("ZMQ thread did not finish in time")
+
+        # Clean up ZMQ resources
+        try:
+            # Note: socket and context are cleaned up by thread exit
+            logger.info("ZMQ resources cleaned up")
+        except Exception as e:
+            logger.warning(f"Error cleaning ZMQ resources: {e}")
+
+        # Clean up other resources
+        try:
+            import gc
+
+            gc.collect()
+            logger.info("Additional resources cleaned up")
+        except Exception as e:
+            logger.warning(f"Error cleaning additional resources: {e}")
+
+        logger.info("Graceful shutdown completed")
+        sys.exit(0)
+
+    # Register signal handlers within this function scope
+    import signal
+
+    def signal_handler(sig, frame):
+        logger.info(f"Received signal {sig}, shutting down gracefully...")
+        shutdown_zmq_server()
+
+    signal.signal(signal.SIGTERM, signal_handler)
+    signal.signal(signal.SIGINT, signal_handler)
+
+    # Pass shutdown_event to ZMQ thread
+    zmq_thread = threading.Thread(
+        target=lambda: zmq_server_thread_with_shutdown(shutdown_event),
+        daemon=False,  # Not daemon - we want to wait for it
+    )
     zmq_thread.start()
-    print(f"Started HNSW ZMQ server thread on port {zmq_port}")
+    logger.info(f"Started HNSW ZMQ server thread on port {zmq_port}")
 
     # Keep the main thread alive
     try:
-        while True:
-            time.sleep(1)
+        while not shutdown_event.is_set():
+            time.sleep(0.1)  # Check shutdown more frequently
     except KeyboardInterrupt:
-        print("HNSW Server shutting down...")
+        logger.info("HNSW Server shutting down...")
+        shutdown_zmq_server()
         return
 
+    # If we reach here, shutdown was triggered by signal
+    logger.info("Main loop exited, process should be shutting down")
+
 
 if __name__ == "__main__":
+    import sys
+
+    # Signal handlers are now registered within create_hnsw_embedding_server
+
     parser = argparse.ArgumentParser(description="HNSW Embedding service")
     parser.add_argument("--zmq-port", type=int, default=5555, help="ZMQ port to run on")
-    parser.add_argument("--passages-file", type=str, help="JSON file containing passage ID to text mapping")
-    parser.add_argument("--embeddings-file", type=str, help="Pickle file containing pre-computed embeddings")
-    parser.add_argument("--use-fp16", action="store_true", default=False)
-    parser.add_argument("--use-int8", action="store_true", default=False)
-    parser.add_argument("--use-cuda-graphs", action="store_true", default=False)
-    parser.add_argument("--max-batch-size", type=int, default=128, help="Maximum batch size before splitting")
-    parser.add_argument("--model-name", type=str, default="sentence-transformers/all-mpnet-base-v2", 
-                        help="Embedding model name")
-    parser.add_argument("--custom-max-length", type=int, default=None, help="Override model's default max sequence length")
-    parser.add_argument("--distance-metric", type=str, default="mips", help="Distance metric to use")
+    parser.add_argument(
+        "--passages-file",
+        type=str,
+        help="JSON file containing passage ID to text mapping",
+    )
+    parser.add_argument(
+        "--model-name",
+        type=str,
+        default="sentence-transformers/all-mpnet-base-v2",
+        help="Embedding model name",
+    )
+    parser.add_argument(
+        "--distance-metric", type=str, default="mips", help="Distance metric to use"
+    )
+    parser.add_argument(
+        "--embedding-mode",
+        type=str,
+        default="sentence-transformers",
+        choices=["sentence-transformers", "openai", "mlx", "ollama"],
+        help="Embedding backend mode",
+    )
 
     args = parser.parse_args()
 
     # Create and start the HNSW embedding server
     create_hnsw_embedding_server(
         passages_file=args.passages_file,
-        embeddings_file=args.embeddings_file,
-        use_fp16=args.use_fp16,
-        use_int8=args.use_int8,
-        use_cuda_graphs=args.use_cuda_graphs,
         zmq_port=args.zmq_port,
-        max_batch_size=args.max_batch_size,
         model_name=args.model_name,
-        custom_max_length_param=args.custom_max_length,
         distance_metric=args.distance_metric,
+        embedding_mode=args.embedding_mode,
     )

packages/leann-backend-hnsw/pyproject.toml

@@ -1,4 +1,4 @@
-# 文件: packages/leann-backend-hnsw/pyproject.toml
+# packages/leann-backend-hnsw/pyproject.toml
 
 [build-system]
 requires = ["scikit-build-core>=0.10", "numpy", "swig"]
@@ -6,13 +6,24 @@ build-backend = "scikit_build_core.build"
 
 [project]
 name = "leann-backend-hnsw"
-version = "0.1.0"
+version = "0.3.2"
 description = "Custom-built HNSW (Faiss) backend for the Leann toolkit."
-dependencies = ["leann-core==0.1.0", "numpy"]
+dependencies = [
+    "leann-core==0.3.2",
+    "numpy",
+    "pyzmq>=23.0.0",
+    "msgpack>=1.0.0",
+]
 
-# 回归到最标准的 scikit-build-core 配置
 [tool.scikit-build]
 wheel.packages = ["leann_backend_hnsw"]
 editable.mode = "redirect"
-cmake.build-type = "Debug"
+cmake.build-type = "Release"
 build.verbose = true
+build.tool-args = ["-j8"]
+
+# CMake definitions to optimize compilation and find Homebrew packages
+[tool.scikit-build.cmake.define]
+CMAKE_BUILD_PARALLEL_LEVEL = "8"
+CMAKE_PREFIX_PATH = {env = "CMAKE_PREFIX_PATH"}
+OpenMP_ROOT = {env = "OpenMP_ROOT"}

packages/leann-core/pyproject.toml

@@ -4,15 +4,49 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "leann-core"
-version = "0.1.0"
-description = "Core API and plugin system for Leann."
+version = "0.3.2"
+description = "Core API and plugin system for LEANN"
 readme = "README.md"
 requires-python = ">=3.9"
 license = { text = "MIT" }
 
+# All required dependencies included
 dependencies = [
-    "numpy>=1.20.0"
+    "numpy>=1.20.0",
+    "tqdm>=4.60.0",
+    "psutil>=5.8.0",
+    "pyzmq>=23.0.0",
+    "msgpack>=1.0.0",
+    "torch>=2.0.0",
+    "sentence-transformers>=2.2.0",
+    "llama-index-core>=0.12.0",
+    "llama-index-readers-file>=0.4.0",  # Essential for document reading
+    "llama-index-embeddings-huggingface>=0.5.5",  # For embeddings
+    "python-dotenv>=1.0.0",
+    "openai>=1.0.0",
+    "huggingface-hub>=0.20.0",
+    "transformers>=4.30.0",
+    "requests>=2.25.0",
+    "accelerate>=0.20.0",
+    "PyPDF2>=3.0.0",
+    "pymupdf>=1.23.0",
+    "pdfplumber>=0.10.0",
+    "nbconvert>=7.0.0",  # For .ipynb file support
+    "gitignore-parser>=0.1.12",  # For proper .gitignore handling
+    "mlx>=0.26.3; sys_platform == 'darwin' and platform_machine == 'arm64'",
+    "mlx-lm>=0.26.0; sys_platform == 'darwin' and platform_machine == 'arm64'",
 ]
 
+[project.optional-dependencies]
+colab = [
+    "torch>=2.0.0,<3.0.0",  # Limit torch version to avoid conflicts
+    "transformers>=4.30.0,<5.0.0",  # Limit transformers version
+    "accelerate>=0.20.0,<1.0.0",  # Limit accelerate version
+]
+
+[project.scripts]
+leann = "leann.cli:main"
+leann_mcp = "leann.mcp:main"
+
 [tool.setuptools.packages.find]
 where = ["src"]

packages/leann-core/src/leann/__init__.py

@@ -1,17 +1,21 @@
-# This file makes the 'leann' directory a Python package.
+# packages/leann-core/src/leann/__init__.py
+import os
+import platform
 
-from .api import LeannBuilder, LeannSearcher, LeannChat, SearchResult
+# Fix OpenMP threading issues on macOS ARM64
+if platform.system() == "Darwin":
+    os.environ["OMP_NUM_THREADS"] = "1"
+    os.environ["MKL_NUM_THREADS"] = "1"
+    os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
+    os.environ["KMP_BLOCKTIME"] = "0"
+    # Additional fixes for PyTorch/sentence-transformers on macOS ARM64 only in CI
+    if os.environ.get("CI") == "true":
+        os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "0"
+        os.environ["TOKENIZERS_PARALLELISM"] = "false"
 
-# Import backends to ensure they are registered
-try:
-    import leann_backend_hnsw
-except ImportError:
-    pass
+from .api import LeannBuilder, LeannChat, LeannSearcher
+from .registry import BACKEND_REGISTRY, autodiscover_backends
 
-try:
-    import leann_backend_diskann
-except ImportError:
-    pass
+autodiscover_backends()
 
-
-__all__ = ['LeannBuilder', 'LeannSearcher', 'LeannChat', 'SearchResult']
+__all__ = ["BACKEND_REGISTRY", "LeannBuilder", "LeannChat", "LeannSearcher"]

packages/leann-core/src/leann/chat.py

@@ -0,0 +1,857 @@
+#!/usr/bin/env python3
+"""
+This file contains the chat generation logic for the LEANN project,
+supporting different backends like Ollama, Hugging Face Transformers, and a simulation mode.
+"""
+
+import difflib
+import logging
+import os
+from abc import ABC, abstractmethod
+from typing import Any, Optional
+
+import torch
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def check_ollama_models(host: str) -> list[str]:
+    """Check available Ollama models and return a list"""
+    try:
+        import requests
+
+        response = requests.get(f"{host}/api/tags", timeout=5)
+        if response.status_code == 200:
+            data = response.json()
+            return [model["name"] for model in data.get("models", [])]
+        return []
+    except Exception:
+        return []
+
+
+def check_ollama_model_exists_remotely(model_name: str) -> tuple[bool, list[str]]:
+    """Check if a model exists in Ollama's remote library and return available tags
+
+    Returns:
+        (model_exists, available_tags): bool and list of matching tags
+    """
+    try:
+        import re
+
+        import requests
+
+        # Split model name and tag
+        if ":" in model_name:
+            base_model, requested_tag = model_name.split(":", 1)
+        else:
+            base_model, requested_tag = model_name, None
+
+        # First check if base model exists in library
+        library_response = requests.get("https://ollama.com/library", timeout=8)
+        if library_response.status_code != 200:
+            return True, []  # Assume exists if can't check
+
+        # Extract model names from library page
+        models_in_library = re.findall(r'href="/library/([^"]+)"', library_response.text)
+
+        if base_model not in models_in_library:
+            return False, []  # Base model doesn't exist
+
+        # If base model exists, get available tags
+        tags_response = requests.get(f"https://ollama.com/library/{base_model}/tags", timeout=8)
+        if tags_response.status_code != 200:
+            return True, []  # Base model exists but can't get tags
+
+        # Extract tags for this model - be more specific to avoid HTML artifacts
+        tag_pattern = rf"{re.escape(base_model)}:[a-zA-Z0-9\.\-_]+"
+        raw_tags = re.findall(tag_pattern, tags_response.text)
+
+        # Clean up tags - remove HTML artifacts and duplicates
+        available_tags = []
+        seen = set()
+        for tag in raw_tags:
+            # Skip if it looks like HTML (contains < or >)
+            if "<" in tag or ">" in tag:
+                continue
+            if tag not in seen:
+                seen.add(tag)
+                available_tags.append(tag)
+
+        # Check if exact model exists
+        if requested_tag is None:
+            # User just requested base model, suggest tags
+            return True, available_tags[:10]  # Return up to 10 tags
+        else:
+            exact_match = model_name in available_tags
+            return exact_match, available_tags[:10]
+
+    except Exception:
+        pass
+
+    # If scraping fails, assume model might exist (don't block user)
+    return True, []
+
+
+def search_ollama_models_fuzzy(query: str, available_models: list[str]) -> list[str]:
+    """Use intelligent fuzzy search for Ollama models"""
+    if not available_models:
+        return []
+
+    query_lower = query.lower()
+    suggestions = []
+
+    # 1. Exact matches first
+    exact_matches = [m for m in available_models if query_lower == m.lower()]
+    suggestions.extend(exact_matches)
+
+    # 2. Starts with query
+    starts_with = [
+        m for m in available_models if m.lower().startswith(query_lower) and m not in suggestions
+    ]
+    suggestions.extend(starts_with)
+
+    # 3. Contains query
+    contains = [m for m in available_models if query_lower in m.lower() and m not in suggestions]
+    suggestions.extend(contains)
+
+    # 4. Base model name matching (remove version numbers)
+    def get_base_name(model_name: str) -> str:
+        """Extract base name without version (e.g., 'llama3:8b' -> 'llama3')"""
+        return model_name.split(":")[0].split("-")[0]
+
+    query_base = get_base_name(query_lower)
+    base_matches = [
+        m
+        for m in available_models
+        if get_base_name(m.lower()) == query_base and m not in suggestions
+    ]
+    suggestions.extend(base_matches)
+
+    # 5. Family/variant matching
+    model_families = {
+        "llama": ["llama2", "llama3", "alpaca", "vicuna", "codellama"],
+        "qwen": ["qwen", "qwen2", "qwen3"],
+        "gemma": ["gemma", "gemma2"],
+        "phi": ["phi", "phi2", "phi3"],
+        "mistral": ["mistral", "mixtral", "openhermes"],
+        "dolphin": ["dolphin", "openchat"],
+        "deepseek": ["deepseek", "deepseek-coder"],
+    }
+
+    query_family = None
+    for family, variants in model_families.items():
+        if any(variant in query_lower for variant in variants):
+            query_family = family
+            break
+
+    if query_family:
+        family_variants = model_families[query_family]
+        family_matches = [
+            m
+            for m in available_models
+            if any(variant in m.lower() for variant in family_variants) and m not in suggestions
+        ]
+        suggestions.extend(family_matches)
+
+    # 6. Use difflib for remaining fuzzy matches
+    remaining_models = [m for m in available_models if m not in suggestions]
+    difflib_matches = difflib.get_close_matches(query_lower, remaining_models, n=3, cutoff=0.4)
+    suggestions.extend(difflib_matches)
+
+    return suggestions[:8]  # Return top 8 suggestions
+
+
+# Remove this function entirely - we don't need external API calls for Ollama
+
+
+# Remove this too - no need for fallback
+
+
+def suggest_similar_models(invalid_model: str, available_models: list[str]) -> list[str]:
+    """Use difflib to find similar model names"""
+    if not available_models:
+        return []
+
+    # Get close matches using fuzzy matching
+    suggestions = difflib.get_close_matches(invalid_model, available_models, n=3, cutoff=0.3)
+    return suggestions
+
+
+def check_hf_model_exists(model_name: str) -> bool:
+    """Quick check if HuggingFace model exists without downloading"""
+    try:
+        from huggingface_hub import model_info
+
+        model_info(model_name)
+        return True
+    except Exception:
+        return False
+
+
+def get_popular_hf_models() -> list[str]:
+    """Return a list of popular HuggingFace models for suggestions"""
+    try:
+        from huggingface_hub import list_models
+
+        # Get popular text-generation models, sorted by downloads
+        models = list_models(
+            filter="text-generation",
+            sort="downloads",
+            direction=-1,
+            limit=20,  # Get top 20 most downloaded
+        )
+
+        # Extract model names and filter for chat/conversation models
+        model_names = []
+        chat_keywords = ["chat", "instruct", "dialog", "conversation", "assistant"]
+
+        for model in models:
+            model_name = model.id if hasattr(model, "id") else str(model)
+            # Prioritize models with chat-related keywords
+            if any(keyword in model_name.lower() for keyword in chat_keywords):
+                model_names.append(model_name)
+            elif len(model_names) < 10:  # Fill up with other popular models
+                model_names.append(model_name)
+
+        return model_names[:10] if model_names else _get_fallback_hf_models()
+
+    except Exception:
+        # Fallback to static list if API call fails
+        return _get_fallback_hf_models()
+
+
+def _get_fallback_hf_models() -> list[str]:
+    """Fallback list of popular HuggingFace models"""
+    return [
+        "microsoft/DialoGPT-medium",
+        "microsoft/DialoGPT-large",
+        "facebook/blenderbot-400M-distill",
+        "microsoft/phi-2",
+        "deepseek-ai/deepseek-llm-7b-chat",
+        "microsoft/DialoGPT-small",
+        "facebook/blenderbot_small-90M",
+        "microsoft/phi-1_5",
+        "facebook/opt-350m",
+        "EleutherAI/gpt-neo-1.3B",
+    ]
+
+
+def search_hf_models_fuzzy(query: str, limit: int = 10) -> list[str]:
+    """Use HuggingFace Hub's native fuzzy search for model suggestions"""
+    try:
+        from huggingface_hub import list_models
+
+        # HF Hub's search is already fuzzy! It handles typos and partial matches
+        models = list_models(
+            search=query,
+            filter="text-generation",
+            sort="downloads",
+            direction=-1,
+            limit=limit,
+        )
+
+        model_names = [model.id if hasattr(model, "id") else str(model) for model in models]
+
+        # If direct search doesn't return enough results, try some variations
+        if len(model_names) < 3:
+            # Try searching for partial matches or common variations
+            variations = []
+
+            # Extract base name (e.g., "gpt3" from "gpt-3.5")
+            base_query = query.lower().replace("-", "").replace(".", "").replace("_", "")
+            if base_query != query.lower():
+                variations.append(base_query)
+
+            # Try common model name patterns
+            if "gpt" in query.lower():
+                variations.extend(["gpt2", "gpt-neo", "gpt-j", "dialoGPT"])
+            elif "llama" in query.lower():
+                variations.extend(["llama2", "alpaca", "vicuna"])
+            elif "bert" in query.lower():
+                variations.extend(["roberta", "distilbert", "albert"])
+
+            # Search with variations
+            for var in variations[:2]:  # Limit to 2 variations to avoid too many API calls
+                try:
+                    var_models = list_models(
+                        search=var,
+                        filter="text-generation",
+                        sort="downloads",
+                        direction=-1,
+                        limit=3,
+                    )
+                    var_names = [
+                        model.id if hasattr(model, "id") else str(model) for model in var_models
+                    ]
+                    model_names.extend(var_names)
+                except Exception:
+                    continue
+
+        # Remove duplicates while preserving order
+        seen = set()
+        unique_models = []
+        for model in model_names:
+            if model not in seen:
+                seen.add(model)
+                unique_models.append(model)
+
+        return unique_models[:limit]
+
+    except Exception:
+        # If search fails, return empty list
+        return []
+
+
+def search_hf_models(query: str, limit: int = 10) -> list[str]:
+    """Simple search for HuggingFace models based on query (kept for backward compatibility)"""
+    return search_hf_models_fuzzy(query, limit)
+
+
+def validate_model_and_suggest(
+    model_name: str, llm_type: str, host: str = "http://localhost:11434"
+) -> Optional[str]:
+    """Validate model name and provide suggestions if invalid"""
+    if llm_type == "ollama":
+        available_models = check_ollama_models(host)
+        if available_models and model_name not in available_models:
+            error_msg = f"Model '{model_name}' not found in your local Ollama installation."
+
+            # Check if the model exists remotely and get available tags
+            model_exists_remotely, available_tags = check_ollama_model_exists_remotely(model_name)
+
+            if model_exists_remotely and model_name in available_tags:
+                # Exact model exists remotely - suggest pulling it
+                error_msg += "\n\nTo install the requested model:\n"
+                error_msg += f"  ollama pull {model_name}\n"
+
+                # Show local alternatives
+                suggestions = search_ollama_models_fuzzy(model_name, available_models)
+                if suggestions:
+                    error_msg += "\nOr use one of these similar installed models:\n"
+                    for i, suggestion in enumerate(suggestions, 1):
+                        error_msg += f"  {i}. {suggestion}\n"
+
+            elif model_exists_remotely and available_tags:
+                # Base model exists but requested tag doesn't - suggest correct tags
+                base_model = model_name.split(":")[0]
+                requested_tag = model_name.split(":", 1)[1] if ":" in model_name else None
+
+                error_msg += (
+                    f"\n\nModel '{base_model}' exists, but tag '{requested_tag}' is not available."
+                )
+                error_msg += f"\n\nAvailable {base_model} models you can install:\n"
+                for i, tag in enumerate(available_tags[:8], 1):
+                    error_msg += f"  {i}. ollama pull {tag}\n"
+                if len(available_tags) > 8:
+                    error_msg += f"  ... and {len(available_tags) - 8} more variants\n"
+
+                # Also show local alternatives
+                suggestions = search_ollama_models_fuzzy(model_name, available_models)
+                if suggestions:
+                    error_msg += "\nOr use one of these similar installed models:\n"
+                    for i, suggestion in enumerate(suggestions, 1):
+                        error_msg += f"  {i}. {suggestion}\n"
+
+            else:
+                # Model doesn't exist remotely - show fuzzy suggestions
+                suggestions = search_ollama_models_fuzzy(model_name, available_models)
+                error_msg += f"\n\nModel '{model_name}' was not found in Ollama's library."
+
+                if suggestions:
+                    error_msg += (
+                        "\n\nDid you mean one of these installed models?\n"
+                        + "\nTry to use ollama pull to install the model you need\n"
+                    )
+
+                    for i, suggestion in enumerate(suggestions, 1):
+                        error_msg += f"  {i}. {suggestion}\n"
+                else:
+                    error_msg += "\n\nYour installed models:\n"
+                    for i, model in enumerate(available_models[:8], 1):
+                        error_msg += f"  {i}. {model}\n"
+                    if len(available_models) > 8:
+                        error_msg += f"  ... and {len(available_models) - 8} more\n"
+
+            error_msg += "\n\nCommands:"
+            error_msg += "\n  ollama list                    # List installed models"
+            if model_exists_remotely and available_tags:
+                if model_name in available_tags:
+                    error_msg += f"\n  ollama pull {model_name}          # Install requested model"
+                else:
+                    error_msg += (
+                        f"\n  ollama pull {available_tags[0]}    # Install recommended variant"
+                    )
+            error_msg += "\n  https://ollama.com/library     # Browse available models"
+            return error_msg
+
+    elif llm_type == "hf":
+        # For HF models, we can do a quick existence check
+        if not check_hf_model_exists(model_name):
+            # Use HF Hub's native fuzzy search directly
+            search_suggestions = search_hf_models_fuzzy(model_name, limit=8)
+
+            error_msg = f"Model '{model_name}' not found on HuggingFace Hub."
+            if search_suggestions:
+                error_msg += "\n\nDid you mean one of these?\n"
+                for i, suggestion in enumerate(search_suggestions, 1):
+                    error_msg += f"  {i}. {suggestion}\n"
+            else:
+                # Fallback to popular models if search returns nothing
+                popular_models = get_popular_hf_models()
+                error_msg += "\n\nPopular chat models:\n"
+                for i, model in enumerate(popular_models[:5], 1):
+                    error_msg += f"  {i}. {model}\n"
+
+            error_msg += f"\nSearch more: https://huggingface.co/models?search={model_name}&pipeline_tag=text-generation"
+            return error_msg
+
+    return None  # Model is valid or we can't check
+
+
+class LLMInterface(ABC):
+    """Abstract base class for a generic Language Model (LLM) interface."""
+
+    @abstractmethod
+    def ask(self, prompt: str, **kwargs) -> str:
+        """
+        Additional keyword arguments (kwargs) for advanced search customization. Example usage:
+            chat.ask(
+                "What is ANN?",
+                top_k=10,
+                complexity=64,
+                beam_width=8,
+                skip_search_reorder=True,
+                recompute_beighbor_embeddings=True,
+                dedup_node_dis=True,
+                prune_ratio=0.1,
+                batch_recompute=True,
+                global_pruning=True
+            )
+
+        Supported kwargs:
+            - complexity (int): Search complexity parameter (default: 32)
+            - beam_width (int): Beam width for search (default: 4)
+            - skip_search_reorder (bool): Skip search reorder step (default: False)
+            - recompute_beighbor_embeddings (bool): Enable ZMQ embedding server for neighbor recomputation (default: False)
+            - dedup_node_dis (bool): Deduplicate nodes by distance (default: False)
+            - prune_ratio (float): Pruning ratio for search (default: 0.0)
+            - batch_recompute (bool): Enable batch recomputation (default: False)
+            - global_pruning (bool): Enable global pruning (default: False)
+        """
+
+        # """
+        # Sends a prompt to the LLM and returns the generated text.
+
+        # Args:
+        #     prompt: The input prompt for the LLM.
+        #     **kwargs: Additional keyword arguments for the LLM backend.
+
+        # Returns:
+        #     The response string from the LLM.
+        # """
+        pass
+
+
+class OllamaChat(LLMInterface):
+    """LLM interface for Ollama models."""
+
+    def __init__(self, model: str = "llama3:8b", host: str = "http://localhost:11434"):
+        self.model = model
+        self.host = host
+        logger.info(f"Initializing OllamaChat with model='{model}' and host='{host}'")
+        try:
+            import requests
+
+            # Check if the Ollama server is responsive
+            if host:
+                requests.get(host)
+
+            # Pre-check model availability with helpful suggestions
+            model_error = validate_model_and_suggest(model, "ollama", host)
+            if model_error:
+                raise ValueError(model_error)
+
+        except ImportError:
+            raise ImportError(
+                "The 'requests' library is required for Ollama. Please install it with 'pip install requests'."
+            )
+        except requests.exceptions.ConnectionError:
+            logger.error(f"Could not connect to Ollama at {host}. Please ensure Ollama is running.")
+            raise ConnectionError(
+                f"Could not connect to Ollama at {host}. Please ensure Ollama is running."
+            )
+
+    def ask(self, prompt: str, **kwargs) -> str:
+        import json
+
+        import requests
+
+        full_url = f"{self.host}/api/generate"
+
+        # Handle thinking budget for reasoning models
+        options = kwargs.copy()
+        thinking_budget = kwargs.get("thinking_budget")
+        if thinking_budget:
+            # Remove thinking_budget from options as it's not a standard Ollama option
+            options.pop("thinking_budget", None)
+            # Only apply reasoning parameters to models that support it
+            reasoning_supported_models = [
+                "gpt-oss:20b",
+                "gpt-oss:120b",
+                "deepseek-r1",
+                "deepseek-coder",
+            ]
+
+            if thinking_budget in ["low", "medium", "high"]:
+                if any(model in self.model.lower() for model in reasoning_supported_models):
+                    options["reasoning"] = {"effort": thinking_budget, "exclude": False}
+                    logger.info(f"Applied reasoning effort={thinking_budget} to model {self.model}")
+                else:
+                    logger.warning(
+                        f"Thinking budget '{thinking_budget}' requested but model '{self.model}' may not support reasoning parameters. Proceeding without reasoning."
+                    )
+
+        payload = {
+            "model": self.model,
+            "prompt": prompt,
+            "stream": False,  # Keep it simple for now
+            "options": options,
+        }
+        logger.debug(f"Sending request to Ollama: {payload}")
+        try:
+            logger.info("Sending request to Ollama and waiting for response...")
+            max_tokens = kwargs.get("max_tokens", 1000)
+            payload["options"]["max_tokens"] = max_tokens
+            response = requests.post(full_url, data=json.dumps(payload))
+            response.raise_for_status()
+
+            # The response from Ollama can be a stream of JSON objects, handle this
+            response_parts = response.text.strip().split("\n")
+            full_response = ""
+            for part in response_parts:
+                if part:
+                    json_part = json.loads(part)
+                    full_response += json_part.get("response", "")
+                    if json_part.get("done"):
+                        break
+            return full_response
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Error communicating with Ollama: {e}")
+            return f"Error: Could not get a response from Ollama. Details: {e}"
+
+
+class HFChat(LLMInterface):
+    """LLM interface for local Hugging Face Transformers models with proper chat templates."""
+
+    def __init__(self, model_name: str = "deepseek-ai/deepseek-llm-7b-chat"):
+        logger.info(f"Initializing HFChat with model='{model_name}'")
+
+        # Pre-check model availability with helpful suggestions
+        model_error = validate_model_and_suggest(model_name, "hf")
+        if model_error:
+            raise ValueError(model_error)
+
+        try:
+            import torch
+            from transformers import AutoModelForCausalLM, AutoTokenizer
+        except ImportError:
+            raise ImportError(
+                "The 'transformers' and 'torch' libraries are required for Hugging Face models. Please install them with 'pip install transformers torch'."
+            )
+
+        # Auto-detect device
+        if torch.cuda.is_available():
+            self.device = "cuda"
+            logger.info("CUDA is available. Using GPU.")
+        elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
+            self.device = "mps"
+            logger.info("MPS is available. Using Apple Silicon GPU.")
+        else:
+            self.device = "cpu"
+            logger.info("No GPU detected. Using CPU.")
+
+        # Load tokenizer and model with timeout protection
+        try:
+            import signal
+
+            def timeout_handler(signum, frame):
+                raise TimeoutError("Model download/loading timed out")
+
+            # Set timeout for model loading (60 seconds)
+            old_handler = signal.signal(signal.SIGALRM, timeout_handler)
+            signal.alarm(60)
+
+            try:
+                logger.info(f"Loading tokenizer for {model_name}...")
+                self.tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+                logger.info(f"Loading model {model_name}...")
+                self.model = AutoModelForCausalLM.from_pretrained(
+                    model_name,
+                    torch_dtype=torch.float16 if self.device != "cpu" else torch.float32,
+                    device_map="auto" if self.device != "cpu" else None,
+                    trust_remote_code=True,
+                )
+                logger.info(f"Successfully loaded {model_name}")
+            finally:
+                signal.alarm(0)  # Cancel the alarm
+                signal.signal(signal.SIGALRM, old_handler)  # Restore old handler
+
+        except TimeoutError:
+            logger.error(f"Model loading timed out for {model_name}")
+            raise RuntimeError(
+                f"Model loading timed out for {model_name}. Please check your internet connection or try a smaller model."
+            )
+        except Exception as e:
+            logger.error(f"Failed to load model {model_name}: {e}")
+            raise
+
+        # Move model to device if not using device_map
+        if self.device != "cpu" and "device_map" not in str(self.model):
+            self.model = self.model.to(self.device)
+
+        # Set pad token if not present
+        if self.tokenizer.pad_token is None:
+            self.tokenizer.pad_token = self.tokenizer.eos_token
+
+    def ask(self, prompt: str, **kwargs) -> str:
+        print("kwargs in HF: ", kwargs)
+        # Check if this is a Qwen model and add /no_think by default
+        is_qwen_model = "qwen" in self.model.config._name_or_path.lower()
+
+        # For Qwen models, automatically add /no_think to the prompt
+        # if is_qwen_model and "/no_think" not in prompt and "/think" not in prompt:
+        #     prompt = prompt + " /no_think"
+
+        # Prepare chat template
+        messages = [{"role": "user", "content": prompt}]
+
+        # Apply chat template if available
+        if hasattr(self.tokenizer, "apply_chat_template"):
+            try:
+                formatted_prompt = self.tokenizer.apply_chat_template(
+                    messages, tokenize=False, add_generation_prompt=True
+                )
+            except Exception as e:
+                logger.warning(f"Chat template failed, using raw prompt: {e}")
+                formatted_prompt = prompt
+        else:
+            # Fallback for models without chat template
+            formatted_prompt = prompt
+
+        # Tokenize input
+        inputs = self.tokenizer(
+            formatted_prompt,
+            return_tensors="pt",
+            padding=True,
+            truncation=True,
+            max_length=2048,
+        )
+
+        # Move inputs to device
+        if self.device != "cpu":
+            inputs = {k: v.to(self.device) for k, v in inputs.items()}
+
+        # Set generation parameters
+        generation_config = {
+            "max_new_tokens": kwargs.get("max_tokens", kwargs.get("max_new_tokens", 512)),
+            "temperature": kwargs.get("temperature", 0.7),
+            "top_p": kwargs.get("top_p", 0.9),
+            "do_sample": kwargs.get("temperature", 0.7) > 0,
+            "pad_token_id": self.tokenizer.eos_token_id,
+            "eos_token_id": self.tokenizer.eos_token_id,
+        }
+
+        # Handle temperature=0 for greedy decoding
+        if generation_config["temperature"] == 0.0:
+            generation_config["do_sample"] = False
+            generation_config.pop("temperature")
+
+        logger.info(f"Generating with HuggingFace model, config: {generation_config}")
+
+        # Generate
+        with torch.no_grad():
+            outputs = self.model.generate(**inputs, **generation_config)
+
+        # Decode response
+        generated_tokens = outputs[0][inputs["input_ids"].shape[1] :]
+        response = self.tokenizer.decode(generated_tokens, skip_special_tokens=True)
+
+        return response.strip()
+
+
+class GeminiChat(LLMInterface):
+    """LLM interface for Google Gemini models."""
+
+    def __init__(self, model: str = "gemini-2.5-flash", api_key: Optional[str] = None):
+        self.model = model
+        self.api_key = api_key or os.getenv("GEMINI_API_KEY")
+
+        if not self.api_key:
+            raise ValueError(
+                "Gemini API key is required. Set GEMINI_API_KEY environment variable or pass api_key parameter."
+            )
+
+        logger.info(f"Initializing Gemini Chat with model='{model}'")
+
+        try:
+            import google.genai as genai
+
+            self.client = genai.Client(api_key=self.api_key)
+        except ImportError:
+            raise ImportError(
+                "The 'google-genai' library is required for Gemini models. Please install it with 'uv pip install google-genai'."
+            )
+
+    def ask(self, prompt: str, **kwargs) -> str:
+        logger.info(f"Sending request to Gemini with model {self.model}")
+
+        try:
+            from google.genai.types import GenerateContentConfig
+
+            generation_config = GenerateContentConfig(
+                temperature=kwargs.get("temperature", 0.7),
+                max_output_tokens=kwargs.get("max_tokens", 1000),
+            )
+
+            # Handle top_p parameter
+            if "top_p" in kwargs:
+                generation_config.top_p = kwargs["top_p"]
+
+            response = self.client.models.generate_content(
+                model=self.model,
+                contents=prompt,
+                config=generation_config,
+            )
+            # Handle potential None response text
+            response_text = response.text
+            if response_text is None:
+                logger.warning("Gemini returned None response text")
+                return ""
+            return response_text.strip()
+        except Exception as e:
+            logger.error(f"Error communicating with Gemini: {e}")
+            return f"Error: Could not get a response from Gemini. Details: {e}"
+
+
+class OpenAIChat(LLMInterface):
+    """LLM interface for OpenAI models."""
+
+    def __init__(self, model: str = "gpt-4o", api_key: Optional[str] = None):
+        self.model = model
+        self.api_key = api_key or os.getenv("OPENAI_API_KEY")
+
+        if not self.api_key:
+            raise ValueError(
+                "OpenAI API key is required. Set OPENAI_API_KEY environment variable or pass api_key parameter."
+            )
+
+        logger.info(f"Initializing OpenAI Chat with model='{model}'")
+
+        try:
+            import openai
+
+            self.client = openai.OpenAI(api_key=self.api_key)
+        except ImportError:
+            raise ImportError(
+                "The 'openai' library is required for OpenAI models. Please install it with 'pip install openai'."
+            )
+
+    def ask(self, prompt: str, **kwargs) -> str:
+        # Default parameters for OpenAI
+        params = {
+            "model": self.model,
+            "messages": [{"role": "user", "content": prompt}],
+            "temperature": kwargs.get("temperature", 0.7),
+        }
+
+        # Handle max_tokens vs max_completion_tokens based on model
+        max_tokens = kwargs.get("max_tokens", 1000)
+        if "o3" in self.model or "o4" in self.model or "o1" in self.model:
+            # o-series models use max_completion_tokens
+            params["max_completion_tokens"] = max_tokens
+            params["temperature"] = 1.0
+        else:
+            # Other models use max_tokens
+            params["max_tokens"] = max_tokens
+
+        # Handle thinking budget for reasoning models
+        thinking_budget = kwargs.get("thinking_budget")
+        if thinking_budget and thinking_budget in ["low", "medium", "high"]:
+            # Check if this is an o-series model (partial match for model names)
+            o_series_models = ["o3", "o3-mini", "o4-mini", "o1", "o3-pro", "o3-deep-research"]
+            if any(model in self.model for model in o_series_models):
+                # Use the correct OpenAI reasoning parameter format
+                params["reasoning_effort"] = thinking_budget
+                logger.info(f"Applied reasoning_effort={thinking_budget} to model {self.model}")
+            else:
+                logger.warning(
+                    f"Thinking budget '{thinking_budget}' requested but model '{self.model}' may not support reasoning parameters. Proceeding without reasoning."
+                )
+
+        # Add other kwargs (excluding thinking_budget as it's handled above)
+        for k, v in kwargs.items():
+            if k not in ["max_tokens", "temperature", "thinking_budget"]:
+                params[k] = v
+
+        logger.info(f"Sending request to OpenAI with model {self.model}")
+
+        try:
+            response = self.client.chat.completions.create(**params)
+            return response.choices[0].message.content.strip()
+        except Exception as e:
+            logger.error(f"Error communicating with OpenAI: {e}")
+            return f"Error: Could not get a response from OpenAI. Details: {e}"
+
+
+class SimulatedChat(LLMInterface):
+    """A simple simulated chat for testing and development."""
+
+    def ask(self, prompt: str, **kwargs) -> str:
+        logger.info("Simulating LLM call...")
+        print("Prompt sent to LLM (simulation):", prompt[:500] + "...")
+        return "This is a simulated answer from the LLM based on the retrieved context."
+
+
+def get_llm(llm_config: Optional[dict[str, Any]] = None) -> LLMInterface:
+    """
+    Factory function to get an LLM interface based on configuration.
+
+    Args:
+        llm_config: A dictionary specifying the LLM type and its parameters.
+                    Example: {"type": "ollama", "model": "llama3"}
+                             {"type": "hf", "model": "distilgpt2"}
+                             None (for simulation mode)
+
+    Returns:
+        An instance of an LLMInterface subclass.
+    """
+    if llm_config is None:
+        llm_config = {
+            "type": "openai",
+            "model": "gpt-4o",
+            "api_key": os.getenv("OPENAI_API_KEY"),
+        }
+
+    llm_type = llm_config.get("type", "openai")
+    model = llm_config.get("model")
+
+    logger.info(f"Attempting to create LLM of type='{llm_type}' with model='{model}'")
+
+    if llm_type == "ollama":
+        return OllamaChat(
+            model=model or "llama3:8b",
+            host=llm_config.get("host", "http://localhost:11434"),
+        )
+    elif llm_type == "hf":
+        return HFChat(model_name=model or "deepseek-ai/deepseek-llm-7b-chat")
+    elif llm_type == "openai":
+        return OpenAIChat(model=model or "gpt-4o", api_key=llm_config.get("api_key"))
+    elif llm_type == "gemini":
+        return GeminiChat(model=model or "gemini-2.5-flash", api_key=llm_config.get("api_key"))
+    elif llm_type == "simulated":
+        return SimulatedChat()
+    else:
+        raise ValueError(f"Unknown LLM type: '{llm_type}'")