8c988cf98b
- Move pytest configuration from pytest.ini to pyproject.toml - Remove unnecessary run_tests.py script (use test extras instead) - Fix main_cli_example.py to properly use command line arguments for LLM config - Add test_readme_examples.py to test code examples from README - Refactor tests to use pytest fixtures and parametrization - Update test documentation to reflect new structure - Set proper environment variables in CI for test execution
115 lines
3.4 KiB
Python
115 lines
3.4 KiB
Python
"""
|
|
Test main_cli_example functionality using pytest.
|
|
"""
|
|
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
|
|
@pytest.fixture
|
|
def test_data_dir():
|
|
"""Return the path to test data directory."""
|
|
return Path("examples/data")
|
|
|
|
|
|
def test_main_cli_simulated(test_data_dir):
|
|
"""Test main_cli with simulated LLM."""
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
cmd = [
|
|
sys.executable,
|
|
"examples/main_cli_example.py",
|
|
"--llm",
|
|
"simulated",
|
|
"--embedding-model",
|
|
"facebook/contriever",
|
|
"--embedding-mode",
|
|
"sentence-transformers",
|
|
"--index-dir",
|
|
temp_dir,
|
|
"--data-dir",
|
|
str(test_data_dir),
|
|
"--query",
|
|
"What is Pride and Prejudice about?",
|
|
]
|
|
|
|
env = os.environ.copy()
|
|
env["HF_HUB_DISABLE_SYMLINKS"] = "1"
|
|
env["TOKENIZERS_PARALLELISM"] = "false"
|
|
|
|
result = subprocess.run(cmd, capture_output=True, text=True, timeout=600, env=env)
|
|
|
|
# Check return code
|
|
assert result.returncode == 0, f"Command failed: {result.stderr}"
|
|
|
|
# Verify output
|
|
output = result.stdout + result.stderr
|
|
assert "Leann index built at" in output or "Using existing index" in output
|
|
assert "This is a simulated answer" in output
|
|
|
|
|
|
@pytest.mark.skipif(not os.environ.get("OPENAI_API_KEY"), reason="OpenAI API key not available")
|
|
def test_main_cli_openai(test_data_dir):
|
|
"""Test main_cli with OpenAI embeddings."""
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
cmd = [
|
|
sys.executable,
|
|
"examples/main_cli_example.py",
|
|
"--llm",
|
|
"simulated", # Use simulated LLM to avoid GPT-4 costs
|
|
"--embedding-model",
|
|
"text-embedding-3-small",
|
|
"--embedding-mode",
|
|
"openai",
|
|
"--index-dir",
|
|
temp_dir,
|
|
"--data-dir",
|
|
str(test_data_dir),
|
|
"--query",
|
|
"What is Pride and Prejudice about?",
|
|
]
|
|
|
|
env = os.environ.copy()
|
|
env["TOKENIZERS_PARALLELISM"] = "false"
|
|
|
|
result = subprocess.run(cmd, capture_output=True, text=True, timeout=600, env=env)
|
|
|
|
assert result.returncode == 0, f"Command failed: {result.stderr}"
|
|
|
|
# Verify cosine distance was used
|
|
output = result.stdout + result.stderr
|
|
assert any(
|
|
msg in output
|
|
for msg in [
|
|
"distance_metric='cosine'",
|
|
"Automatically setting distance_metric='cosine'",
|
|
"Using cosine distance",
|
|
]
|
|
)
|
|
|
|
|
|
@pytest.mark.xfail(sys.platform == "darwin", reason="May fail on macOS due to C++ ABI issues")
|
|
def test_main_cli_error_handling(test_data_dir):
|
|
"""Test main_cli with invalid parameters."""
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
cmd = [
|
|
sys.executable,
|
|
"examples/main_cli_example.py",
|
|
"--llm",
|
|
"invalid_llm_type",
|
|
"--index-dir",
|
|
temp_dir,
|
|
"--data-dir",
|
|
str(test_data_dir),
|
|
]
|
|
|
|
result = subprocess.run(cmd, capture_output=True, text=True, timeout=60)
|
|
|
|
# Should fail with invalid LLM type
|
|
assert result.returncode != 0
|
|
assert "Unknown LLM type" in result.stderr or "invalid_llm_type" in result.stderr
|