tomevault-io Testing Patterns
Pytest Configuration
pyproject.toml
[tool.pytest.ini_options]
testpaths = ["tests"]
asyncio_mode = "auto"
markers = [
"slow: marks tests as slow (deselect with '-m \"not slow\"')",
"integration: marks integration tests",
"e2e: marks end-to-end tests",
"llm: marks tests that call LLM APIs (real or mocked)",
]
filterwarnings = [
"error",
"ignore::DeprecationWarning:some_library.*",
]
Coverage Configuration
[tool.coverage.run]
branch = true
source = ["src"]
omit = ["tests/*", "*/migrations/*"]
[tool.coverage.report]
fail_under = 80
show_missing = true
exclude_lines = [
"pragma: no cover",
"if TYPE_CHECKING:",
"if __name__ == .__main__.",
"@overload",
]
Fixture Patterns
Scoped Fixtures
import pytest
@pytest.fixture(scope="session")
def db_engine():
"""One engine for the entire test session."""
engine = create_engine("sqlite:///:memory:")
yield engine
engine.dispose()
@pytest.fixture(scope="function")
def db_session(db_engine):
"""Fresh transaction per test, rolled back after."""
connection = db_engine.connect()
transaction = connection.begin()
session = Session(bind=connection)
yield session
session.close()
transaction.rollback()
connection.close()
Factory Fixtures
@pytest.fixture
def make_user(db_session):
"""Factory fixture for creating test users."""
created = []
def _make_user(name="test", email=None, **kwargs):
email = email or f"{name}@test.com"
user = User(name=name, email=email, **kwargs)
db_session.add(user)
db_session.flush()
created.append(user)
return user
yield _make_user
for user in created:
db_session.delete(user)
Tmp Path Fixtures
@pytest.fixture
def config_dir(tmp_path):
"""Temporary config directory with default files."""
config = tmp_path / "config"
config.mkdir()
(config / "settings.yaml").write_text("debug: true\n")
return config
Mocking Patterns
LLM API Mocking
@pytest.fixture
def mock_llm_response(mocker):
"""Mock LLM API with realistic response structure."""
def _mock(content="test response", tokens=50, model="gpt-4"):
mock = mocker.patch("litellm.completion")
mock.return_value = MockResponse(
choices=[MockChoice(message=MockMessage(content=content))],
usage=MockUsage(
prompt_tokens=tokens,
completion_tokens=tokens * 2,
total_tokens=tokens * 3,
),
model=model,
)
return mock
return _mock
def test_llm_chain_returns_parsed(mock_llm_response):
mock_llm_response(content='{"name": "test", "score": 42}')
result = my_chain.invoke({"query": "test"})
assert result.name == "test"
assert result.score == 42
Structured Output Mocking
@pytest.fixture
def mock_structured_output(mocker):
"""Mock with_structured_output for Pydantic models."""
def _mock(model_class, **field_values):
instance = model_class(**field_values)
mock_chain = mocker.MagicMock()
mock_chain.invoke.return_value = instance
return mock_chain
return _mock
Environment Mocking
def test_loads_api_key_from_env(monkeypatch):
monkeypatch.setenv("OPENAI_API_KEY", "test-key")
config = load_config()
assert config.api_key == "test-key"
def test_raises_without_api_key(monkeypatch):
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
with pytest.raises(ConfigError, match="OPENAI_API_KEY"):
load_config()
Async Testing
import pytest
from unittest.mock import AsyncMock
@pytest.mark.asyncio
async def test_async_pipeline():
"""Test async LLM pipeline."""
mock_provider = AsyncMock()
mock_provider.acomplete.return_value = "test response"
result = await pipeline.arun(provider=mock_provider, query="test")
assert result.status == "success"
mock_provider.acomplete.assert_awaited_once()
@pytest.mark.asyncio
async def test_timeout_handling():
"""Verify timeout raises appropriately."""
import asyncio
async def slow_response(*args, **kwargs):
await asyncio.sleep(10)
mock_provider = AsyncMock(side_effect=slow_response)
with pytest.raises(asyncio.TimeoutError):
await asyncio.wait_for(
pipeline.arun(provider=mock_provider, query="test"),
timeout=1.0,
)
Property-Based Testing
from hypothesis import given, settings, strategies as st
@given(st.text(min_size=1, max_size=100))
def test_tokenizer_roundtrip(text):
"""Encoding then decoding should return original text."""
tokens = tokenizer.encode(text)
decoded = tokenizer.decode(tokens)
assert decoded == text
@given(st.lists(st.integers(min_value=0, max_value=100), min_size=1))
@settings(max_examples=500)
def test_chunk_sizes_sum_to_total(sizes):
"""Chunking preserves total token count."""
text = " ".join(["word"] * sum(sizes))
chunks = chunk_text(text, max_tokens=50)
total = sum(len(c.split()) for c in chunks)
assert total == sum(sizes)
Parametrize Patterns
@pytest.mark.parametrize("model,expected_provider", [
("gpt-4", "openai"),
("claude-3-sonnet", "anthropic"),
("ollama/llama3", "ollama"),
("unknown-model", None),
])
def test_model_routing(model, expected_provider):
provider = resolve_provider(model)
assert provider == expected_provider
@pytest.mark.parametrize("input_text,expected_error", [
("", "Input cannot be empty"),
("x" * 10001, "Input exceeds maximum length"),
("<script>alert(1)</script>", "Input contains disallowed HTML"),
])
def test_input_validation_rejects_bad_input(input_text, expected_error):
with pytest.raises(ValidationError, match=expected_error):
validate_input(input_text)
Test Anti-Patterns to Avoid
- Testing implementation details — Test behavior, not internal state
- Fragile assertions — Don't assert on exact log messages or timestamps
- Test interdependence — Each test must run independently and in any order
- Over-mocking — If you mock everything, you test nothing
- Missing edge cases — Empty inputs, None values, boundary conditions
- Ignoring flaky tests — Fix or quarantine, never ignore
- Testing third-party code — Mock the boundary, don't test their library
CI Integration
# .github/workflows/test.yml (excerpt)
jobs:
test:
steps:
- run: uv run pytest tests/unit -x --cov --cov-report=xml
- run: uv run pytest tests/integration -x -m "not slow"
- run: uv run pytest tests/e2e -x --timeout=120
Converted and distributed by TomeVault — claim your Tome and manage your conversions.