""" Integration tests for LLM provider configuration (LLM-02). Tests verify that: 1. The LiteLLM Router model_list contains entries for all three providers (Ollama/fast, Anthropic/quality, OpenAI/quality). 2. A request with model="fast" routes to the Ollama configuration. 3. A request with model="quality" routes to an Anthropic or OpenAI configuration. 4. Provider entries reference the correct model identifiers from CLAUDE.md. These tests inspect the router configuration directly and mock acompletion to verify routing without live API calls. """ from __future__ import annotations from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi.testclient import TestClient from llm_pool.main import app from llm_pool.router import _model_list, llm_router client = TestClient(app) def _make_completion_response(content: str = "test") -> MagicMock: response = MagicMock() response.choices = [MagicMock()] response.choices[0].message.content = content return response class TestProviderConfiguration: """LLM-02: Provider configuration — all three providers are present and correct.""" def test_model_list_has_three_entries(self) -> None: """The model_list must have exactly three entries (fast, quality x2).""" assert len(_model_list) == 3 def test_fast_group_present_in_model_list(self) -> None: """The 'fast' model group must exist in the model_list.""" fast_entries = [m for m in _model_list if m["model_name"] == "fast"] assert len(fast_entries) >= 1, "No 'fast' model group found in model_list" def test_quality_group_present_in_model_list(self) -> None: """The 'quality' model group must have at least two entries (Anthropic + OpenAI).""" quality_entries = [m for m in _model_list if m["model_name"] == "quality"] assert len(quality_entries) >= 2, "Expected at least 2 'quality' entries (Anthropic + OpenAI)" def test_fast_group_uses_ollama_model(self) -> None: """The fast group must route to an ollama/* model.""" fast_entries = [m for m in _model_list if m["model_name"] == "fast"] assert fast_entries, "No fast entry found" ollama_models = [ e for e in fast_entries if e["litellm_params"]["model"].startswith("ollama/") ] assert ollama_models, f"Fast group does not use an ollama model: {fast_entries}" def test_fast_group_has_ollama_api_base(self) -> None: """The fast group entry must specify an api_base pointing to Ollama.""" fast_entries = [m for m in _model_list if m["model_name"] == "fast"] for entry in fast_entries: params = entry["litellm_params"] assert "api_base" in params, f"Fast group entry missing api_base: {entry}" def test_quality_group_has_anthropic_entry(self) -> None: """Quality group must include an anthropic/* model.""" quality_entries = [m for m in _model_list if m["model_name"] == "quality"] anthropic_entries = [ e for e in quality_entries if e["litellm_params"]["model"].startswith("anthropic/") ] assert anthropic_entries, f"No Anthropic entry in quality group: {quality_entries}" def test_quality_group_has_openai_entry(self) -> None: """Quality group must include an openai/* model as the fallback.""" quality_entries = [m for m in _model_list if m["model_name"] == "quality"] openai_entries = [ e for e in quality_entries if e["litellm_params"]["model"].startswith("openai/") ] assert openai_entries, f"No OpenAI entry in quality group: {quality_entries}" def test_anthropic_model_is_claude_sonnet(self) -> None: """Anthropic entry must use the correct model from CLAUDE.md architecture.""" quality_entries = [m for m in _model_list if m["model_name"] == "quality"] anthropic_entry = next( (e for e in quality_entries if e["litellm_params"]["model"].startswith("anthropic/")), None, ) assert anthropic_entry is not None model = anthropic_entry["litellm_params"]["model"] assert "claude-sonnet" in model, f"Expected claude-sonnet model, got: {model}" def test_openai_model_is_gpt4o(self) -> None: """OpenAI entry must use gpt-4o as specified in architecture.""" quality_entries = [m for m in _model_list if m["model_name"] == "quality"] openai_entry = next( (e for e in quality_entries if e["litellm_params"]["model"].startswith("openai/")), None, ) assert openai_entry is not None model = openai_entry["litellm_params"]["model"] assert "gpt-4o" in model, f"Expected gpt-4o model, got: {model}" def test_fast_request_calls_acompletion_with_fast_model(self) -> None: """A fast model request must invoke acompletion with model='fast'.""" mock_response = _make_completion_response("ollama says hi") with patch("llm_pool.router.llm_router.acompletion", new_callable=AsyncMock) as mock_complete: mock_complete.return_value = mock_response response = client.post( "/complete", json={ "model": "fast", "messages": [{"role": "user", "content": "Hi"}], "tenant_id": "tenant-fast", }, ) assert response.status_code == 200 call_kwargs = mock_complete.call_args assert call_kwargs is not None called_model = call_kwargs.kwargs.get("model") or (call_kwargs.args[0] if call_kwargs.args else None) assert called_model == "fast" def test_quality_request_calls_acompletion_with_quality_model(self) -> None: """A quality model request must invoke acompletion with model='quality'.""" mock_response = _make_completion_response("anthropic says hi") with patch("llm_pool.router.llm_router.acompletion", new_callable=AsyncMock) as mock_complete: mock_complete.return_value = mock_response response = client.post( "/complete", json={ "model": "quality", "messages": [{"role": "user", "content": "Hi"}], "tenant_id": "tenant-quality", }, ) assert response.status_code == 200 call_kwargs = mock_complete.call_args assert call_kwargs is not None called_model = call_kwargs.kwargs.get("model") or (call_kwargs.args[0] if call_kwargs.args else None) assert called_model == "quality" def test_router_fallback_config_quality_falls_to_fast(self) -> None: """The Router fallbacks config must specify quality -> fast cross-group fallback.""" # Access the Router's fallbacks attribute fallbacks = getattr(llm_router, "fallbacks", None) assert fallbacks is not None, "Router has no fallbacks configured" # Find the quality -> fast fallback entry quality_fallback = None for fb in fallbacks: if isinstance(fb, dict) and "quality" in fb: quality_fallback = fb break assert quality_fallback is not None, ( f"No quality->fast fallback found. Current fallbacks: {fallbacks}" ) fallback_targets = quality_fallback["quality"] assert "fast" in fallback_targets, ( f"Quality fallback does not target 'fast' group: {fallback_targets}" )