Skip to content

Evaluate a hosted model

Run a hosted, API-served model and score its generated SQL. The solver is a PromptSolver that calls the model through litellm; set the model id and provider credentials in the environment.

Prerequisites

uv add "evaldata[litellm]"

Write the eval

The solver is a PromptSolver(model=...). Create test_hosted_ai.py:

"""Hosted-AI text-to-SQL example evals: a `PromptSolver` calling a hosted model."""

import os
import tempfile
from collections.abc import Iterator
from pathlib import Path

import duckdb
import pytest

from evaldata import EvalCase, ResultSetEquivalence, assert_eval, eval_case
from evaldata.platforms import duckdb_platform
from evaldata.solvers import PromptSolver

_DB_PATH = Path(tempfile.mkdtemp(prefix="evaldata_ex03_")) / "shop.duckdb"
_PLATFORM = duckdb_platform(name="examples-hosted-ai", path=str(_DB_PATH))
_MODEL = os.getenv("EVALDATA_HOSTED_MODEL", "openai/gpt-4o-mini")


@pytest.fixture(scope="module", autouse=True)
def _seed_db() -> Iterator[None]:
    con = duckdb.connect(str(_DB_PATH))
    con.execute("CREATE TABLE customers (id INTEGER, name VARCHAR, country VARCHAR)")
    con.execute("INSERT INTO customers VALUES (1, 'Ada', 'GB'), (2, 'Bo', 'US'), (3, 'Cy', 'US')")
    con.execute("CREATE TABLE orders (id INTEGER, customer_id INTEGER, amount DECIMAL(10, 2))")
    con.execute("INSERT INTO orders VALUES (1, 1, 10.00), (2, 1, 5.50), (3, 2, 20.00), (4, 2, 7.25)")
    con.close()
    yield


@eval_case(
    input="How many orders are there? Name the output column order_count.",
    expected={"rows": [{"order_count": 4}]},
    platform=_PLATFORM,
)
def test_order_count(case: EvalCase) -> None:
    """Hosted model counts orders; scored on exact rows."""
    solver = PromptSolver(model=_MODEL, timeout=120, temperature=0)
    assert_eval(case, solver, scorers=[ResultSetEquivalence()])


@eval_case(
    input=("How many distinct customers placed an order? Name the output column customer_count."),
    expected={"rows": [{"customer_count": 2}]},
    platform=_PLATFORM,
)
def test_distinct_customers(case: EvalCase) -> None:
    """Hosted model counts distinct ordering customers; scored on exact rows."""
    solver = PromptSolver(model=_MODEL, timeout=120, temperature=0)
    assert_eval(case, solver, scorers=[ResultSetEquivalence()])

The example reads the model id from EVALDATA_HOSTED_MODEL and passes it to PromptSolver(model=...). You can pass a model id directly instead. litellm reads provider credentials from the environment, for example OPENAI_API_KEY.

Run it against the live model

uv run pytest test_hosted_ai.py -q

Use fixed responses in CI

Add a conftest.py next to your test that returns the structured {"sql": ...} response the solver expects for each question:

"""Mocked model replies for the hosted-AI example, so it runs without a key or network."""

from typing import Any

import litellm
import pytest

_SQL_BY_QUESTION = {
    "order_count": "SELECT count(*) AS order_count FROM orders",
    "customer_count": "SELECT count(DISTINCT customer_id) AS customer_count FROM orders",
}


def _mock_sql(messages: list[dict[str, Any]]) -> str:
    """Pick the correct SQL for a request by matching its question text.

    Args:
        messages: The chat messages of the request, whose prompt names the expected
            output column.

    Returns:
        The raw SQL reply for the matched question.

    Raises:
        AssertionError: When no known question is present in the messages.
    """
    prompt = " ".join(m.get("content", "") for m in messages)
    for marker, sql in _SQL_BY_QUESTION.items():
        if marker in prompt:
            return sql
    msg = f"no mock SQL for prompt: {prompt!r}"  # pragma: no cover
    raise AssertionError(msg)  # pragma: no cover


@pytest.fixture(autouse=True)
def _mock_completion(monkeypatch: pytest.MonkeyPatch) -> None:
    """Patch model calls to return a deterministic reply per question, with no network."""
    real_completion = litellm.completion

    def fake(**kwargs: Any) -> Any:
        return real_completion(**kwargs, mock_response=_mock_sql(kwargs["messages"]))

    monkeypatch.setattr("litellm.completion", fake)

conftest.py makes the test use fixed responses:

uv run pytest test_hosted_ai.py -q

Remove conftest.py to call the hosted model.

Run it from a clone

This is the bundled examples/03_hosted_ai/ example. If you've cloned the repo, run it with uv run pytest examples/03_hosted_ai. It includes fixed responses.

Next steps