Skip to content

Evaluate a local Ollama model

Run a self-hosted Ollama model as the system under test, scoring its generated SQL against expected rows on a local DuckDB. The solver is a PromptSolver that calls the model through litellm.

Prerequisites

uv add "evaldata[litellm]"     # PromptSolver + litellm
ollama pull qwen2.5-coder:1.5b # any model you like; a coder model scores best

Write the eval

The solver is a PromptSolver(model=...) with temperature=0 for stable generation. The questions ask for plain column selections, so the output column names come straight from the table — which keeps exact-row ResultSetEquivalence scoring reliable.

Create test_local_ai.py:

"""Local-AI text-to-SQL example evals: a `PromptSolver` calling a local Ollama model."""

import os
import tempfile
from collections.abc import Iterator
from pathlib import Path

import duckdb
import pytest

from evaldata import EvalCase, ResultSetEquivalence, assert_eval, eval_case
from evaldata.platforms import duckdb_platform
from evaldata.solvers import PromptSolver

pytestmark = pytest.mark.e2e

_DB_PATH = Path(tempfile.mkdtemp(prefix="evaldata_ex02_")) / "shop.duckdb"
_PLATFORM = duckdb_platform(name="examples-local-ai", path=str(_DB_PATH))
_MODEL = os.environ.get("EVALDATA_LOCAL_MODEL", "")


@pytest.fixture(scope="module", autouse=True)
def _require_local_model() -> None:
    # Fail loudly (never skip) when these examples run without a configured local model.
    if not _MODEL:  # pragma: no cover
        msg = "set EVALDATA_LOCAL_MODEL to your local model's id, e.g. ollama_chat/qwen2.5-coder:1.5b"
        raise RuntimeError(msg)


@pytest.fixture(scope="module", autouse=True)
def _seed_db() -> Iterator[None]:
    con = duckdb.connect(str(_DB_PATH))
    con.execute("CREATE TABLE customers (id INTEGER, name VARCHAR, country VARCHAR)")
    con.execute("INSERT INTO customers VALUES (1, 'Ada', 'GB'), (2, 'Bo', 'US'), (3, 'Cy', 'US')")
    con.execute("CREATE TABLE orders (id INTEGER, customer_id INTEGER, amount DECIMAL(10, 2))")
    con.execute("INSERT INTO orders VALUES (1, 1, 10.00), (2, 1, 5.50), (3, 2, 20.00), (4, 2, 7.25)")
    con.close()
    yield


@eval_case(
    input="Return the id column of every row in the orders table, one row per order.",
    expected={"rows": [{"id": 1}, {"id": 2}, {"id": 3}, {"id": 4}]},
    platform=_PLATFORM,
)
def test_select_order_ids(case: EvalCase) -> None:
    """Local model selects every order id; scored on exact rows."""
    solver = PromptSolver(model=_MODEL, timeout=120, temperature=0)
    assert_eval(case, solver, scorers=[ResultSetEquivalence()])


@eval_case(
    input="Return the name column of every customer whose country is 'US', one row per customer.",
    expected={"rows": [{"name": "Bo"}, {"name": "Cy"}]},
    platform=_PLATFORM,
)
def test_select_us_customer_names(case: EvalCase) -> None:
    """Local model selects US customer names; scored on exact rows."""
    solver = PromptSolver(model=_MODEL, timeout=120, temperature=0)
    assert_eval(case, solver, scorers=[ResultSetEquivalence()])

The example reads the model id from EVALDATA_LOCAL_MODEL and passes it to PromptSolver(model=...) — that's just the model argument, so you can pass a literal instead. If Ollama runs somewhere other than the default, set OLLAMA_API_BASE (litellm reads it).

Run it

uv run pytest test_local_ai.py -q

A failure here means the model produced SQL whose result didn't match the expected rows — exactly the regression you'd want CI to catch when you change the model or prompt.

Run it from a clone

This is the bundled examples/02_local_ai/ example. If you've cloned the repo, run it directly with uv run pytest examples/02_local_ai — no copying needed.

Next steps