Skip to content

Getting started

Write and run your first eval against a local DuckDB database — no model and no network, so there's nothing to set up beyond installing the package.

Install

uv add evaldata   # core, includes the DuckDB adapter

The shape of an eval

Every eval is the same four pieces:

  • a case — a question (input) and its expected answer, on a platform
  • a solver — the system under test that turns the question into SQL
  • one or more scorers — how the result is judged against expected
  • a platform — the database the SQL runs on

evaldata runs on pytest: a case is a test function decorated with @eval_case, and assert_eval runs the solver's SQL on the platform and asserts the scorers pass.

Write your first eval

Create test_first_eval.py:

import tempfile
from collections.abc import Iterator
from decimal import Decimal
from pathlib import Path

import duckdb
import pytest

from evaldata import CallableSolver, EvalCase, ResultSetEquivalence, assert_eval, eval_case
from evaldata.platforms import duckdb_platform

_DB = Path(tempfile.mkdtemp()) / "shop.duckdb"
platform = duckdb_platform(name="shop", path=str(_DB))


@pytest.fixture(scope="module", autouse=True)
def _seed() -> Iterator[None]:
    con = duckdb.connect(str(_DB))
    con.execute("CREATE TABLE orders (id INTEGER, customer_id INTEGER, amount DECIMAL(10, 2))")
    con.execute("INSERT INTO orders VALUES (1, 1, 10.00), (2, 1, 5.50), (3, 2, 20.00), (4, 2, 7.25)")
    con.close()
    yield


@eval_case(
    input="What is the total order amount?",
    expected={"rows": [{"total": Decimal("42.75")}]},
    platform=platform,
)
def test_total_order_amount(case: EvalCase) -> None:
    solver = CallableSolver(lambda c: "SELECT sum(amount) AS total FROM orders")
    assert_eval(case, solver, scorers=[ResultSetEquivalence()])

Here's what each piece does:

  • @eval_case(...) declares the case and injects a prepared EvalCase as the case fixture. You don't need a conftest.py — installing evaldata registers its pytest plugin.
  • CallableSolver is the simplest solver: a function returning the SQL to run. Here it's fixed SQL so the result is deterministic; in a real eval this is where your model goes (see the guides).
  • ResultSetEquivalence scores by comparing the solver's result rows to expected["rows"].
  • assert_eval ties it together: run the solver, execute its SQL on the platform, score, and fail the test if a scorer fails.

Run it

uv run pytest test_first_eval.py -q

A passing run looks like:

1 passed

It passes because the executed SQL returns 42.75. Change the expected total and rerun to watch it fail — that failure is the regression signal you'd catch in CI when a prompt or model drifts.

The full set of expected types and scorers

The same pattern covers every expected-type and scorer — an untyped result set, a typed one (values and column types), a gold query (compared on its executed result, not its SQL text), and an ExpectationSuite of structural checks:

"""Deterministic text-to-SQL example evals: a `CallableSolver` returning fixed SQL."""

import tempfile
from collections.abc import Iterator
from decimal import Decimal
from pathlib import Path

import duckdb
import pytest

from evaldata import (
    CallableSolver,
    EvalCase,
    ExpectationSuiteScorer,
    ResultSetEquivalence,
    assert_eval,
    eval_case,
)
from evaldata.platforms import duckdb_platform

_DB_PATH = Path(tempfile.mkdtemp(prefix="evaldata_ex01_")) / "shop.duckdb"
_PLATFORM = duckdb_platform(name="examples-deterministic", path=str(_DB_PATH))


@pytest.fixture(scope="module", autouse=True)
def _seed_db() -> Iterator[None]:
    con = duckdb.connect(str(_DB_PATH))
    con.execute("CREATE TABLE customers (id INTEGER, name VARCHAR, country VARCHAR)")
    con.execute("INSERT INTO customers VALUES (1, 'Ada', 'GB'), (2, 'Bo', 'US'), (3, 'Cy', 'US')")
    con.execute("CREATE TABLE orders (id INTEGER, customer_id INTEGER, amount DECIMAL(10, 2))")
    con.execute("INSERT INTO orders VALUES (1, 1, 10.00), (2, 1, 5.50), (3, 2, 20.00), (4, 2, 7.25)")
    con.close()
    yield


# Untyped result set: compare values only (no column types asserted).
@eval_case(
    input="What is the total order amount?",
    expected={"rows": [{"total": Decimal("42.75")}]},
    platform=_PLATFORM,
)
def test_untyped_result_set(case: EvalCase) -> None:
    """Compare result values only, asserting no column types."""
    solver = CallableSolver(lambda c: "SELECT sum(amount) AS total FROM orders")
    assert_eval(case, solver, scorers=[ResultSetEquivalence()])


# Typed result set: assert the column types alongside the values. Fails if the right value
# comes back with the wrong type (e.g. DOUBLE or VARCHAR).
@eval_case(
    input="What is the total order amount?",
    expected={
        "rows": [{"total": Decimal("42.75")}],
        "schema": [{"name": "total", "type": "DECIMAL(38, 2)"}],
    },
    platform=_PLATFORM,
)
def test_typed_result_set(case: EvalCase) -> None:
    """Compare result values plus a column-type assertion."""
    solver = CallableSolver(lambda c: "SELECT sum(amount) AS total FROM orders")
    assert_eval(case, solver, scorers=[ResultSetEquivalence()])


# Gold query: the reference query's executed RESULT is the expected answer (execution
# accuracy). The comparison is on the executed result, not the SQL text, so any query that
# returns the same rows passes.
@eval_case(
    input="What is the total order amount per customer?",
    expected={
        "kind": "gold_query",
        "sql": ("SELECT customer_id, sum(amount) AS total FROM orders GROUP BY customer_id"),
    },
    platform=_PLATFORM,
)
def test_gold_query(case: EvalCase) -> None:
    """Score against a reference query's executed result (execution accuracy)."""
    solver = CallableSolver(
        lambda c: "SELECT customer_id, sum(amount) AS total FROM orders GROUP BY 1 ORDER BY customer_id DESC"
    )
    assert_eval(case, solver, scorers=[ResultSetEquivalence()])


# Expectation suite: assert structural properties of the result.
@eval_case(
    input="List every customer with their id and name.",
    expected={
        "kind": "expectation_suite",
        "expectations": [
            {"kind": "row_count", "exact": 3},
            {"kind": "not_null", "column": "id"},
            {"kind": "unique", "column": "id"},
        ],
    },
    platform=_PLATFORM,
)
def test_expectation_suite(case: EvalCase) -> None:
    """Assert structural properties of the result."""
    solver = CallableSolver(lambda c: "SELECT id, name FROM customers")
    assert_eval(case, solver, scorers=[ExpectationSuiteScorer()])

This is the runnable example from examples/01_deterministic/ in the repo.

Recap

  • An eval is a case + a solver + scorers, run on a platform.
  • A case is a @eval_case-decorated test; assert_eval runs the solver's SQL and scores it.
  • CallableSolver runs fixed SQL — swap in a model with PromptSolver to test text-to-SQL.

Next steps