Set goals

Goals let you define pass/fail criteria on evaluator scores. When a goal is set, the SDK displays PASS or FAIL verdicts in the terminal output alongside each score.

A goal is a pure pass/fail gate. Which way is better (and how scores are normalized) is metric semantics you set on the Evaluator itself — see Direction and normalization below.

Goal types

Method	Meaning
`Goal.gte(0.8)`	Score must be `>= 0.8`
`Goal.lte(0.1)`	Score must be `<= 0.1`
`Goal.between(0.2, 0.8)`	Score must be in `[0.2, 0.8]`

All goal methods accept an optional metric parameter ("avg", "min", "max", "std") to target a specific statistic instead of the default average.

Basic usage

from mistralai.evaluations import Evaluator, Goal

Evaluator(
    name="accuracy",
    description="1 if the expected answer appears in the output.",
    scorer=accuracy_scorer,
    goal=Goal.gte(0.8),
)

from mistralai.evaluations import Evaluator, Goal

Evaluator(
    name="accuracy",
    description="1 if the expected answer appears in the output.",
    scorer=accuracy_scorer,
    goal=Goal.gte(0.8),
)

Per-generation and aggregate goals

Evaluators support two goal levels:

goal: evaluated against each individual generation score.
aggregate_goal: evaluated against the aggregated result across all generations.

Evaluator(
    name="accuracy",
    description="1 if the expected answer appears in the output.",
    scorer=accuracy_scorer,
    goal=Goal.gte(0.7),           # each generation must score >= 0.7
    aggregate_goal=Goal.gte(0.9), # overall average must be >= 0.9
)

Evaluator(
    name="accuracy",
    description="1 if the expected answer appears in the output.",
    scorer=accuracy_scorer,
    goal=Goal.gte(0.7),           # each generation must score >= 0.7
    aggregate_goal=Goal.gte(0.9), # overall average must be >= 0.9
)

When you call run.show():

Run level: shows whether the aggregate passes the aggregate_goal (falls back to goal on the average if no aggregate is set).
Record level (with num_generations > 1): shows how many generations passed the goal (for example, "2/3 PASS"), with aggregate verdict when applicable.
Generation level: shows PASS/FAIL for each individual score against goal.

Goals on run evaluators

Run evaluators also support goals:

from mistralai.evaluations import RunEvaluator, Goal

RunEvaluator(
    name="f1",
    description="Harmonic mean of precision and recall across all records.",
    scorer=f1_scorer,
    goal=Goal.gte(0.85),
)

from mistralai.evaluations import RunEvaluator, Goal

RunEvaluator(
    name="f1",
    description="Harmonic mean of precision and recall across all records.",
    scorer=f1_scorer,
    goal=Goal.gte(0.85),
)

Direction and normalization

A goal only gates pass/fail. To express which way is better and the scale a metric lives on, set these fields on the Evaluator:

Field	Type	Description
`direction`	`"maximize"` \| `"minimize"`	Which way is better. Inferred from the goal when unset (`gte` → `maximize`, `lte` → `minimize`, otherwise `maximize`).
`min_value` / `max_value`	`float`	Min-max normalization window. Set both or neither; `min_value` must be `< max_value`.
`weight`	`float`	Optimization pressure (default `1.0`). Set `weight=0` to make the metric a pure constraint: its goal still gates, but it is excluded from the optimization objective.

Evaluator(
    name="latency_ms",
    description="End-to-end response time in milliseconds.",
    scorer=latency_scorer,
    goal=Goal.lte(1000),     # gate + infers direction="minimize"
    min_value=200,
    max_value=2000,          # 850 ms normalizes to ~0.64 (lower is better)
)

Evaluator(
    name="accuracy",
    description="1 if the expected answer appears in the output.",
    scorer=accuracy_scorer,  # no goal → direction defaults to "maximize", window to [0, 1]
)

Evaluator(
    name="latency_ms",
    description="End-to-end response time in milliseconds.",
    scorer=latency_scorer,
    goal=Goal.lte(1000),     # gate + infers direction="minimize"
    min_value=200,
    max_value=2000,          # 850 ms normalizes to ~0.64 (lower is better)
)

Evaluator(
    name="accuracy",
    description="1 if the expected answer appears in the output.",
    scorer=accuracy_scorer,  # no goal → direction defaults to "maximize", window to [0, 1]
)

Full example

import asyncio
import os

from mistralai.evaluations import (
    Evaluation, Evaluator, Goal, Mistral, Project,
    RunEvaluator, RunEvaluatorContext, ScorerContext, System, TaskContext,
)

client = Mistral(api_key=os.environ["MISTRAL_API_KEY"])

dataset = [
    {"prompt": "What is 2+2?", "expected": "4"},
    {"prompt": "Capital of France?", "expected": "Paris"},
    {"prompt": "Largest planet?", "expected": "Jupiter"},
]

async def task(ctx: TaskContext) -> str:
    r = await client.chat.complete_async(
        model=str(ctx.system.params["model"]),
        messages=[{"role": "user", "content": ctx.input_record["prompt"]}],
    )
    return str(r.choices[0].message.content)

def accuracy_scorer(ctx: ScorerContext) -> int:
    return 1 if ctx.input_record["expected"].lower() in str(ctx.output).lower() else 0

def accuracy_gate(ctx: RunEvaluatorContext):
    return ctx.statistics["accuracy"].avg

async def main():
    run = await client.evaluation.run(
        project=Project(name="QA Pipeline"),
        evaluation=Evaluation(name="Accuracy with Goals"),
        system=System(name="mistral-small", params={"model": "mistral-small-latest"}),
        dataset=dataset,
        task=task,
        evaluators=[
            Evaluator(
                name="accuracy",
                description="1 if the expected answer appears in the output.",
                scorer=accuracy_scorer,
                goal=Goal.gte(0.8),
                aggregate_goal=Goal.gte(0.9),
            ),
        ],
        run_evaluators=[
            RunEvaluator(
                name="accuracy_avg",
                description="Average accuracy across all records.",
                scorer=accuracy_gate,
                goal=Goal.gte(0.85),
            ),
        ],
        num_generations=3,
    )
    run.show(level="generations")

asyncio.run(main())

import asyncio
import os

from mistralai.evaluations import (
    Evaluation, Evaluator, Goal, Mistral, Project,
    RunEvaluator, RunEvaluatorContext, ScorerContext, System, TaskContext,
)

client = Mistral(api_key=os.environ["MISTRAL_API_KEY"])

dataset = [
    {"prompt": "What is 2+2?", "expected": "4"},
    {"prompt": "Capital of France?", "expected": "Paris"},
    {"prompt": "Largest planet?", "expected": "Jupiter"},
]

async def task(ctx: TaskContext) -> str:
    r = await client.chat.complete_async(
        model=str(ctx.system.params["model"]),
        messages=[{"role": "user", "content": ctx.input_record["prompt"]}],
    )
    return str(r.choices[0].message.content)

def accuracy_scorer(ctx: ScorerContext) -> int:
    return 1 if ctx.input_record["expected"].lower() in str(ctx.output).lower() else 0

def accuracy_gate(ctx: RunEvaluatorContext):
    return ctx.statistics["accuracy"].avg

async def main():
    run = await client.evaluation.run(
        project=Project(name="QA Pipeline"),
        evaluation=Evaluation(name="Accuracy with Goals"),
        system=System(name="mistral-small", params={"model": "mistral-small-latest"}),
        dataset=dataset,
        task=task,
        evaluators=[
            Evaluator(
                name="accuracy",
                description="1 if the expected answer appears in the output.",
                scorer=accuracy_scorer,
                goal=Goal.gte(0.8),
                aggregate_goal=Goal.gte(0.9),
            ),
        ],
        run_evaluators=[
            RunEvaluator(
                name="accuracy_avg",
                description="Average accuracy across all records.",
                scorer=accuracy_gate,
                goal=Goal.gte(0.85),
            ),
        ],
        num_generations=3,
    )
    run.show(level="generations")

asyncio.run(main())