← Lesson
Why Evals Exist
Joeven
Run
Reset
Python loads on first run
def goal_satisfied(trace, expected_tool, expected_contains): tools = [e["tool"] for e in trace if e["kind"] == "tool"] finals = [e["text"] for e in trace if e["kind"] == "final"] if not finals: return {"ok": False, "why": "no final"} if expected_tool not in tools: return {"ok": False, "why": "missing tool " + expected_tool} if expected_contains not in finals[-1]: return {"ok": False, "why": "answer missing fact"} return {"ok": True, "why": "pass"} good = [ {"kind": "tool", "tool": "search_kb", "args": {"q": "refund"}}, {"kind": "final", "text": "Refunds take 5-7 business days."}, ] bad = [ {"kind": "final", "text": "Refunds take 5-7 business days."}, ] print("GOOD", goal_satisfied(good, "search_kb", "5-7")) print("HALLUCINATED-BUT-PRETTY", goal_satisfied(bad, "search_kb", "5-7")) print("eval saw the missing tool; a chatbot metric would not")
Run to execute this in your browser. Nothing is sent to a server.