Joeven — Learn Autonomous AI Agents

Python loads on first run

GOLD = [
    {"q": "refund days", "gold": {"billing"}, "refuse": False},
    {"q": "OOM worker", "gold": {"runbook"}, "refuse": False},
    {"q": "equine dental", "gold": set(), "refuse": True},
]

CHUNKS = {
    "billing": "Refunds take 5-7 days. Never cash.",
    "runbook": "Runner OOM: raise worker memory.",
    "menu": "Pizza on Fridays.",
}

def retrieve(q, k=1):
    qw = set(q.lower().split())
    ranked = sorted(CHUNKS.items(), key=lambda kv: -len(qw & set(kv[1].lower().split())))
    return [cid for cid, _ in ranked[:k]]

def eval_set(k=1, tau_words=1):
    hits = 0
    n_has_gold = 0
    refuse_ok = 0
    for row in GOLD:
        ids = retrieve(row["q"], k=k)
        qw = set(row["q"].lower().split())
        best = max(len(qw & set(CHUNKS[i].lower().split())) for i in ids)
        refused = best < tau_words
        if row["refuse"]:
            refuse_ok += int(refused)
        else:
            n_has_gold += 1
            hits += int(len(set(ids) & row["gold"]) > 0)
    return {
        "recall_at_k": hits / n_has_gold,
        "refuse_accuracy": refuse_ok / sum(1 for r in GOLD if r["refuse"]),
    }

print(eval_set(k=1))
print(eval_set(k=2))

Run to execute this in your browser. Nothing is sent to a server.