Joeven — Learn Autonomous AI Agents

← LessonAccuracy, Precision, Recall, F1Joeven

Python loads on first run

def scores(tp, fp, tn, fn):
    all_n = tp + fp + tn + fn
    acc = (tp + tn) / all_n
    prec = tp / (tp + fp) if (tp + fp) else 0.0
    rec = tp / (tp + fn) if (tp + fn) else 0.0
    f1 = (2 * prec * rec / (prec + rec)) if (prec + rec) else 0.0
    return acc, prec, rec, f1

# retriever: gold chunk in top-k?
print("balanced", [round(x, 3) for x in scores(40, 10, 40, 10)])
print("always no", [round(x, 3) for x in scores(0, 0, 80, 20)])
print("always yes", [round(x, 3) for x in scores(20, 80, 0, 0)])

# cutoff on cosine: each row is (score, gold)
rows = [
    (0.91, True),
    (0.72, True),
    (0.40, False),
    (0.33, True),
    (0.20, False),
    (0.15, False),
]

def confusion(cutoff):
    tp = fp = tn = fn = 0
    for score, gold in rows:
        pred = score >= cutoff
        if pred and gold:
            tp += 1
        elif pred and not gold:
            fp += 1
        elif (not pred) and (not gold):
            tn += 1
        else:
            fn += 1
    return tp, fp, tn, fn

for c in [0.3, 0.5, 0.8]:
    tp, fp, tn, fn = confusion(c)
    print("cutoff", c, "counts", (tp, fp, tn, fn), "F1", round(scores(tp, fp, tn, fn)[3], 3))

Run to execute this in your browser. Nothing is sent to a server.