← Lesson
Cross-Entropy, KL, and Perplexity
Joeven
Run
Reset
Python loads on first run
import math def entropy(ps): return -sum(p * math.log(p, 2) for p in ps if p > 0) def cross_entropy(p, q): total = 0.0 for pi, qi in zip(p, q): if pi == 0: continue qi = max(qi, 1e-12) total -= pi * math.log(qi, 2) return total def kl(p, q): return cross_entropy(p, q) - entropy(p) def perplexity_bits(h): return 2 ** h true = [0, 1, 0] good = [0.05, 0.90, 0.05] bad = [0.40, 0.20, 0.40] flat = [1 / 3, 1 / 3, 1 / 3] print("H(true) one-hot", entropy([1.0])) print("CE good", round(cross_entropy(true, good), 3), "ppl", round(perplexity_bits(cross_entropy(true, good)), 3)) print("CE bad ", round(cross_entropy(true, bad), 3), "ppl", round(perplexity_bits(cross_entropy(true, bad)), 3)) print("CE flat", round(cross_entropy(true, flat), 3)) p = [0.7, 0.3] q = [0.6, 0.4] print("KL(p||q)", round(kl(p, q), 3)) print("KL(q||p)", round(kl(q, p), 3))
Run to execute this in your browser. Nothing is sent to a server.