← Lesson
Logits and Unembedding
Joeven
Run
Reset
Python loads on first run
import math vocab = ["search", "sql", "finish"] h = [0.8, 0.1] U = [ [1.0, 0.0], [0.2, 1.0], [0.0, 0.3], ] def dot(a, b): return sum(x * y for x, y in zip(a, b)) logits = [dot(h, row) for row in U] m = max(logits) exps = [math.exp(z - m) for z in logits] z = sum(exps) probs = [e / z for e in exps] winner = vocab[max(range(len(logits)), key=lambda i: logits[i])] print("h", h) print("logits", [round(x, 3) for x in logits]) print("probs", [round(p, 3) for p in probs]) print("sum probs", round(sum(probs), 3)) print("argmax", winner) print("argmax from probs", vocab[max(range(len(probs)), key=lambda i: probs[i])])
Run to execute this in your browser. Nothing is sent to a server.