← Lesson
Residuals and LayerNorm
Joeven
Run
Reset
Python loads on first run
def add(a, b): return [x + y for x, y in zip(a, b)] def rmsnorm(v, eps=1e-5): ms = sum(x * x for x in v) / len(v) scale = (ms + eps) ** 0.5 return [x / scale for x in v] def mag(v): return sum(x * x for x in v) ** 0.5 x = [3.0, -1.0, 2.0] delta = [0.5, 0.0, -0.5] y = add(x, delta) print("x", x, "mag", round(mag(x), 3)) print("x + delta", y, "mag", round(mag(y), 3)) nx = rmsnorm(x) ny = rmsnorm(y) print("rmsnorm(x)", [round(v, 3) for v in nx], "mag", round(mag(nx), 3)) print("rmsnorm(x+delta)", [round(v, 3) for v in ny], "mag", round(mag(ny), 3)) print("direction kept, size tamed") huge = [30.0, -10.0, 20.0] print("rmsnorm(10*x)", [round(v, 3) for v in rmsnorm(huge)])
Run to execute this in your browser. Nothing is sent to a server.