← Lesson
The Transformer Block
Joeven
Run
Reset
Python loads on first run
def relu(xs): return [max(0.0, x) for x in xs] def add(a, b): return [x + y for x, y in zip(a, b)] def mlp(v): W1 = [[1.0, 0.0], [0.0, 1.0], [1.0, 1.0], [0.5, -0.5]] b1 = [0.0, 0.0, -0.2, 0.0] h = relu([sum(w * x for w, x in zip(row, v)) + b for row, b in zip(W1, b1)]) W2 = [[0.3, 0.1, 0.2, 0.0], [0.0, 0.3, 0.1, 0.2]] return [sum(w * x for w, x in zip(row, h)) for row in W2] x = [ [1.0, 0.0], [0.0, 1.0], ] print("in", x) attn = [ add([0.7 * a for a in x[0]], [0.3 * a for a in x[1]]), add([0.7 * a for a in x[1]], [0.3 * a for a in x[0]]), ] x = [add(a, d) for a, d in zip(x, attn)] print("after residual attn", [[round(v, 3) for v in row] for row in x]) ff = [mlp(v) for v in x] x = [add(a, d) for a, d in zip(x, ff)] print("after residual mlp", [[round(v, 3) for v in row] for row in x]) print("length still", len(x), "width still", len(x[0]))
Run to execute this in your browser. Nothing is sent to a server.