← Lesson
Padding and Masks
Joeven
Run
Reset
Python loads on first run
PAD = 0 seqs = [ [4, 5, 6], [7, 8], ] max_len = max(len(s) for s in seqs) def right_pad(s): return s + [PAD] * (max_len - len(s)) batch = [right_pad(s) for s in seqs] print("batch", batch) def attn_legal(query_i, key_j, seq, causal=True): if seq[key_j] == PAD: return False if causal and key_j > query_i: return False if seq[query_i] == PAD: return False return True seq = batch[1] print("mask for shorter row", seq) for i in range(max_len): row = [("Y" if attn_legal(i, j, seq) else ".") for j in range(max_len)] print("q" + str(i), " ".join(row)) print("long row", batch[0]) for i in range(max_len): row = [("Y" if attn_legal(i, j, batch[0]) else ".") for j in range(max_len)] print("q" + str(i), " ".join(row))
Run to execute this in your browser. Nothing is sent to a server.