← Lesson
Byte Pair Encoding
Joeven
Run
Reset
Python loads on first run
from collections import Counter def pair_counts(seq): return Counter(zip(seq, seq[1:])) def merge(seq, pair, new_tok): out = [] i = 0 while i < len(seq): if i < len(seq) - 1 and (seq[i], seq[i + 1]) == pair: out.append(new_tok) i += 2 else: out.append(seq[i]) i += 1 return out text = "low lower newest newest" seq = list(text) print("start n", len(seq)) print("start", seq) for step in range(1, 6): pair, n = pair_counts(seq).most_common(1)[0] name = pair[0] + pair[1] seq = merge(seq, pair, name) print("step", step, "glue", pair, "count", n, "new", name, "n", len(seq)) print("result n", len(seq)) print("result", seq)
Run to execute this in your browser. Nothing is sent to a server.