← Lesson
Special Tokens
Joeven
Run
Reset
Python loads on first run
stoi = { "<pad>": 0, "<eos>": 1, "<user>": 2, "<assistant>": 3, "refund": 4, "now": 5, } itos = {i: t for t, i in stoi.items()} def encode_turn(role, text): rid = stoi["<" + role + ">"] ids = [rid] for w in text.split(): ids.append(stoi[w]) ids.append(stoi["<eos>"]) return ids ids = encode_turn("user", "refund now") + encode_turn("assistant", "refund") print("ids", ids) print("tokens", [itos[i] for i in ids]) swapped = encode_turn("assistant", "refund now") + encode_turn("user", "refund") print("swapped ids", swapped) print("swapped tokens", [itos[i] for i in swapped]) print("pad id", stoi["<pad>"], "must not be generated") print("eos id", stoi["<eos>"], "ends the turn")
Run to execute this in your browser. Nothing is sent to a server.