← Lesson
Prefill and Decode
Joeven
Run
Reset
Python loads on first run
def ms_prefill(n_prompt, cost_per_tok=0.02): return n_prompt * n_prompt * 0.000002 + n_prompt * cost_per_tok def ms_decode(n_out, cache_len, cost_per_tok=0.05): total = 0.0 for i in range(n_out): total += cost_per_tok + cache_len * 0.00001 cache_len += 1 return total prompt = 4000 out = 80 p = ms_prefill(prompt) d = ms_decode(out, prompt) print("prompt", prompt, "out", out) print("prefill ms", round(p, 1)) print("decode ms", round(d, 1)) print("first token waits on prefill; the 80 output tokens are extra") print("double prompt prefill", round(ms_prefill(8000), 1)) print("double out decode", round(ms_decode(160, prompt), 1)) print("short prompt 500 prefill", round(ms_prefill(500), 1))
Run to execute this in your browser. Nothing is sent to a server.