← Lesson
Overlap, Offsets, and Chunk Ids
Joeven
Run
Reset
Python loads on first run
TEXT = "Refunds take 5-7 days. Never refund in cash. Raise worker memory on OOM." def window(text, size, overlap): step = max(size - overlap, 1) out = [] i = 0 n = 0 while i < len(text): piece = text[i:i + size] n += 1 out.append({ "id": "c" + str(n), "start": i, "end": i + len(piece), "text": piece, }) i += step return out chunks = window(TEXT, size=28, overlap=8) for c in chunks: span = TEXT[c["start"]:c["end"]] print(c["id"], c["start"], c["end"], span) print(" matches", span == c["text"])
Run to execute this in your browser. Nothing is sent to a server.