type Cnt

bend-ml-bpe-tokenizer@0.1.3.0 · main.bend:172

import Base

172# =====================================================================
173# Training (needs no proof: it generates tables; a table is valid if wf(table))
174# =====================================================================
175# ---------------------------------------------------------------
176# training: count adjacent pairs, merge the most frequent one, repeat
177# (tie-break: the first pair that appeared, as in minbpe)
178# ---------------------------------------------------------------
179type Cnt is Data:
180  Cnt{a: Tk, b: Tk, n: Nat}