~/jendhub

def train.go

bend-ml-bpe-tokenizer@0.1.3.0 · main.bend:240

import bend-ml-bpe-tokenizer@0.1.3.0/main.bend as BendMlBpeTokenizer
import Base

240# up to n merges, ids from `next` on; m is the current best pair (best(count(xs)))
241def train.go(n: Nat, +next: Nat, +xs: List<&2, Tk>, m: Maybe<&1, Cnt>) -> List<&2, Rule>:
242  match n m:
243    case 0n _:
244      Nil{}
245    case 1n+p None{}:
246      Nil{}
247    case 1n+p Some{Cnt{+a, +b, k}}:
248      +ys = merge(a, b, M{next}, xs)
249      Rule{next, a, b} <> train.go(p, 1n+next, ys, best(count.go(ys, Nil{})))

Uses (9)

Used by (2)