def tw

bend-ml-bpe-tokenizer@0.1.3.0 · main.bend:629

import Base

629# train.go generates rules with ids next, next+1, ...: none repeats, whatever the corpus
630def tw(n: Nat, +next: Nat, +xs: List<&2, Tk>, m: Maybe<&1, Cnt>, +done: List<&2, Rule>, +bl: {True{} == below(done, next) : Bool}) -> {True{} == wf.go(train.go(n, next, xs, m), done) : Bool}:
631  match n m:
632    case 0n _:
633      {==}
634    case 1n++p None{}:
635      {==}
636    case 1n++p Some{Cnt{+a, +b, k}}:
637      and_intro(Bool.not(has(done, next)), wf.go(train.go(p, 1n+next, merge(a, b, M{next}, xs), best(count.go(merge(a, b, M{next}, xs), Nil{}))), Rule{next, a, b} <> done), below_fresh(done, next, bl), tw(p, 1n+next, merge(a, b, M{next}, xs), best(count.go(merge(a, b, M{next}, xs), Nil{})), Rule{next, a, b} <> done, below_cons(a, b, done, next, bl)))