type Tk

bend-ml-bpe-tokenizer@0.1.3.0 · main.bend:20

import Base

20# Token: a raw byte (B) or the result of a merge rule, by rule id (M).
21type Tk is Data:
22  B{n: Nat}
23  M{k: Nat}