law roundtrip_trained

bend-ml-bpe-tokenizer@0.1.3.0 · main.bend:649

import Base

649# LAW 5 (roundtrip_trained): train a table on any corpus, with as many merges as you like:
650# decoding what it encodes always returns the original bytes. No hypothesis at all.
651law roundtrip_trained:
652  for +n: Nat
653  for +corpus: List<&2, Nat>
654  for +bs: List<&2, Nat>
655  {decode(train(n, 0n, lift(corpus)), encode(train(n, 0n, lift(corpus)), lift(bs))) == bs : List<&2, Nat>}