BendMlBpeTokenizer.exp.go: def of bend-ml-bpe-tokenizer@0.1.3.0
Import: import bend-ml-bpe-tokenizer@0.1.3.0/main.bend as BendMlBpeTokenizer
Lines 98-112 of bend-ml-bpe-tokenizer@0.1.3.0/main.bend: https://jend.leeeet.dev/read.txt?path=bend-ml-bpe-tokenizer@0.1.3.0/main.bend&start_line=98&end_line=112

import Base

# Expansion of a token into bytes. The table has the newest rule first and
# a rule only refers to older tokens, so expanding a and b looks only at
# the rest of the table. `hit` is always `hits(table, tk)`.
def exp.go(table: List<&2, Rule>, hit: Bool, tk: Tk) -> List<&2, Nat>:
  match table hit tk:
    case Nil{} _ B{n}:
      n <> Nil{}
    case Nil{} _ M{j}:
      Nil{}
    case Con{r, rest} _ B{n}:
      n <> Nil{}
    case Con{Rule{i, +a, +b}, +rest} True{} M{j}:
      List.append(&2, Nat, exp.go(rest, hits(rest, a), a), exp.go(rest, hits(rest, b), b))
    case Con{r, +rest} False{} M{+j}:
      exp.go(rest, hits(rest, M{j}), M{j})

Uses (7):
  https://jend.leeeet.dev/definition.txt?path=Base/base.bend&name=Bool
  https://jend.leeeet.dev/definition.txt?path=Base/base.bend&name=List
  https://jend.leeeet.dev/definition.txt?path=Base/base.bend&name=List.append
  https://jend.leeeet.dev/definition.txt?path=Base/base.bend&name=Nat
  https://jend.leeeet.dev/definition.txt?path=bend-ml-bpe-tokenizer@0.1.3.0/main.bend&name=Rule
  https://jend.leeeet.dev/definition.txt?path=bend-ml-bpe-tokenizer@0.1.3.0/main.bend&name=Tk
  https://jend.leeeet.dev/definition.txt?path=bend-ml-bpe-tokenizer@0.1.3.0/main.bend&name=hits