def exp.go

bend-ml-bpe-tokenizer@0.1.3.0 · main.bend:98

import Base

98# Expansion of a token into bytes. The table has the newest rule first and
99# a rule only refers to older tokens, so expanding a and b looks only at
100# the rest of the table. `hit` is always `hits(table, tk)`.
101def exp.go(table: List<&2, Rule>, hit: Bool, tk: Tk) -> List<&2, Nat>:
102  match table hit tk:
103    case Nil{} _ B{n}:
104      n <> Nil{}
105    case Nil{} _ M{j}:
106      Nil{}
107    case Con{r, rest} _ B{n}:
108      n <> Nil{}
109    case Con{Rule{i, +a, +b}, +rest} True{} M{j}:
110      List.append(&2, Nat, exp.go(rest, hits(rest, a), a), exp.go(rest, hits(rest, b), b))
111    case Con{r, +rest} False{} M{+j}:
112      exp.go(rest, hits(rest, M{j}), M{j})