def merge.go

bend-ml-bpe-tokenizer@0.1.3.0 · main.bend:52

import Base

52# Replaces each occurrence (non-overlapping, left to right) of the pair
53# (a, b) with the token c. `hit` is always `peek(a, b, xs)`: the caller computes it,
54# because a match can only open parameters.
55def merge.go(xs: List<&2, Tk>, hit: Bool, +a: Tk, +b: Tk, +c: Tk) -> List<&2, Tk>:
56  match xs hit:
57    case Con{x, Con{y, +t}} True{}:
58      c <> merge.go(t, peek(a, b, t), a, b, c)
59    case Con{x, Nil{}} True{}:
60      x <> Nil{}
61    case Con{x, +t} False{}:
62      x <> merge.go(t, peek(a, b, t), a, b, c)
63    case Nil{} _:
64      Nil{}