def lex

cswap-bend-cli@0.1.0.0 · json/lex.bend:258

import Base

258# Tokens of s. `fuel` is its length.
259def lex(fuel: Nat, s: String) -> List<&2, Tok>:
260  match fuel s:
261    case _ SNil{}:
262      []
263    case 0n _:
264      [TErr{"internal: lexer out of fuel"}]
265    case 1n+p SCon{+c, +t}:
266      tok.cons(tok.of(c, t), lex(p, String.drop(t, tok.adv(c, t))))