def tokenize

gralha-xml-doc@0.1.0.1 · xml.bend:226

import Base
import gralha-text-scan@0.1.0.1/scan.bend as U

226def tokenize(s: String) -> List<&2, Token>:
227  match s:
228    case SNil{}: Nil{}
229    case SCon{c, t}:
230      lex.text(U.until(SCon{c, t}, 60, ""), next => tokenize(next))