def tokens

emerging-ezjson@1.3.0.0 · src/lex.bend:417

import emerging-ezjson@1.3.0.0/src/lex.bend as Lex

import Base

417# a text as tokens. Clean words and strings are spans of the text
418def tokens(txt: String) -> List<&2, Tok>:
419  tokens.go(txt, Go{"", 0, GIdle{}, []})

Uses (6)

Used by (1)

emerging-ezjson@1.3.0.0/src/parse:parse