src/pull.bend

emerging-ezjson@1.3.0.0 · 1,664 lines · 146 definitions

1# src/pull: one JSON text, one event at a time.
2# `cursor` holds the unread suffix of the source. `next` returns one event and
3# the cursor after it. `skip` drops one value. A span event keeps the suffix
4# it points at, so it stays readable while that event is held and is gone
5# when the event is dropped. Commas and colons are not events.
6import Base
7import emerging-ezjson@1.3.0.0/src/value.bend as V
8import emerging-ezjson@1.3.0.0/src/lex.bend as Lex
9
10# where the next token of an open array or object has to be
11type Slot is Data:
12  SVal{}
13  SCom{}
14  SKey{}
15  SCol{}
16
17# an open array or object. `fresh` is the empty container, before any value
18type Frame is Data:
19  FArr{slot: Slot, fresh: Bool}
20  FObj{slot: Slot, fresh: Bool}
21
22# one event. A span is the first `nn` chars of a source suffix, not a copy
23type Ev is Data:
24  ENull{}
25  EBool{b: Bool}
26  ENum{src: String, nn: U32}
27  EStr{body: String}
28  EStrS{src: String, nn: U32}
29  EKey{body: String}
30  EKeyS{src: String, nn: U32}
31  EBeginArr{}
32  EEndArr{}
33  EBeginObj{}
34  EEndObj{}
35  EEnd{}
36  EErr{}
37
38# the cursor. `rest` is the unread suffix. `done` is set once the root value
39# has ended. `bad` sticks after the first error
40type Cur is Data:
41  Cur{rest: String, stack: List<&2, Frame>, done: Bool, bad: Bool}
42
43# read the next event, or drop one whole value
44type Goal is Data:
45  GNext{}
46  GSkip{depth: Nat}
47
48# what a step hands back: an event, or a finished skip
49type Stop is Data:
50  SEv{ev: Ev}
51  SSkip{}
52
53# a bare word, once it has been classified
54type Atom is Data:
55  ANull{}
56  ATrue{}
57  AFalse{}
58  ANum{}
59  ABad{}
60
61# a string's text: a span of the source, or chars decoded from escapes
62type Piece is Data:
63  PSpan{cut: String, nn: U32}
64  POwn{body: String}
65
66# what one character of a string does. The usual one is KGo: stay in the
67# string. The others leave it
68type Skind is Data:
69  KGo{}
70  KEnd{}
71  KEsc{}
72  KBad{}
73
74# where a finished value lands, or that the slot cannot take one
75type Spot is Data:
76  Spot{stack: List<&2, Frame>, root: Bool}
77  SpotBad{}
78
79# what the scanner is inside. Skip-modes (`MSk*`) walk a string they will
80# not keep, so a dropped value does not build its text
81type Mode is Data:
82  MIdle{}
83  # a bare word's characters, reversed, so the unread array is not kept
84  MWord{buf: List<&2, Char>, nn: U32, num: V.Num}
85  MSpan{cut: String, nn: U32}
86  MCopy{buf: List<&2, Char>}
87  MEsc{buf: List<&2, Char>}
88  MUni{left: Nat, acc: U32, buf: List<&2, Char>}
89  MHi{hi: U32, buf: List<&2, Char>}
90  MHiEsc{hi: U32, buf: List<&2, Char>}
91  MLo{left: Nat, acc: U32, hi: U32, buf: List<&2, Char>}
92  MSkStr{}
93  MSkEsc{}
94  MSkUni{left: Nat, acc: U32}
95  MSkHi{hi: U32}
96  MSkHiEsc{hi: U32}
97  MSkLo{left: Nat, acc: U32, hi: U32}
98  MDone{stop: Stop, saved: String}
99  # the cursor rest is the suffix the stepper is already on, so closing a
100  # long string does not keep that suffix a second time
101  MRest{stop: Stop}
102  # the quote is consumed; the next character is the first of the string
103  MQuote{}
104  # resume `mode` on `rest` after a plain word, which the stepper did not
105  # walk byte by byte
106  MJump{rest: String, mode: Mode}
107
108# the scanner's state, apart from the unread suffix it is walking
109type St is Data:
110  S{mode: Mode, stack: List<&2, Frame>, goal: Goal, done: Bool, bad: Bool}
111
112# what a walk hands back: a finished step, or a budget used up, to resume on
113# `txt` in state `st`
114type Out is Type:
115  Fin{got: (Stop & Cur)}
116  More{txt: String, st: St}
117
118# a failed step. The unread suffix is dropped
119def err.st(goal: Goal) -> St:
120  S{MDone{SEv{EErr{}}, ""}, Nil{}, goal, False{}, True{}}
121
122# a failed cursor
123def bad.cur() -> (Stop & Cur):
124  (SEv{EErr{}}, Cur{"", Nil{}, False{}, True{}})
125
126# the first `nn` chars of `src`, reversed, for the escape copier
127def copy.rev(src: String, +nn: U32, zero: Bool, acc: List<&2, Char>) -> List<&2, Char>:
128  match src zero:
129    case s True{}:
130      acc
131    case SNil{} False{}:
132      acc
133    case SCon{h, t} False{}:
134      copy.rev(t, U32.sub(nn, 1), U32.is_zero(U32.sub(nn, 1)), h <> acc)
135
136# a value can start here
137def expect.val.go(stack: List<&2, Frame>) -> Bool:
138  match stack:
139    case Nil{}:
140      True{}
141    case Con{FArr{SVal{}, fresh}, rest}:
142      True{}
143    case Con{FObj{SVal{}, fresh}, rest}:
144      True{}
145    case other:
146      False{}
147
148# a value can start here, and the root is not already finished
149def expect.val(stack: List<&2, Frame>, done: Bool) -> Bool:
150  match done:
151    case True{}:
152      False{}
153    case False{}:
154      expect.val.go(stack)
155
156# a string can start here: a value, or an object key
157def expect.str.go(stack: List<&2, Frame>) -> Bool:
158  match stack:
159    case Nil{}:
160      True{}
161    case Con{FArr{SVal{}, fresh}, rest}:
162      True{}
163    case Con{FObj{SVal{}, fresh}, rest}:
164      True{}
165    case Con{FObj{SKey{}, fresh}, rest}:
166      True{}
167    case other:
168      False{}
169
170# a string can start here, and the root is not already finished
171def expect.str(stack: List<&2, Frame>, done: Bool) -> Bool:
172  match done:
173    case True{}:
174      False{}
175    case False{}:
176      expect.str.go(stack)
177
178# a string can start here for this goal. A skip of one whole value must
179# start a value, so a key there fails at once, as it would once read
180def expect.str.for(stack: List<&2, Frame>, goal: Goal, done: Bool) -> Bool:
181  match goal:
182    case GSkip{0n}:
183      expect.val(stack, done)
184    case _:
185      expect.str(stack, done)
186
187# the stack after a value, and whether that value was the root
188def place.spot(stack: List<&2, Frame>) -> Spot:
189  match stack:
190    case Nil{}:
191      Spot{Nil{}, True{}}
192    case Con{FArr{SVal{}, fresh}, frames}:
193      Spot{FArr{SCom{}, False{}} <> frames, False{}}
194    case Con{FObj{SVal{}, fresh}, frames}:
195      Spot{FObj{SCom{}, False{}} <> frames, False{}}
196    case other:
197      SpotBad{}
198
199# a value has landed in a slot that can take it
200def place.val.ok(ev: Ev, rest: String, goal: Goal, stack: List<&2, Frame>, root: Bool) -> St:
201  match goal:
202    case GNext{}:
203      S{MDone{SEv{ev}, rest}, stack, GNext{}, root, False{}}
204    case GSkip{0n}:
205      S{MDone{SSkip{}, rest}, stack, GSkip{0n}, root, False{}}
206    case GSkip{1n+left}:
207      S{MIdle{}, stack, GSkip{1n+left}, False{}, False{}}
208
209# a value has landed. `rest` is the suffix the cursor keeps
210def place.val.on(ev: Ev, rest: String, goal: Goal, spot: Spot) -> St:
211  match spot:
212    case SpotBad{}:
213      err.st(goal)
214    case Spot{stack, root}:
215      place.val.ok(ev, rest, goal, stack, root)
216
217# a value has landed
218def place.val(ev: Ev, rest: String, stack: List<&2, Frame>, goal: Goal) -> St:
219  place.val.on(ev, rest, goal, place.spot(stack))
220
221# an object key has landed. The value comes after the colon
222def place.key(ev: Ev, rest: String, frames: List<&2, Frame>, goal: Goal) -> St:
223  match goal:
224    case GNext{}:
225      S{MDone{SEv{ev}, rest}, FObj{SCol{}, False{}} <> frames, goal, False{}, False{}}
226    case GSkip{0n}:
227      err.st(goal)
228    case GSkip{1n+left}:
229      S{MIdle{}, FObj{SCol{}, False{}} <> frames, goal, False{}, False{}}
230
231# a string event, span or owned, as a key or a value
232def str.ev(piece: Piece, key: Bool) -> Ev:
233  match piece key:
234    case PSpan{cut, nn} True{}:
235      EKeyS{cut, nn}
236    case PSpan{cut, nn} False{}:
237      EStrS{cut, nn}
238    case POwn{body} True{}:
239      EKey{body}
240    case POwn{body} False{}:
241      EStr{body}
242
243# a string value inside a skip, once the slot is known
244def str.cont.on(goal: Goal, spot: Spot) -> St:
245  match spot:
246    case SpotBad{}:
247      err.st(goal)
248    case Spot{stack2, root}:
249      S{MIdle{}, stack2, goal, False{}, False{}}
250
251# a string value inside a skip: keep the slot, not the text
252def str.cont(stack: List<&2, Frame>, goal: Goal) -> St:
253  str.cont.on(goal, place.spot(stack))
254
255# a string value. A skip that is still inside a container keeps scanning
256def str.val(ev: Ev, rest: String, stack: List<&2, Frame>, goal: Goal) -> St:
257  match goal:
258    case GSkip{1n+left}:
259      str.cont(stack, goal)
260    case GNext{}:
261      place.val(ev, rest, stack, goal)
262    case GSkip{0n}:
263      place.val(ev, rest, stack, goal)
264
265# a finished string. A key goes to the open object; anything else is a value
266def str.place(piece: Piece, rest: String, stack: List<&2, Frame>, goal: Goal) -> St:
267  match stack:
268    case Con{FObj{SKey{}, fresh}, frames}:
269      place.key(str.ev(piece, True{}), rest, frames, goal)
270    case other:
271      str.val(str.ev(piece, False{}), rest, stack, goal)
272
273# a string a skip will not keep
274def str.skip(rest: String, stack: List<&2, Frame>, goal: Goal) -> St:
275  match goal:
276    case GNext{}:
277      err.st(goal)
278    case GSkip{depth}:
279      str.place(POwn{""}, rest, stack, goal)
280
281# a value has landed, and the cursor rest is the suffix already in hand
282def place.val.ok.here(ev: Ev, goal: Goal, stack: List<&2, Frame>, root: Bool) -> St:
283  match goal:
284    case GNext{}:
285      S{MRest{SEv{ev}}, stack, GNext{}, root, False{}}
286    case GSkip{0n}:
287      S{MRest{SSkip{}}, stack, GSkip{0n}, root, False{}}
288    case GSkip{1n+left}:
289      S{MIdle{}, stack, GSkip{1n+left}, False{}, False{}}
290
291# a value has landed on the suffix in hand, once the slot is known
292def place.val.on.here(ev: Ev, goal: Goal, spot: Spot) -> St:
293  match spot:
294    case SpotBad{}:
295      err.st(goal)
296    case Spot{stack, root}:
297      place.val.ok.here(ev, goal, stack, root)
298
299# a value has landed on the suffix in hand
300def place.val.here(ev: Ev, stack: List<&2, Frame>, goal: Goal) -> St:
301  place.val.on.here(ev, goal, place.spot(stack))
302
303# an object key has landed on the suffix in hand
304def place.key.here(ev: Ev, frames: List<&2, Frame>, goal: Goal) -> St:
305  match goal:
306    case GNext{}:
307      S{MRest{SEv{ev}}, FObj{SCol{}, False{}} <> frames, goal, False{}, False{}}
308    case GSkip{0n}:
309      err.st(goal)
310    case GSkip{1n+left}:
311      S{MIdle{}, FObj{SCol{}, False{}} <> frames, goal, False{}, False{}}
312
313# a string value on the suffix in hand. A skip inside a container keeps scanning
314def str.val.here(ev: Ev, stack: List<&2, Frame>, goal: Goal) -> St:
315  match goal:
316    case GSkip{1n+left}:
317      str.cont(stack, goal)
318    case GNext{}:
319      place.val.here(ev, stack, goal)
320    case GSkip{0n}:
321      place.val.here(ev, stack, goal)
322
323# a finished string whose rest is the suffix the stepper is on
324def str.place.here(piece: Piece, stack: List<&2, Frame>, goal: Goal) -> St:
325  match stack:
326    case Con{FObj{SKey{}, fresh}, frames}:
327      place.key.here(str.ev(piece, True{}), frames, goal)
328    case other:
329      str.val.here(str.ev(piece, False{}), stack, goal)
330
331# a skipped string closed on the suffix in hand
332def str.skip.here(stack: List<&2, Frame>, goal: Goal) -> St:
333  match goal:
334    case GNext{}:
335      err.st(goal)
336    case GSkip{depth}:
337      str.place.here(POwn{""}, stack, goal)
338
339# one character of a string: stay in it, close it, start an escape, or fail
340def span.kind(cls: Lex.Class, ctrl: Bool) -> Skind:
341  match cls ctrl:
342    case Lex.CQuote{} c:
343      KEnd{}
344    case Lex.CBack{} False{}:
345      KEsc{}
346    case k True{}:
347      KBad{}
348    case k False{}:
349      KGo{}
350
351# classify one string character. `ch` is a code point, so the keep is a word
352def span.kind.of(+ch: Char) -> Skind:
353  span.kind(Lex.classify(ch), U32.is_lt(Char.to_u32(ch), 32))
354
355# `]` or `}` of a skip: depth 1 finishes the value, deeper keeps going.
356# The cursor rest is the suffix the stepper is already on
357def close.skip.go(stack: List<&2, Frame>, left: Nat, root: Bool) -> St:
358  match left:
359    case 0n:
360      S{MRest{SSkip{}}, stack, GSkip{0n}, root, False{}}
361    case 1n+more:
362      S{MIdle{}, stack, GSkip{1n+more}, False{}, False{}}
363
364# `]` or `}` of a skip, once the frame has been popped
365def close.skip(stack: List<&2, Frame>, depth: Nat, root: Bool) -> St:
366  match depth:
367    case 0n:
368      err.st(GSkip{0n})
369    case 1n+left:
370      close.skip.go(stack, left, root)
371
372# the parent after a container closes, for a reader
373def close.next(ev: Ev, stack: List<&2, Frame>, root: Bool) -> St:
374  S{MRest{SEv{ev}}, stack, GNext{}, root, False{}}
375
376# a container closed. `frames` is the stack with that frame popped
377def close.out(ev: Ev, frames: List<&2, Frame>, goal: Goal) -> St:
378  match frames goal:
379    case Nil{} GNext{}:
380      close.next(ev, Nil{}, True{})
381    case Nil{} GSkip{depth}:
382      close.skip(Nil{}, depth, True{})
383    case Con{FArr{SVal{}, fresh}, outer} GNext{}:
384      close.next(ev, FArr{SCom{}, False{}} <> outer, False{})
385    case Con{FObj{SVal{}, fresh}, outer} GNext{}:
386      close.next(ev, FObj{SCom{}, False{}} <> outer, False{})
387    case Con{FArr{SVal{}, fresh}, outer} GSkip{depth}:
388      close.skip(FArr{SCom{}, False{}} <> outer, depth, False{})
389    case Con{FObj{SVal{}, fresh}, outer} GSkip{depth}:
390      close.skip(FObj{SCom{}, False{}} <> outer, depth, False{})
391    case frames2 g:
392      err.st(g)
393
394# a `]`. Empty, or just after a value
395def close.arr(stack: List<&2, Frame>, goal: Goal) -> St:
396  match stack:
397    case Con{FArr{SVal{}, True{}}, frames}:
398      close.out(EEndArr{}, frames, goal)
399    case Con{FArr{SCom{}, fresh}, frames}:
400      close.out(EEndArr{}, frames, goal)
401    case other:
402      err.st(goal)
403
404# a `}`. Empty, or just after a value
405def close.obj(stack: List<&2, Frame>, goal: Goal) -> St:
406  match stack:
407    case Con{FObj{SKey{}, True{}}, frames}:
408      close.out(EEndObj{}, frames, goal)
409    case Con{FObj{SCom{}, fresh}, frames}:
410      close.out(EEndObj{}, frames, goal)
411    case other:
412      err.st(goal)
413
414# a comma between values, or between object members
415def punct.comma(stack: List<&2, Frame>, goal: Goal) -> St:
416  match stack:
417    case Con{FArr{SCom{}, fresh}, frames}:
418      S{MIdle{}, FArr{SVal{}, False{}} <> frames, goal, False{}, False{}}
419    case Con{FObj{SCom{}, fresh}, frames}:
420      S{MIdle{}, FObj{SKey{}, False{}} <> frames, goal, False{}, False{}}
421    case other:
422      err.st(goal)
423
424# a colon after a key
425def punct.colon(stack: List<&2, Frame>, goal: Goal) -> St:
426  match stack:
427    case Con{FObj{SCol{}, fresh}, frames}:
428      S{MIdle{}, FObj{SVal{}, False{}} <> frames, goal, False{}, False{}}
429    case other:
430      err.st(goal)
431
432# `[` pushed. A reader returns the begin; a skip counts the container
433def punct.arr.go(stack: List<&2, Frame>, goal: Goal) -> St:
434  match goal:
435    case GNext{}:
436      S{MRest{SEv{EBeginArr{}}}, stack, goal, False{}, False{}}
437    case GSkip{depth}:
438      S{MIdle{}, stack, GSkip{1n+depth}, False{}, False{}}
439
440# `[`, once it is known whether a value may start
441def punct.arr.if(stack: List<&2, Frame>, goal: Goal, ok: Bool) -> St:
442  match ok:
443    case True{}:
444      punct.arr.go(FArr{SVal{}, True{}} <> stack, goal)
445    case False{}:
446      err.st(goal)
447
448# `[`, when a value may start
449def punct.arr(+stack: List<&2, Frame>, goal: Goal, done: Bool) -> St:
450  punct.arr.if(stack, goal, expect.val(stack, done))
451
452# `{` pushed
453def punct.obj.go(stack: List<&2, Frame>, goal: Goal) -> St:
454  match goal:
455    case GNext{}:
456      S{MRest{SEv{EBeginObj{}}}, stack, goal, False{}, False{}}
457    case GSkip{depth}:
458      S{MIdle{}, stack, GSkip{1n+depth}, False{}, False{}}
459
460# `{`, once it is known whether a value may start
461def punct.obj.if(stack: List<&2, Frame>, goal: Goal, ok: Bool) -> St:
462  match ok:
463    case True{}:
464      punct.obj.go(FObj{SKey{}, True{}} <> stack, goal)
465    case False{}:
466      err.st(goal)
467
468# `{`, when a value may start
469def punct.obj(+stack: List<&2, Frame>, goal: Goal, done: Bool) -> St:
470  punct.obj.if(stack, goal, expect.val(stack, done))
471
472# one punctuation token, once the root is still open. The suffix stays
473# with the stepper: a bracket closes on it, and a comma does not keep it
474def punct.go(tok: Lex.Tok, stack: List<&2, Frame>, goal: Goal, done: Bool) -> St:
475  match tok:
476    case Lex.TOpenArr{}:
477      punct.arr(stack, goal, done)
478    case Lex.TOpenObj{}:
479      punct.obj(stack, goal, done)
480    case Lex.TCloseArr{}:
481      close.arr(stack, goal)
482    case Lex.TCloseObj{}:
483      close.obj(stack, goal)
484    case Lex.TColon{}:
485      punct.colon(stack, goal)
486    case Lex.TComma{}:
487      punct.comma(stack, goal)
488    case other:
489      err.st(goal)
490
491# one punctuation token
492def punct.on(tok: Lex.Tok, stack: List<&2, Frame>, goal: Goal, done: Bool) -> St:
493  match done:
494    case True{}:
495      err.st(goal)
496    case False{}:
497      punct.go(tok, stack, goal, done)
498
499# whitespace, kept as the same state
500def idle.space(stack: List<&2, Frame>, goal: Goal, done: Bool) -> St:
501  S{MIdle{}, stack, goal, done, False{}}
502
503# a `"` may open a string here
504def look.quote(stack: List<&2, Frame>, goal: Goal, done: Bool, ok: Bool) -> St:
505  match ok:
506    case True{}:
507      S{MQuote{}, stack, goal, done, False{}}
508    case False{}:
509      err.st(goal)
510
511# a word's first character. The buffer is that character, not the array behind it
512def look.word(+ch: Char, stack: List<&2, Frame>, goal: Goal, done: Bool, ok: Bool) -> St:
513  match ok:
514    case True{}:
515      S{MWord{ch <> [], 1, V.num.ok.step(V.NStart{}, Char.to_u32(ch))},
516        stack, goal, done, False{}}
517    case False{}:
518      err.st(goal)
519
520# one character between tokens. The suffix stays with the stepper: a bracket
521# closes on it, a comma does not keep it, and a word keeps only its own char
522def look.plan(
523  ch: Char,
524  +stack: List<&2, Frame>,
525  +goal: Goal,
526  +done: Bool,
527  cls: Lex.Class
528) -> St:
529  match cls:
530    case Lex.CSpace{}:
531      idle.space(stack, goal, done)
532    case Lex.CPunct{tok}:
533      punct.on(tok, stack, goal, done)
534    case Lex.CQuote{}:
535      look.quote(stack, goal, done, expect.str.for(stack, goal, done))
536    case Lex.CBack{}:
537      err.st(goal)
538    case Lex.COther{}:
539      look.word(ch, stack, goal, done, expect.val(stack, done))
540
541# the end of the text, between tokens
542def idle.eof(stack: List<&2, Frame>, goal: Goal, done: Bool, bad: Bool) -> (Stop & Cur):
543  match goal done bad:
544    case GNext{} True{} False{}:
545      (SEv{EEnd{}}, Cur{"", stack, True{}, False{}})
546    case g d b:
547      bad.cur()
548
549# a number, when the word is not null, true, or false
550def word.kind.num(ok: Bool) -> Atom:
551  match ok:
552    case True{}:
553      ANum{}
554    case False{}:
555      ABad{}
556
557# null, true, false, or a number that kept a legal spelling
558def word.kind.go(num: V.Num, is_null: Bool, is_true: Bool, is_false: Bool) -> Atom:
559  match is_null is_true is_false:
560    case True{} x y:
561      ANull{}
562    case False{} True{} y:
563      ATrue{}
564    case False{} False{} True{}:
565      AFalse{}
566    case False{} False{} False{}:
567      word.kind.num(V.num.ok.done(num))
568
569# what a bare word is
570def word.kind(+cut: String, +nn: U32, num: V.Num) -> Atom:
571  word.kind.go(num, V.span.eq(cut, nn, "null"), V.span.eq(cut, nn, "true"),
572    V.span.eq(cut, nn, "false"))
573
574# a word that ended at the end of the text
575def word.eof.place(
576  atom: Atom,
577  cut: String,
578  nn: U32,
579  stack: List<&2, Frame>,
580  goal: Goal
581) -> St:
582  match atom:
583    case ABad{}:
584      err.st(goal)
585    case ANull{}:
586      place.val(ENull{}, "", stack, goal)
587    case ATrue{}:
588      place.val(EBool{True{}}, "", stack, goal)
589    case AFalse{}:
590      place.val(EBool{False{}}, "", stack, goal)
591    case ANum{}:
592      place.val(ENum{cut, nn}, "", stack, goal)
593
594# the delimiter of a skipped word, once the value is inside the container.
595# The delimiter is consumed. The caller resumes on the suffix after it
596def lay.plan(stack: List<&2, Frame>, goal: Goal, cls: Lex.Class) -> St:
597  match cls:
598    case Lex.CSpace{}:
599      S{MIdle{}, stack, goal, False{}, False{}}
600    case Lex.CPunct{tok}:
601      punct.on(tok, stack, goal, False{})
602    case other:
603      err.st(goal)
604
605# place the word, then take the delimiter, because the skip is still open
606def see.more.on(goal: Goal, cls: Lex.Class, spot: Spot) -> St:
607  match spot:
608    case SpotBad{}:
609      err.st(goal)
610    case Spot{stack, root}:
611      lay.plan(stack, goal, cls)
612
613# place the word, then take the delimiter
614def see.more(stack: List<&2, Frame>, goal: Goal, cls: Lex.Class) -> St:
615  see.more.on(goal, cls, place.spot(stack))
616
617# a code point that ends a bare word: the same bytes `classify` calls
618# punctuation, a quote, a backslash, or whitespace. A form feed stays in the word
619def word.special(+cp: U32) -> Bool:
620  Bool.or(U32.is_eq(cp, 9),
621    Bool.or(U32.is_eq(cp, 10),
622      Bool.or(U32.is_eq(cp, 13),
623        Bool.or(U32.is_eq(cp, 32),
624          Bool.or(U32.is_eq(cp, 34),
625            Bool.or(U32.is_eq(cp, 44),
626              Bool.or(U32.is_eq(cp, 58),
627                Bool.or(U32.is_eq(cp, 91),
628                  Bool.or(U32.is_eq(cp, 92),
629                    Bool.or(U32.is_eq(cp, 93),
630                      Bool.or(U32.is_eq(cp, 123), U32.is_eq(cp, 125))))))))))))
631
632# where a plain word run stopped. A stop keeps the delimiter, already consumed
633type Wst is Data:
634  WMore{buf: List<&2, Char>, nn: U32, num: V.Num}
635  WStop{buf: List<&2, Char>, nn: U32, num: V.Num, ch: Char}
636  WEof{buf: List<&2, Char>, nn: U32, num: V.Num}
637
638# one more plain byte of a word. The buffer grows by that character
639def word.step(ch: Char, +cp: U32, buf: List<&2, Char>, nn: U32, num: V.Num) -> Wst:
640  WMore{ch <> buf, U32.add(nn, 1), V.num.ok.step(num, cp)}
641
642# keep the byte when it is not a delimiter
643def word.keep.go(
644  ch: Char,
645  cp: U32,
646  buf: List<&2, Char>,
647  nn: U32,
648  num: V.Num,
649  keep: Bool
650) -> Wst:
651  match keep:
652    case True{}:
653      word.step(ch, cp, buf, nn, num)
654    case False{}:
655      WStop{buf, nn, num, ch}
656
657# a digit is never a delimiter, so a long number does not walk `classify`
658def word.digit.go(
659  ch: Char,
660  +cp: U32,
661  buf: List<&2, Char>,
662  nn: U32,
663  num: V.Num,
664  dig: Bool
665) -> Wst:
666  match dig:
667    case True{}:
668      word.step(ch, cp, buf, nn, num)
669    case False{}:
670      word.keep.go(ch, cp, buf, nn, num, Bool.not(word.special(cp)))
671
672# one byte of a word: another plain byte, or the delimiter that ends it
673def word.decide(ch: Char, +cp: U32, buf: List<&2, Char>, nn: U32, num: V.Num) -> Wst:
674  word.digit.go(ch, cp, buf, nn, num, V.num.ok.digit(cp))
675
676# the suffix after a plain word. Tail call, so a long number is one loop
677def word.run(txt: String, st: Wst) -> (String & Wst):
678  match txt st:
679    case rest WStop{buf, nn, num, ch}:
680      (rest, WStop{buf, nn, num, ch})
681    case rest WEof{buf, nn, num}:
682      (rest, WEof{buf, nn, num})
683    case SNil{} WMore{buf, nn, num}:
684      ("", WEof{buf, nn, num})
685    case SCon{+c, t} WMore{buf, nn, num}:
686      word.run(t, word.decide(c, Char.to_u32(c), buf, nn, num))
687
688# drop an unread suffix. Tail call, so a failed long value does not stack
689def word.drop(txt: String) -> Bool:
690  match txt:
691    case SNil{}:
692      True{}
693    case SCon{c, t}:
694      word.drop(t)
695
696# drop a word's characters. Tail call, so a long spelling does not stack
697def word.forget(buf: List<&2, Char>) -> Bool:
698  match buf:
699    case Nil{}:
700      True{}
701    case Con{c, t}:
702      word.forget(t)
703
704# a failed word, once its buffer and the unread suffix are gone
705def word.burst.err(goal: Goal, dropped: Bool) -> St:
706  match dropped:
707    case d:
708      err.st(goal)
709
710# a failed word drops the buffer after the unread suffix
711def word.burst.dead(goal: Goal, dropped: Bool, buf: List<&2, Char>) -> St:
712  match dropped:
713    case d:
714      word.burst.err(goal, word.forget(buf))
715
716# a finished event already carries its suffix. Anything else continues on `rest`
717def jump.keep(dropped: Bool, st: St) -> St:
718  match dropped st:
719    case d S{mode, stack, goal, done, bad}:
720      S{mode, stack, goal, done, bad}
721
722# a state that already stopped drops `rest`. Anything else resumes on it
723def jump.unless.done(+rest: String, st: St) -> St:
724  match st:
725    case S{MDone{stop, saved}, stack, goal, done, bad}:
726      jump.keep(word.drop(rest), S{MDone{stop, saved}, stack, goal, done, bad})
727    case S{mode, stack, goal, done, bad}:
728      S{MJump{rest, mode}, stack, goal, done, bad}
729
730# a word that ended because the text did, once its spelling is in hand
731def word.burst.eof.raw(
732  +raw: String,
733  +nn: U32,
734  num: V.Num,
735  stack: List<&2, Frame>,
736  goal: Goal
737) -> St:
738  match goal:
739    case GSkip{1n+left}:
740      word.burst.err(goal, word.drop(raw))
741    case GNext{}:
742      word.eof.place(word.kind(raw, nn, num), raw, nn, stack, goal)
743    case GSkip{0n}:
744      word.eof.place(word.kind(raw, nn, num), raw, nn, stack, goal)
745
746# the empty suffix of an eof is dropped, then the word is placed
747def word.burst.eof.at(
748  dropped: Bool,
749  buf: List<&2, Char>,
750  nn: U32,
751  num: V.Num,
752  stack: List<&2, Frame>,
753  goal: Goal
754) -> St:
755  match dropped:
756    case d:
757      word.burst.eof.raw(Lex.text(buf), nn, num, stack, goal)
758
759# the skip is still inside a container, so the delimiter is taken and the
760# stepper continues. Only this path classifies that byte
761def word.burst.skip(rest: String, +ch: Char, stack: List<&2, Frame>, goal: Goal) -> St:
762  jump.unless.done(rest, see.more(stack, goal, Lex.classify(ch)))
763
764# a value landed on the delimiter. The suffix is consed back once, not copied.
765# The number keeps its own spelling, so it does not hold the rest of the array
766def word.burst.goal(
767  ev: Ev,
768  ch: Char,
769  rest: String,
770  stack: List<&2, Frame>,
771  goal: Goal
772) -> St:
773  match goal:
774    case GNext{}:
775      place.val(ev, SCon{ch, rest}, stack, goal)
776    case GSkip{0n}:
777      place.val(ev, SCon{ch, rest}, stack, goal)
778    case GSkip{1n+left}:
779      word.burst.skip(rest, ch, stack, goal)
780
781# both the spelling and the unread suffix are dropped when the word is not one
782def word.burst.junk(goal: Goal, dropped: Bool, rest: String) -> St:
783  match dropped:
784    case d:
785      word.burst.err(goal, word.drop(rest))
786
787# place the word. A number keeps its own spelling; the delimiter stays unread
788def word.burst.atom(
789  rest: String,
790  ch: Char,
791  raw: String,
792  nn: U32,
793  stack: List<&2, Frame>,
794  goal: Goal,
795  atom: Atom
796) -> St:
797  match atom:
798    case ABad{}:
799      word.burst.junk(goal, word.drop(raw), SCon{ch, rest})
800    case ANull{}:
801      word.burst.goal(ENull{}, ch, rest, stack, goal)
802    case ATrue{}:
803      word.burst.goal(EBool{True{}}, ch, rest, stack, goal)
804    case AFalse{}:
805      word.burst.goal(EBool{False{}}, ch, rest, stack, goal)
806    case ANum{}:
807      word.burst.goal(ENum{raw, nn}, ch, rest, stack, goal)
808
809# the delimiter is classified once, and only when a skip must consume it.
810# The plain bytes never were
811def word.burst.raw(
812  rest: String,
813  ch: Char,
814  +raw: String,
815  +nn: U32,
816  num: V.Num,
817  stack: List<&2, Frame>,
818  goal: Goal
819) -> St:
820  word.burst.atom(rest, ch, raw, nn, stack, goal, word.kind(raw, nn, num))
821
822# a word ended on a delimiter. Its spelling is its own text
823def word.burst.stop(
824  rest: String,
825  ch: Char,
826  buf: List<&2, Char>,
827  nn: U32,
828  num: V.Num,
829  stack: List<&2, Frame>,
830  goal: Goal
831) -> St:
832  word.burst.raw(rest, ch, Lex.text(buf), nn, num, stack, goal)
833
834# both the unread suffix and the buffer are dropped when the word cannot finish
835def word.burst.fail(goal: Goal, dropped: Bool, buf: List<&2, Char>) -> St:
836  match dropped:
837    case d:
838      word.burst.dead(goal, True{}, buf)
839
840# `got` is the suffix after the run, and why it stopped
841def word.burst.at(got: (String & Wst), stack: List<&2, Frame>, goal: Goal) -> St:
842  (rest, wst) = got
843  match wst:
844    case WEof{buf, nn, num}:
845      word.burst.eof.at(word.drop(rest), buf, nn, num, stack, goal)
846    case WStop{buf, nn, num, ch}:
847      word.burst.stop(rest, ch, buf, nn, num, stack, goal)
848    case WMore{buf, nn, num}:
849      word.burst.fail(goal, word.drop(rest), buf)
850
851# plain digits and letters in one loop. The delimiter is consed back once
852def word.burst(
853  txt: String,
854  buf: List<&2, Char>,
855  nn: U32,
856  num: V.Num,
857  stack: List<&2, Frame>,
858  goal: Goal,
859  bad: Bool
860) -> St:
861  match bad:
862    case True{}:
863      word.burst.dead(goal, word.drop(txt), buf)
864    case False{}:
865      word.burst.at(word.run(txt, WMore{buf, nn, num}), stack, goal)
866
867# a backslash in a span: copy the prefix, or drop it when skipping
868def span.esc(
869  +cut: String,
870  +nn: U32,
871  stack: List<&2, Frame>,
872  goal: Goal,
873  done: Bool
874) -> St:
875  match goal:
876    case GNext{}:
877      S{MEsc{copy.rev(cut, nn, U32.is_zero(nn), [])}, stack, goal, done, False{}}
878    case GSkip{depth}:
879      S{MSkEsc{}, stack, goal, done, False{}}
880
881# the next state after one character of a span. The unread tail stays with
882# the stepper, so the usual character does not keep it
883def span.advance(
884  key: Skind,
885  cut: String,
886  nn: U32,
887  stack: List<&2, Frame>,
888  goal: Goal,
889  done: Bool
890) -> St:
891  match key:
892    case KGo{}:
893      S{MSpan{cut, U32.add(nn, 1)}, stack, goal, done, False{}}
894    case KEnd{}:
895      str.place.here(PSpan{cut, nn}, stack, goal)
896    case KEsc{}:
897      span.esc(cut, nn, stack, goal, done)
898    case KBad{}:
899      err.st(goal)
900
901# the first character inside a string. A normal one opens the span on this
902# suffix; the stepper then walks the tail, so the keep is once per string
903def quote.go(
904  key: Skind,
905  ch: Char,
906  tl: String,
907  stack: List<&2, Frame>,
908  goal: Goal,
909  done: Bool
910) -> St:
911  match key:
912    case KGo{}:
913      S{MSpan{SCon{ch, tl}, 1}, stack, goal, done, False{}}
914    case KEnd{}:
915      str.place.here(PSpan{"", 0}, stack, goal)
916    case KEsc{}:
917      span.esc("", 0, stack, goal, done)
918    case KBad{}:
919      err.st(goal)
920
921# the first character inside a string
922def quote.first(
923  +ch: Char,
924  tl: String,
925  stack: List<&2, Frame>,
926  goal: Goal,
927  done: Bool
928) -> St:
929  quote.go(span.kind.of(ch), ch, tl, stack, goal, done)
930
931# the next state after one character of a string being skipped
932def sk.advance(key: Skind, stack: List<&2, Frame>, goal: Goal, done: Bool) -> St:
933  match key:
934    case KGo{}:
935      S{MSkStr{}, stack, goal, done, False{}}
936    case KEnd{}:
937      str.skip.here(stack, goal)
938    case KEsc{}:
939      S{MSkEsc{}, stack, goal, done, False{}}
940    case KBad{}:
941      err.st(goal)
942
943# one character of a string that is being decoded
944def copy.hit.go(
945  +ch: Char,
946  +tl: String,
947  buf: List<&2, Char>,
948  stack: List<&2, Frame>,
949  goal: Goal,
950  done: Bool,
951  cls: Lex.Class,
952  ctrl: Bool
953) -> St:
954  match cls ctrl:
955    case Lex.CQuote{} c:
956      str.place(POwn{Lex.text(buf)}, tl, stack, goal)
957    case Lex.CBack{} False{}:
958      S{MEsc{buf}, stack, goal, done, False{}}
959    case k True{}:
960      err.st(goal)
961    case k False{}:
962      S{MCopy{ch <> buf}, stack, goal, done, False{}}
963
964# one character of a string that is being decoded
965def copy.hit(
966  +ch: Char,
967  +tl: String,
968  buf: List<&2, Char>,
969  stack: List<&2, Frame>,
970  goal: Goal,
971  done: Bool,
972  bad: Bool,
973  cls: Lex.Class,
974  ctrl: Bool
975) -> St:
976  match bad:
977    case True{}:
978      err.st(goal)
979    case False{}:
980      copy.hit.go(ch, tl, buf, stack, goal, done, cls, ctrl)
981
982# the character after a backslash, once `u` has been noticed
983def esc.put(
984  _tl: String,
985  buf: List<&2, Char>,
986  stack: List<&2, Frame>,
987  goal: Goal,
988  done: Bool,
989  got: Maybe<&2, Char>
990) -> St:
991  match got:
992    case Some{ch}:
993      S{MCopy{ch <> buf}, stack, goal, done, False{}}
994    case None{}:
995      err.st(goal)
996
997# the character after a backslash
998def esc.hit.go(
999  +ch: Char,
1000  +tl: String,
1001  buf: List<&2, Char>,
1002  stack: List<&2, Frame>,
1003  goal: Goal,
1004  done: Bool,
1005  is_u: Bool
1006) -> St:
1007  match is_u:
1008    case True{}:
1009      S{MUni{3n, 0, buf}, stack, goal, done, False{}}
1010    case False{}:
1011      esc.put(tl, buf, stack, goal, done, Lex.unescape(ch))
1012
1013# the character after a backslash
1014def esc.hit(
1015  +ch: Char,
1016  +tl: String,
1017  buf: List<&2, Char>,
1018  stack: List<&2, Frame>,
1019  goal: Goal,
1020  done: Bool,
1021  bad: Bool
1022) -> St:
1023  match bad:
1024    case True{}:
1025      err.st(goal)
1026    case False{}:
1027      esc.hit.go(ch, tl, buf, stack, goal, done, U32.is_eq(Char.to_u32(ch), 117))
1028
1029# a `\u` code point that is not a surrogate
1030def uni.end.go(
1031  _tl: String,
1032  cp: U32,
1033  buf: List<&2, Char>,
1034  stack: List<&2, Frame>,
1035  goal: Goal,
1036  done: Bool,
1037  sur: Lex.Sur
1038) -> St:
1039  match sur:
1040    case Lex.SOk{}:
1041      S{MCopy{Char.from_u32(cp) <> buf}, stack, goal, done, False{}}
1042    case Lex.SHi{}:
1043      S{MHi{cp, buf}, stack, goal, done, False{}}
1044    case Lex.SLo{}:
1045      err.st(goal)
1046
1047# four hex digits are in
1048def uni.end(
1049  tl: String,
1050  +cp: U32,
1051  buf: List<&2, Char>,
1052  stack: List<&2, Frame>,
1053  goal: Goal,
1054  done: Bool
1055) -> St:
1056  uni.end.go(tl, cp, buf, stack, goal, done, Lex.unicode.sur(cp))
1057
1058# one more hex digit, or the code point
1059def uni.put(
1060  tl: String,
1061  left: Nat,
1062  acc: U32,
1063  dig: U32,
1064  buf: List<&2, Char>,
1065  stack: List<&2, Frame>,
1066  goal: Goal,
1067  done: Bool
1068) -> St:
1069  match left:
1070    case 0n:
1071      uni.end(tl, (acc * 16 + dig : U32), buf, stack, goal, done)
1072    case 1n+p:
1073      S{MUni{p, (acc * 16 + dig : U32), buf}, stack, goal, done, False{}}
1074
1075# one hex digit of a `\u` escape
1076def uni.hit.go(
1077  +ch: Char,
1078  +tl: String,
1079  left: Nat,
1080  acc: U32,
1081  buf: List<&2, Char>,
1082  stack: List<&2, Frame>,
1083  goal: Goal,
1084  done: Bool,
1085  ok: Bool
1086) -> St:
1087  match ok:
1088    case True{}:
1089      uni.put(tl, left, acc, Lex.hex(ch), buf, stack, goal, done)
1090    case False{}:
1091      err.st(goal)
1092
1093# one hex digit of a `\u` escape
1094def uni.hit(
1095  +ch: Char,
1096  +tl: String,
1097  left: Nat,
1098  acc: U32,
1099  buf: List<&2, Char>,
1100  stack: List<&2, Frame>,
1101  goal: Goal,
1102  done: Bool,
1103  bad: Bool
1104) -> St:
1105  match bad:
1106    case True{}:
1107      err.st(goal)
1108    case False{}:
1109      uni.hit.go(ch, tl, left, acc, buf, stack, goal, done, Lex.hex.ok(Char.to_u32(ch)))
1110
1111# the character after a high surrogate, on a live cursor
1112def hi.hit.go(
1113  _tl: String,
1114  hi: U32,
1115  buf: List<&2, Char>,
1116  stack: List<&2, Frame>,
1117  goal: Goal,
1118  done: Bool,
1119  cls: Lex.Class
1120) -> St:
1121  match cls:
1122    case Lex.CBack{}:
1123      S{MHiEsc{hi, buf}, stack, goal, done, False{}}
1124    case other:
1125      err.st(goal)
1126
1127# the character after a high surrogate: it has to be a backslash
1128def hi.hit(
1129  tl: String,
1130  hi: U32,
1131  buf: List<&2, Char>,
1132  stack: List<&2, Frame>,
1133  goal: Goal,
1134  done: Bool,
1135  bad: Bool,
1136  cls: Lex.Class
1137) -> St:
1138  match bad:
1139    case True{}:
1140      err.st(goal)
1141    case False{}:
1142      hi.hit.go(tl, hi, buf, stack, goal, done, cls)
1143
1144# the character after the backslash of a pair: it has to be `u`
1145def hiesc.hit.go(
1146  _tl: String,
1147  hi: U32,
1148  buf: List<&2, Char>,
1149  stack: List<&2, Frame>,
1150  goal: Goal,
1151  done: Bool,
1152  is_u: Bool
1153) -> St:
1154  match is_u:
1155    case True{}:
1156      S{MLo{3n, 0, hi, buf}, stack, goal, done, False{}}
1157    case False{}:
1158      err.st(goal)
1159
1160# the character after the backslash of a pair
1161def hiesc.hit(
1162  +ch: Char,
1163  tl: String,
1164  hi: U32,
1165  buf: List<&2, Char>,
1166  stack: List<&2, Frame>,
1167  goal: Goal,
1168  done: Bool,
1169  bad: Bool
1170) -> St:
1171  match bad:
1172    case True{}:
1173      err.st(goal)
1174    case False{}:
1175      hiesc.hit.go(tl, hi, buf, stack, goal, done, U32.is_eq(Char.to_u32(ch), 117))
1176
1177# the low surrogate is in, or it is not one
1178def lo.end.go(
1179  _tl: String,
1180  lo: U32,
1181  hi: U32,
1182  buf: List<&2, Char>,
1183  stack: List<&2, Frame>,
1184  goal: Goal,
1185  done: Bool,
1186  sur: Lex.Sur
1187) -> St:
1188  match sur:
1189    case Lex.SLo{}:
1190      S{MCopy{Char.from_u32(Lex.unicode.pair(hi, lo)) <> buf}, stack, goal, done, False{}}
1191    case other:
1192      err.st(goal)
1193
1194# four hex digits of the low surrogate are in
1195def lo.end(
1196  tl: String,
1197  +lo: U32,
1198  hi: U32,
1199  buf: List<&2, Char>,
1200  stack: List<&2, Frame>,
1201  goal: Goal,
1202  done: Bool
1203) -> St:
1204  lo.end.go(tl, lo, hi, buf, stack, goal, done, Lex.unicode.sur(lo))
1205
1206# one more hex digit of the low surrogate, or the code point
1207def lo.put(
1208  tl: String,
1209  left: Nat,
1210  acc: U32,
1211  hi: U32,
1212  dig: U32,
1213  buf: List<&2, Char>,
1214  stack: List<&2, Frame>,
1215  goal: Goal,
1216  done: Bool
1217) -> St:
1218  match left:
1219    case 0n:
1220      lo.end(tl, (acc * 16 + dig : U32), hi, buf, stack, goal, done)
1221    case 1n+p:
1222      S{MLo{p, (acc * 16 + dig : U32), hi, buf}, stack, goal, done, False{}}
1223
1224# one hex digit of the low surrogate
1225def lo.hit.go(
1226  +ch: Char,
1227  +tl: String,
1228  left: Nat,
1229  acc: U32,
1230  hi: U32,
1231  buf: List<&2, Char>,
1232  stack: List<&2, Frame>,
1233  goal: Goal,
1234  done: Bool,
1235  ok: Bool
1236) -> St:
1237  match ok:
1238    case True{}:
1239      lo.put(tl, left, acc, hi, Lex.hex(ch), buf, stack, goal, done)
1240    case False{}:
1241      err.st(goal)
1242
1243# one hex digit of the low surrogate
1244def lo.hit(
1245  +ch: Char,
1246  +tl: String,
1247  left: Nat,
1248  acc: U32,
1249  hi: U32,
1250  buf: List<&2, Char>,
1251  stack: List<&2, Frame>,
1252  goal: Goal,
1253  done: Bool,
1254  bad: Bool
1255) -> St:
1256  match bad:
1257    case True{}:
1258      err.st(goal)
1259    case False{}:
1260      lo.hit.go(ch, tl, left, acc, hi, buf, stack, goal, done, Lex.hex.ok(Char.to_u32(ch)))
1261
1262# a decoded short escape puts the skip back in the string
1263def sk.esc.put(
1264  _tl: String,
1265  stack: List<&2, Frame>,
1266  goal: Goal,
1267  done: Bool,
1268  got: Maybe<&2, Char>
1269) -> St:
1270  match got:
1271    case Some{ch}:
1272      S{MSkStr{}, stack, goal, done, False{}}
1273    case None{}:
1274      err.st(goal)
1275
1276# a short escape, or the start of `\u`, inside a skipped string
1277def sk.esc.go(
1278  +ch: Char,
1279  tl: String,
1280  stack: List<&2, Frame>,
1281  goal: Goal,
1282  done: Bool,
1283  is_u: Bool
1284) -> St:
1285  match is_u:
1286    case True{}:
1287      S{MSkUni{3n, 0}, stack, goal, done, False{}}
1288    case False{}:
1289      sk.esc.put(tl, stack, goal, done, Lex.unescape(ch))
1290
1291# the character after a backslash in a skipped string
1292def sk.esc(
1293  +ch: Char,
1294  tl: String,
1295  stack: List<&2, Frame>,
1296  goal: Goal,
1297  done: Bool,
1298  bad: Bool
1299) -> St:
1300  match bad:
1301    case True{}:
1302      err.st(goal)
1303    case False{}:
1304      sk.esc.go(ch, tl, stack, goal, done, U32.is_eq(Char.to_u32(ch), 117))
1305
1306# a skipped `\u` that is not a surrogate
1307def sk.uni.end.go(
1308  _tl: String,
1309  stack: List<&2, Frame>,
1310  goal: Goal,
1311  done: Bool,
1312  sur: Lex.Sur,
1313  cp: U32
1314) -> St:
1315  match sur:
1316    case Lex.SOk{}:
1317      S{MSkStr{}, stack, goal, done, False{}}
1318    case Lex.SHi{}:
1319      S{MSkHi{cp}, stack, goal, done, False{}}
1320    case Lex.SLo{}:
1321      err.st(goal)
1322
1323# four hex digits of a skipped `\u` are in
1324def sk.uni.end(
1325  tl: String,
1326  +cp: U32,
1327  stack: List<&2, Frame>,
1328  goal: Goal,
1329  done: Bool
1330) -> St:
1331  sk.uni.end.go(tl, stack, goal, done, Lex.unicode.sur(cp), cp)
1332
1333# one more skipped hex digit, or the code point
1334def sk.uni.put(
1335  tl: String,
1336  left: Nat,
1337  acc: U32,
1338  dig: U32,
1339  stack: List<&2, Frame>,
1340  goal: Goal,
1341  done: Bool
1342) -> St:
1343  match left:
1344    case 0n:
1345      sk.uni.end(tl, (acc * 16 + dig : U32), stack, goal, done)
1346    case 1n+p:
1347      S{MSkUni{p, (acc * 16 + dig : U32)}, stack, goal, done, False{}}
1348
1349# one hex digit of a skipped `\u`
1350def sk.uni.go(
1351  +ch: Char,
1352  tl: String,
1353  left: Nat,
1354  acc: U32,
1355  stack: List<&2, Frame>,
1356  goal: Goal,
1357  done: Bool,
1358  ok: Bool
1359) -> St:
1360  match ok:
1361    case True{}:
1362      sk.uni.put(tl, left, acc, Lex.hex(ch), stack, goal, done)
1363    case False{}:
1364      err.st(goal)
1365
1366# one hex digit of a skipped `\u`
1367def sk.uni(
1368  +ch: Char,
1369  tl: String,
1370  left: Nat,
1371  acc: U32,
1372  stack: List<&2, Frame>,
1373  goal: Goal,
1374  done: Bool,
1375  bad: Bool
1376) -> St:
1377  match bad:
1378    case True{}:
1379      err.st(goal)
1380    case False{}:
1381      sk.uni.go(ch, tl, left, acc, stack, goal, done, Lex.hex.ok(Char.to_u32(ch)))
1382
1383# the character after a skipped high surrogate
1384def sk.hi(stack: List<&2, Frame>, goal: Goal, done: Bool, hi: U32, cls: Lex.Class) -> St:
1385  match cls:
1386    case Lex.CBack{}:
1387      S{MSkHiEsc{hi}, stack, goal, done, False{}}
1388    case other:
1389      err.st(goal)
1390
1391# `u` after the backslash of a skipped pair
1392def sk.hiesc(stack: List<&2, Frame>, goal: Goal, done: Bool, hi: U32, is_u: Bool) -> St:
1393  match is_u:
1394    case True{}:
1395      S{MSkLo{3n, 0, hi}, stack, goal, done, False{}}
1396    case False{}:
1397      err.st(goal)
1398
1399# the low half of a skipped pair is in, or it is not one
1400def sk.lo.end(_tl: String, stack: List<&2, Frame>, goal: Goal, done: Bool, sur: Lex.Sur) -> St:
1401  match sur:
1402    case Lex.SLo{}:
1403      S{MSkStr{}, stack, goal, done, False{}}
1404    case other:
1405      err.st(goal)
1406
1407# one more hex digit of a skipped low surrogate, or the code point
1408def sk.lo.put(
1409  tl: String,
1410  left: Nat,
1411  acc: U32,
1412  hi: U32,
1413  dig: U32,
1414  stack: List<&2, Frame>,
1415  goal: Goal,
1416  done: Bool
1417) -> St:
1418  match left:
1419    case 0n:
1420      sk.lo.end(tl, stack, goal, done, Lex.unicode.sur((acc * 16 + dig : U32)))
1421    case 1n+p:
1422      S{MSkLo{p, (acc * 16 + dig : U32), hi}, stack, goal, done, False{}}
1423
1424# one hex digit of a skipped low surrogate
1425def sk.lo.go(
1426  +ch: Char,
1427  tl: String,
1428  left: Nat,
1429  acc: U32,
1430  hi: U32,
1431  stack: List<&2, Frame>,
1432  goal: Goal,
1433  done: Bool,
1434  ok: Bool
1435) -> St:
1436  match ok:
1437    case True{}:
1438      sk.lo.put(tl, left, acc, hi, Lex.hex(ch), stack, goal, done)
1439    case False{}:
1440      err.st(goal)
1441
1442# the levels of the walk's budget: next.grow runs levels 0 to 32 of leaves of
1443# next.leaf() jumps, so a walk may jump back from a plain word 4 (2^33 - 1)
1444# times, more than the 2^32 - 1 it had. A byte step keeps the count
1445# and shrinks the suffix instead, so a long string does not burn it. The
1446# levels, not the count, are the literal, so a proof about `next` never
1447# expands four billion successors
1448def next.levels() -> Nat:
1449  33n
1450
1451# one character, between events. A plain word is one tail loop. A finished
1452# event returns; everything else tail-calls on the shorter suffix. A string
1453# character and a bracket are not handed to the step twice, so a long row or
1454# a long array moves the suffix instead of keeping it
1455def next.go(fuel: Nat, txt: String, st: St) -> Out:
1456  match fuel txt st:
1457    case n any S{MDone{stop, saved}, stack, goal, done, bad}:
1458      Fin{(stop, Cur{saved, stack, done, bad})}
1459    case n txt S{MRest{stop}, stack, goal, done, bad}:
1460      Fin{(stop, Cur{txt, stack, done, bad})}
1461    case 0n any S{mode, stack, goal, done, bad}:
1462      More{any, S{mode, stack, goal, done, bad}}
1463    case 1n+left any S{MJump{rest, mode}, stack, goal, done, bad}:
1464      next.go(left, rest, S{mode, stack, goal, done, bad})
1465    case 1n+left SNil{} S{MQuote{}, stack, goal, done, bad}:
1466      Fin{bad.cur()}
1467    case 1n+left SCon{ch, tl} S{MQuote{}, stack, goal, done, True{}}:
1468      next.go(1n+left, tl, err.st(goal))
1469    case 1n+left SCon{ch, +tl} S{MQuote{}, stack, goal, done, False{}}:
1470      next.go(1n+left, tl, quote.first(ch, tl, stack, goal, done))
1471    case 1n+left SNil{} S{MIdle{}, stack, goal, done, bad}:
1472      Fin{idle.eof(stack, goal, done, bad)}
1473    case 1n+left SCon{ch, tl} S{MIdle{}, stack, goal, done, True{}}:
1474      next.go(1n+left, tl, err.st(goal))
1475    case 1n+left SCon{ch, tl} S{MIdle{}, stack, goal, done, False{}}:
1476      next.go(1n+left, tl, look.plan(ch, stack, goal, done, Lex.classify(ch)))
1477    case 1n+left txt S{MWord{buf, nn, num}, stack, goal, done, bad}:
1478      next.go(left, "", word.burst(txt, buf, nn, num, stack, goal, bad))
1479    case 1n+left SNil{} S{MSpan{cut, nn}, stack, goal, done, bad}:
1480      Fin{bad.cur()}
1481    case 1n+left SCon{ch, tl} S{MSpan{cut, nn}, stack, goal, done, True{}}:
1482      next.go(1n+left, tl, err.st(goal))
1483    case 1n+left SCon{ch, tl} S{MSpan{cut, nn}, stack, goal, done, False{}}:
1484      next.go(1n+left, tl, span.advance(span.kind.of(ch), cut, nn, stack, goal, done))
1485    case 1n+left SNil{} S{MCopy{buf}, stack, goal, done, bad}:
1486      Fin{bad.cur()}
1487    case 1n+left SCon{+ch, +tl} S{MCopy{buf}, stack, goal, done, bad}:
1488      next.go(1n+left, tl, copy.hit(ch, tl, buf, stack, goal, done, bad, Lex.classify(ch),
1489        U32.is_lt(Char.to_u32(ch), 32)))
1490    case 1n+left SNil{} S{MEsc{buf}, stack, goal, done, bad}:
1491      Fin{bad.cur()}
1492    case 1n+left SCon{+ch, +tl} S{MEsc{buf}, stack, goal, done, bad}:
1493      next.go(1n+left, tl, esc.hit(ch, tl, buf, stack, goal, done, bad))
1494    case 1n+left SNil{} S{MUni{left2, acc, buf}, stack, goal, done, bad}:
1495      Fin{bad.cur()}
1496    case 1n+left SCon{+ch, +tl} S{MUni{left2, acc, buf}, stack, goal, done, bad}:
1497      next.go(1n+left, tl, uni.hit(ch, tl, left2, acc, buf, stack, goal, done, bad))
1498    case 1n+left SNil{} S{MHi{hi, buf}, stack, goal, done, bad}:
1499      Fin{bad.cur()}
1500    case 1n+left SCon{ch, tl} S{MHi{hi, buf}, stack, goal, done, bad}:
1501      next.go(1n+left, tl, hi.hit(tl, hi, buf, stack, goal, done, bad, Lex.classify(ch)))
1502    case 1n+left SNil{} S{MHiEsc{hi, buf}, stack, goal, done, bad}:
1503      Fin{bad.cur()}
1504    case 1n+left SCon{+ch, tl} S{MHiEsc{hi, buf}, stack, goal, done, bad}:
1505      next.go(1n+left, tl, hiesc.hit(ch, tl, hi, buf, stack, goal, done, bad))
1506    case 1n+left SNil{} S{MLo{left2, acc, hi, buf}, stack, goal, done, bad}:
1507      Fin{bad.cur()}
1508    case 1n+left SCon{+ch, +tl} S{MLo{left2, acc, hi, buf}, stack, goal, done, bad}:
1509      next.go(1n+left, tl, lo.hit(ch, tl, left2, acc, hi, buf, stack, goal, done, bad))
1510    case 1n+left SNil{} S{MSkStr{}, stack, goal, done, bad}:
1511      Fin{bad.cur()}
1512    case 1n+left SCon{ch, tl} S{MSkStr{}, stack, goal, done, True{}}:
1513      next.go(1n+left, tl, err.st(goal))
1514    case 1n+left SCon{ch, tl} S{MSkStr{}, stack, goal, done, False{}}:
1515      next.go(1n+left, tl, sk.advance(span.kind.of(ch), stack, goal, done))
1516    case 1n+left SNil{} S{MSkEsc{}, stack, goal, done, bad}:
1517      Fin{bad.cur()}
1518    case 1n+left SCon{+ch, tl} S{MSkEsc{}, stack, goal, done, bad}:
1519      next.go(1n+left, tl, sk.esc(ch, tl, stack, goal, done, bad))
1520    case 1n+left SNil{} S{MSkUni{left2, acc}, stack, goal, done, bad}:
1521      Fin{bad.cur()}
1522    case 1n+left SCon{+ch, tl} S{MSkUni{left2, acc}, stack, goal, done, bad}:
1523      next.go(1n+left, tl, sk.uni(ch, tl, left2, acc, stack, goal, done, bad))
1524    case 1n+left SNil{} S{MSkHi{hi}, stack, goal, done, bad}:
1525      Fin{bad.cur()}
1526    case 1n+left SCon{ch, tl} S{MSkHi{hi}, stack, goal, done, bad}:
1527      next.go(1n+left, tl, sk.hi(stack, goal, done, hi, Lex.classify(ch)))
1528    case 1n+left SNil{} S{MSkHiEsc{hi}, stack, goal, done, bad}:
1529      Fin{bad.cur()}
1530    case 1n+left SCon{+ch, tl} S{MSkHiEsc{hi}, stack, goal, done, bad}:
1531      next.go(1n+left, tl, sk.hiesc(stack, goal, done, hi, U32.is_eq(Char.to_u32(ch), 117)))
1532    case 1n+left SNil{} S{MSkLo{left2, acc, hi}, stack, goal, done, bad}:
1533      Fin{bad.cur()}
1534    case 1n+left SCon{+ch, tl} S{MSkLo{left2, acc, hi}, stack, goal, done, bad}:
1535      next.go(1n+left, tl, sk.lo.go(ch, tl, left2, acc, hi, stack, goal, done,
1536        Lex.hex.ok(Char.to_u32(ch))))
1537
1538# the jumps one leaf of the budget allows. A plain word takes two (the burst
1539# and the jump back), so most events finish in one leaf. A leaf that runs out
1540# is not cut short: next.run resumes it in the next leaf
1541def next.leaf() -> Nat:
1542  4n
1543
1544# a walk with a budget of 2^lvl jumps: one step's budget at depth 0, and
1545# the budget below it twice otherwise. A finished step passes through
1546def next.run(lvl: Nat, res: Out) -> Out:
1547  match lvl res:
1548    case 0n More{txt, st}:
1549      next.go(next.leaf(), txt, st)
1550    case 1n+(+pp) More{txt, st}:
1551      next.run(pp, next.run(pp, More{txt, st}))
1552    case _ Fin{got}:
1553      Fin{got}
1554
1555# a walk that runs level `lvl`, then the next level up, until it finishes or
1556# `left` levels are done. Most events finish at level 0, one step's budget, so
1557# a walk pays for the levels only when it jumps many times
1558def next.grow(left: Nat, +lvl: Nat, res: Out) -> Out:
1559  match left res:
1560    case 1n+pp More{txt, st}:
1561      next.grow(pp, 1n+lvl, next.run(lvl, More{txt, st}))
1562    case _ More{txt, st}:
1563      More{txt, st}
1564    case _ Fin{got}:
1565      Fin{got}
1566
1567# a step's answer. A budget used up is a failure, as it always was
1568def next.fin(res: Out) -> (Stop & Cur):
1569  match res:
1570    case Fin{got}:
1571      got
1572    case More{_, _}:
1573      bad.cur()
1574
1575# a walk from the start of `txt` in state `st`, with the whole budget
1576def next.walk(txt: String, st: St) -> (Stop & Cur):
1577  next.fin(next.grow(next.levels(), 0n, More{txt, st}))
1578
1579# the cursor after an end: nothing left to read, and the root is over
1580def next.ended(cur: Cur) -> Cur:
1581  Cur{_, stack, _, _} = cur
1582  Cur{"", stack, True{}, False{}}
1583
1584# an event from a step. A skip that surfaces here is a broken reader. An
1585# error always comes with the failed cursor, and an end with an ended one, so
1586# both stick: the steps build them that way, and this says so where it is used
1587def next.use(got: (Stop & Cur)) -> (Ev & Cur):
1588  (stop, cur) = got
1589  match stop:
1590    case SEv{EErr{}}:
1591      (EErr{}, Cur{"", Nil{}, False{}, True{}})
1592    case SEv{EEnd{}}:
1593      (EEnd{}, next.ended(cur))
1594    case SEv{ev}:
1595      (ev, cur)
1596    case SSkip{}:
1597      (EErr{}, Cur{"", Nil{}, False{}, True{}})
1598
1599# the next event, unless the cursor has already failed
1600def next.open(rest: String, stack: List<&2, Frame>, done: Bool, bad: Bool) -> (Ev & Cur):
1601  match bad:
1602    case True{}:
1603      (EErr{}, Cur{"", Nil{}, False{}, True{}})
1604    case False{}:
1605      next.use(next.walk(rest, S{MIdle{}, stack, GNext{}, done, False{}}))
1606
1607# the cursor after a skip. An event other than a failure means the skip
1608# stopped on something that was not one value
1609def skip.use(got: (Stop & Cur)) -> Cur:
1610  (stop, cur) = got
1611  match stop:
1612    case SSkip{}:
1613      cur
1614    case SEv{EErr{}}:
1615      cur
1616    case SEv{ev}:
1617      Cur{"", Nil{}, False{}, True{}}
1618
1619# drop one value, unless the cursor has already failed or the root is over
1620def skip.open(rest: String, stack: List<&2, Frame>, done: Bool, bad: Bool) -> Cur:
1621  match done bad:
1622    case d True{}:
1623      Cur{"", Nil{}, False{}, True{}}
1624    case True{} False{}:
1625      Cur{"", Nil{}, False{}, True{}}
1626    case False{} False{}:
1627      skip.use(next.walk(rest, S{MIdle{}, stack, GSkip{0n}, False{}, False{}}))
1628
1629# a cursor at the start of `src`. It holds that string as its unread suffix,
1630# so dropping the caller's own variable does not drop the cursor, and
1631# dropping the cursor drops what it has not read yet
1632def cursor(src: String) -> Cur:
1633  Cur{src, Nil{}, False{}, False{}}
1634
1635# the next event. A comma or a colon is consumed on the way to the event
1636# after it. After the root value, further calls are the end, or an error
1637# when another value follows
1638def next(cur: Cur) -> (Ev & Cur):
1639  Cur{rest, stack, done, bad} = cur
1640  next.open(rest, stack, done, bad)
1641
1642# drop the next value: one scalar, or one array or object with everything
1643# inside it. The cursor is left where that value ended. A string with no
1644# escapes is not copied; a string being skipped is not decoded into a buffer
1645def skip(cur: Cur) -> Cur:
1646  Cur{rest, stack, done, bad} = cur
1647  skip.open(rest, stack, done, bad)
1648
1649# the text of a string, a key, or a number. A span is copied here. None when
1650# the event is not one of those
1651def text(ev: Ev) -> Maybe<&2, String>:
1652  match ev:
1653    case EStr{body}:
1654      Some{body}
1655    case EKey{body}:
1656      Some{body}
1657    case EStrS{src, +nn}:
1658      Some{V.span.str(src, nn, U32.is_zero(nn))}
1659    case EKeyS{src, +nn}:
1660      Some{V.span.str(src, nn, U32.is_zero(nn))}
1661    case ENum{src, +nn}:
1662      Some{V.span.str(src, nn, U32.is_zero(nn))}
1663    case other:
1664      None{}