src/pull.bend
emerging-ezjson@1.1.0.0 · 1,664 lines · 0 definitions
1# src/pull: one JSON text, one event at a time.
2# `cursor` holds the unread suffix of the source. `next` returns one event and
3# the cursor after it. `skip` drops one value. A span event keeps the suffix
4# it points at, so it stays readable while that event is held and is gone
5# when the event is dropped. Commas and colons are not events.
6import Base
7import emerging-ezjson@1.1.0.0/src/value.bend as V
8import emerging-ezjson@1.1.0.0/src/lex.bend as Lex
9
10# where the next token of an open array or object has to be
11type Slot is Data:
12 SVal{}
13 SCom{}
14 SKey{}
15 SCol{}
16
17# an open array or object. `fresh` is the empty container, before any value
18type Frame is Data:
19 FArr{slot: Slot, fresh: Bool}
20 FObj{slot: Slot, fresh: Bool}
21
22# one event. A span is the first `nn` chars of a source suffix, not a copy
23type Ev is Data:
24 ENull{}
25 EBool{b: Bool}
26 ENum{src: String, nn: U32}
27 EStr{body: String}
28 EStrS{src: String, nn: U32}
29 EKey{body: String}
30 EKeyS{src: String, nn: U32}
31 EBeginArr{}
32 EEndArr{}
33 EBeginObj{}
34 EEndObj{}
35 EEnd{}
36 EErr{}
37
38# the cursor. `rest` is the unread suffix. `done` is set once the root value
39# has ended. `bad` sticks after the first error
40type Cur is Data:
41 Cur{rest: String, stack: List<&2, Frame>, done: Bool, bad: Bool}
42
43# read the next event, or drop one whole value
44type Goal is Data:
45 GNext{}
46 GSkip{depth: Nat}
47
48# what a step hands back: an event, or a finished skip
49type Stop is Data:
50 SEv{ev: Ev}
51 SSkip{}
52
53# a bare word, once it has been classified
54type Atom is Data:
55 ANull{}
56 ATrue{}
57 AFalse{}
58 ANum{}
59 ABad{}
60
61# a string's text: a span of the source, or chars decoded from escapes
62type Piece is Data:
63 PSpan{cut: String, nn: U32}
64 POwn{body: String}
65
66# what one character of a string does. The usual one is KGo: stay in the
67# string. The others leave it
68type Skind is Data:
69 KGo{}
70 KEnd{}
71 KEsc{}
72 KBad{}
73
74# where a finished value lands, or that the slot cannot take one
75type Spot is Data:
76 Spot{stack: List<&2, Frame>, root: Bool}
77 SpotBad{}
78
79# what the scanner is inside. Skip-modes (`MSk*`) walk a string they will
80# not keep, so a dropped value does not build its text
81type Mode is Data:
82 MIdle{}
83 # a bare word's characters, reversed, so the unread array is not kept
84 MWord{buf: List<&2, Char>, nn: U32, num: V.Num}
85 MSpan{cut: String, nn: U32}
86 MCopy{buf: List<&2, Char>}
87 MEsc{buf: List<&2, Char>}
88 MUni{left: Nat, acc: U32, buf: List<&2, Char>}
89 MHi{hi: U32, buf: List<&2, Char>}
90 MHiEsc{hi: U32, buf: List<&2, Char>}
91 MLo{left: Nat, acc: U32, hi: U32, buf: List<&2, Char>}
92 MSkStr{}
93 MSkEsc{}
94 MSkUni{left: Nat, acc: U32}
95 MSkHi{hi: U32}
96 MSkHiEsc{hi: U32}
97 MSkLo{left: Nat, acc: U32, hi: U32}
98 MDone{stop: Stop, saved: String}
99 # the cursor rest is the suffix the stepper is already on, so closing a
100 # long string does not keep that suffix a second time
101 MRest{stop: Stop}
102 # the quote is consumed; the next character is the first of the string
103 MQuote{}
104 # resume `mode` on `rest` after a plain word, which the stepper did not
105 # walk byte by byte
106 MJump{rest: String, mode: Mode}
107
108# the scanner's state, apart from the unread suffix it is walking
109type St is Data:
110 S{mode: Mode, stack: List<&2, Frame>, goal: Goal, done: Bool, bad: Bool}
111
112# what a walk hands back: a finished step, or a budget used up, to resume on
113# `txt` in state `st`
114type Out is Type:
115 Fin{got: (Stop & Cur)}
116 More{txt: String, st: St}
117
118# a failed step. The unread suffix is dropped
119def err.st(goal: Goal) -> St:
120 S{MDone{SEv{EErr{}}, ""}, Nil{}, goal, False{}, True{}}
121
122# a failed cursor
123def bad.cur() -> (Stop & Cur):
124 (SEv{EErr{}}, Cur{"", Nil{}, False{}, True{}})
125
126# the first `nn` chars of `src`, reversed, for the escape copier
127def copy.rev(src: String, +nn: U32, zero: Bool, acc: List<&2, Char>) -> List<&2, Char>:
128 match src zero:
129 case s True{}:
130 acc
131 case SNil{} False{}:
132 acc
133 case SCon{h, t} False{}:
134 copy.rev(t, U32.sub(nn, 1), U32.is_zero(U32.sub(nn, 1)), h <> acc)
135
136# a value can start here
137def expect.val.go(stack: List<&2, Frame>) -> Bool:
138 match stack:
139 case Nil{}:
140 True{}
141 case Con{FArr{SVal{}, fresh}, rest}:
142 True{}
143 case Con{FObj{SVal{}, fresh}, rest}:
144 True{}
145 case other:
146 False{}
147
148# a value can start here, and the root is not already finished
149def expect.val(stack: List<&2, Frame>, done: Bool) -> Bool:
150 match done:
151 case True{}:
152 False{}
153 case False{}:
154 expect.val.go(stack)
155
156# a string can start here: a value, or an object key
157def expect.str.go(stack: List<&2, Frame>) -> Bool:
158 match stack:
159 case Nil{}:
160 True{}
161 case Con{FArr{SVal{}, fresh}, rest}:
162 True{}
163 case Con{FObj{SVal{}, fresh}, rest}:
164 True{}
165 case Con{FObj{SKey{}, fresh}, rest}:
166 True{}
167 case other:
168 False{}
169
170# a string can start here, and the root is not already finished
171def expect.str(stack: List<&2, Frame>, done: Bool) -> Bool:
172 match done:
173 case True{}:
174 False{}
175 case False{}:
176 expect.str.go(stack)
177
178# a string can start here for this goal. A skip of one whole value must
179# start a value, so a key there fails at once, as it would once read
180def expect.str.for(stack: List<&2, Frame>, goal: Goal, done: Bool) -> Bool:
181 match goal:
182 case GSkip{0n}:
183 expect.val(stack, done)
184 case _:
185 expect.str(stack, done)
186
187# the stack after a value, and whether that value was the root
188def place.spot(stack: List<&2, Frame>) -> Spot:
189 match stack:
190 case Nil{}:
191 Spot{Nil{}, True{}}
192 case Con{FArr{SVal{}, fresh}, frames}:
193 Spot{FArr{SCom{}, False{}} <> frames, False{}}
194 case Con{FObj{SVal{}, fresh}, frames}:
195 Spot{FObj{SCom{}, False{}} <> frames, False{}}
196 case other:
197 SpotBad{}
198
199# a value has landed in a slot that can take it
200def place.val.ok(ev: Ev, rest: String, goal: Goal, stack: List<&2, Frame>, root: Bool) -> St:
201 match goal:
202 case GNext{}:
203 S{MDone{SEv{ev}, rest}, stack, GNext{}, root, False{}}
204 case GSkip{0n}:
205 S{MDone{SSkip{}, rest}, stack, GSkip{0n}, root, False{}}
206 case GSkip{1n+left}:
207 S{MIdle{}, stack, GSkip{1n+left}, False{}, False{}}
208
209# a value has landed. `rest` is the suffix the cursor keeps
210def place.val.on(ev: Ev, rest: String, goal: Goal, spot: Spot) -> St:
211 match spot:
212 case SpotBad{}:
213 err.st(goal)
214 case Spot{stack, root}:
215 place.val.ok(ev, rest, goal, stack, root)
216
217# a value has landed
218def place.val(ev: Ev, rest: String, stack: List<&2, Frame>, goal: Goal) -> St:
219 place.val.on(ev, rest, goal, place.spot(stack))
220
221# an object key has landed. The value comes after the colon
222def place.key(ev: Ev, rest: String, frames: List<&2, Frame>, goal: Goal) -> St:
223 match goal:
224 case GNext{}:
225 S{MDone{SEv{ev}, rest}, FObj{SCol{}, False{}} <> frames, goal, False{}, False{}}
226 case GSkip{0n}:
227 err.st(goal)
228 case GSkip{1n+left}:
229 S{MIdle{}, FObj{SCol{}, False{}} <> frames, goal, False{}, False{}}
230
231# a string event, span or owned, as a key or a value
232def str.ev(piece: Piece, key: Bool) -> Ev:
233 match piece key:
234 case PSpan{cut, nn} True{}:
235 EKeyS{cut, nn}
236 case PSpan{cut, nn} False{}:
237 EStrS{cut, nn}
238 case POwn{body} True{}:
239 EKey{body}
240 case POwn{body} False{}:
241 EStr{body}
242
243# a string value inside a skip, once the slot is known
244def str.cont.on(goal: Goal, spot: Spot) -> St:
245 match spot:
246 case SpotBad{}:
247 err.st(goal)
248 case Spot{stack2, root}:
249 S{MIdle{}, stack2, goal, False{}, False{}}
250
251# a string value inside a skip: keep the slot, not the text
252def str.cont(stack: List<&2, Frame>, goal: Goal) -> St:
253 str.cont.on(goal, place.spot(stack))
254
255# a string value. A skip that is still inside a container keeps scanning
256def str.val(ev: Ev, rest: String, stack: List<&2, Frame>, goal: Goal) -> St:
257 match goal:
258 case GSkip{1n+left}:
259 str.cont(stack, goal)
260 case GNext{}:
261 place.val(ev, rest, stack, goal)
262 case GSkip{0n}:
263 place.val(ev, rest, stack, goal)
264
265# a finished string. A key goes to the open object; anything else is a value
266def str.place(piece: Piece, rest: String, stack: List<&2, Frame>, goal: Goal) -> St:
267 match stack:
268 case Con{FObj{SKey{}, fresh}, frames}:
269 place.key(str.ev(piece, True{}), rest, frames, goal)
270 case other:
271 str.val(str.ev(piece, False{}), rest, stack, goal)
272
273# a string a skip will not keep
274def str.skip(rest: String, stack: List<&2, Frame>, goal: Goal) -> St:
275 match goal:
276 case GNext{}:
277 err.st(goal)
278 case GSkip{depth}:
279 str.place(POwn{""}, rest, stack, goal)
280
281# a value has landed, and the cursor rest is the suffix already in hand
282def place.val.ok.here(ev: Ev, goal: Goal, stack: List<&2, Frame>, root: Bool) -> St:
283 match goal:
284 case GNext{}:
285 S{MRest{SEv{ev}}, stack, GNext{}, root, False{}}
286 case GSkip{0n}:
287 S{MRest{SSkip{}}, stack, GSkip{0n}, root, False{}}
288 case GSkip{1n+left}:
289 S{MIdle{}, stack, GSkip{1n+left}, False{}, False{}}
290
291# a value has landed on the suffix in hand, once the slot is known
292def place.val.on.here(ev: Ev, goal: Goal, spot: Spot) -> St:
293 match spot:
294 case SpotBad{}:
295 err.st(goal)
296 case Spot{stack, root}:
297 place.val.ok.here(ev, goal, stack, root)
298
299# a value has landed on the suffix in hand
300def place.val.here(ev: Ev, stack: List<&2, Frame>, goal: Goal) -> St:
301 place.val.on.here(ev, goal, place.spot(stack))
302
303# an object key has landed on the suffix in hand
304def place.key.here(ev: Ev, frames: List<&2, Frame>, goal: Goal) -> St:
305 match goal:
306 case GNext{}:
307 S{MRest{SEv{ev}}, FObj{SCol{}, False{}} <> frames, goal, False{}, False{}}
308 case GSkip{0n}:
309 err.st(goal)
310 case GSkip{1n+left}:
311 S{MIdle{}, FObj{SCol{}, False{}} <> frames, goal, False{}, False{}}
312
313# a string value on the suffix in hand. A skip inside a container keeps scanning
314def str.val.here(ev: Ev, stack: List<&2, Frame>, goal: Goal) -> St:
315 match goal:
316 case GSkip{1n+left}:
317 str.cont(stack, goal)
318 case GNext{}:
319 place.val.here(ev, stack, goal)
320 case GSkip{0n}:
321 place.val.here(ev, stack, goal)
322
323# a finished string whose rest is the suffix the stepper is on
324def str.place.here(piece: Piece, stack: List<&2, Frame>, goal: Goal) -> St:
325 match stack:
326 case Con{FObj{SKey{}, fresh}, frames}:
327 place.key.here(str.ev(piece, True{}), frames, goal)
328 case other:
329 str.val.here(str.ev(piece, False{}), stack, goal)
330
331# a skipped string closed on the suffix in hand
332def str.skip.here(stack: List<&2, Frame>, goal: Goal) -> St:
333 match goal:
334 case GNext{}:
335 err.st(goal)
336 case GSkip{depth}:
337 str.place.here(POwn{""}, stack, goal)
338
339# one character of a string: stay in it, close it, start an escape, or fail
340def span.kind(cls: Lex.Class, ctrl: Bool) -> Skind:
341 match cls ctrl:
342 case Lex.CQuote{} c:
343 KEnd{}
344 case Lex.CBack{} False{}:
345 KEsc{}
346 case k True{}:
347 KBad{}
348 case k False{}:
349 KGo{}
350
351# classify one string character. `ch` is a code point, so the keep is a word
352def span.kind.of(+ch: Char) -> Skind:
353 span.kind(Lex.classify(ch), U32.is_lt(Char.to_u32(ch), 32))
354
355# `]` or `}` of a skip: depth 1 finishes the value, deeper keeps going.
356# The cursor rest is the suffix the stepper is already on
357def close.skip.go(stack: List<&2, Frame>, left: Nat, root: Bool) -> St:
358 match left:
359 case 0n:
360 S{MRest{SSkip{}}, stack, GSkip{0n}, root, False{}}
361 case 1n+more:
362 S{MIdle{}, stack, GSkip{1n+more}, False{}, False{}}
363
364# `]` or `}` of a skip, once the frame has been popped
365def close.skip(stack: List<&2, Frame>, depth: Nat, root: Bool) -> St:
366 match depth:
367 case 0n:
368 err.st(GSkip{0n})
369 case 1n+left:
370 close.skip.go(stack, left, root)
371
372# the parent after a container closes, for a reader
373def close.next(ev: Ev, stack: List<&2, Frame>, root: Bool) -> St:
374 S{MRest{SEv{ev}}, stack, GNext{}, root, False{}}
375
376# a container closed. `frames` is the stack with that frame popped
377def close.out(ev: Ev, frames: List<&2, Frame>, goal: Goal) -> St:
378 match frames goal:
379 case Nil{} GNext{}:
380 close.next(ev, Nil{}, True{})
381 case Nil{} GSkip{depth}:
382 close.skip(Nil{}, depth, True{})
383 case Con{FArr{SVal{}, fresh}, outer} GNext{}:
384 close.next(ev, FArr{SCom{}, False{}} <> outer, False{})
385 case Con{FObj{SVal{}, fresh}, outer} GNext{}:
386 close.next(ev, FObj{SCom{}, False{}} <> outer, False{})
387 case Con{FArr{SVal{}, fresh}, outer} GSkip{depth}:
388 close.skip(FArr{SCom{}, False{}} <> outer, depth, False{})
389 case Con{FObj{SVal{}, fresh}, outer} GSkip{depth}:
390 close.skip(FObj{SCom{}, False{}} <> outer, depth, False{})
391 case frames2 g:
392 err.st(g)
393
394# a `]`. Empty, or just after a value
395def close.arr(stack: List<&2, Frame>, goal: Goal) -> St:
396 match stack:
397 case Con{FArr{SVal{}, True{}}, frames}:
398 close.out(EEndArr{}, frames, goal)
399 case Con{FArr{SCom{}, fresh}, frames}:
400 close.out(EEndArr{}, frames, goal)
401 case other:
402 err.st(goal)
403
404# a `}`. Empty, or just after a value
405def close.obj(stack: List<&2, Frame>, goal: Goal) -> St:
406 match stack:
407 case Con{FObj{SKey{}, True{}}, frames}:
408 close.out(EEndObj{}, frames, goal)
409 case Con{FObj{SCom{}, fresh}, frames}:
410 close.out(EEndObj{}, frames, goal)
411 case other:
412 err.st(goal)
413
414# a comma between values, or between object members
415def punct.comma(stack: List<&2, Frame>, goal: Goal) -> St:
416 match stack:
417 case Con{FArr{SCom{}, fresh}, frames}:
418 S{MIdle{}, FArr{SVal{}, False{}} <> frames, goal, False{}, False{}}
419 case Con{FObj{SCom{}, fresh}, frames}:
420 S{MIdle{}, FObj{SKey{}, False{}} <> frames, goal, False{}, False{}}
421 case other:
422 err.st(goal)
423
424# a colon after a key
425def punct.colon(stack: List<&2, Frame>, goal: Goal) -> St:
426 match stack:
427 case Con{FObj{SCol{}, fresh}, frames}:
428 S{MIdle{}, FObj{SVal{}, False{}} <> frames, goal, False{}, False{}}
429 case other:
430 err.st(goal)
431
432# `[` pushed. A reader returns the begin; a skip counts the container
433def punct.arr.go(stack: List<&2, Frame>, goal: Goal) -> St:
434 match goal:
435 case GNext{}:
436 S{MRest{SEv{EBeginArr{}}}, stack, goal, False{}, False{}}
437 case GSkip{depth}:
438 S{MIdle{}, stack, GSkip{1n+depth}, False{}, False{}}
439
440# `[`, once it is known whether a value may start
441def punct.arr.if(stack: List<&2, Frame>, goal: Goal, ok: Bool) -> St:
442 match ok:
443 case True{}:
444 punct.arr.go(FArr{SVal{}, True{}} <> stack, goal)
445 case False{}:
446 err.st(goal)
447
448# `[`, when a value may start
449def punct.arr(+stack: List<&2, Frame>, goal: Goal, done: Bool) -> St:
450 punct.arr.if(stack, goal, expect.val(stack, done))
451
452# `{` pushed
453def punct.obj.go(stack: List<&2, Frame>, goal: Goal) -> St:
454 match goal:
455 case GNext{}:
456 S{MRest{SEv{EBeginObj{}}}, stack, goal, False{}, False{}}
457 case GSkip{depth}:
458 S{MIdle{}, stack, GSkip{1n+depth}, False{}, False{}}
459
460# `{`, once it is known whether a value may start
461def punct.obj.if(stack: List<&2, Frame>, goal: Goal, ok: Bool) -> St:
462 match ok:
463 case True{}:
464 punct.obj.go(FObj{SKey{}, True{}} <> stack, goal)
465 case False{}:
466 err.st(goal)
467
468# `{`, when a value may start
469def punct.obj(+stack: List<&2, Frame>, goal: Goal, done: Bool) -> St:
470 punct.obj.if(stack, goal, expect.val(stack, done))
471
472# one punctuation token, once the root is still open. The suffix stays
473# with the stepper: a bracket closes on it, and a comma does not keep it
474def punct.go(tok: Lex.Tok, stack: List<&2, Frame>, goal: Goal, done: Bool) -> St:
475 match tok:
476 case Lex.TOpenArr{}:
477 punct.arr(stack, goal, done)
478 case Lex.TOpenObj{}:
479 punct.obj(stack, goal, done)
480 case Lex.TCloseArr{}:
481 close.arr(stack, goal)
482 case Lex.TCloseObj{}:
483 close.obj(stack, goal)
484 case Lex.TColon{}:
485 punct.colon(stack, goal)
486 case Lex.TComma{}:
487 punct.comma(stack, goal)
488 case other:
489 err.st(goal)
490
491# one punctuation token
492def punct.on(tok: Lex.Tok, stack: List<&2, Frame>, goal: Goal, done: Bool) -> St:
493 match done:
494 case True{}:
495 err.st(goal)
496 case False{}:
497 punct.go(tok, stack, goal, done)
498
499# whitespace, kept as the same state
500def idle.space(stack: List<&2, Frame>, goal: Goal, done: Bool) -> St:
501 S{MIdle{}, stack, goal, done, False{}}
502
503# a `"` may open a string here
504def look.quote(stack: List<&2, Frame>, goal: Goal, done: Bool, ok: Bool) -> St:
505 match ok:
506 case True{}:
507 S{MQuote{}, stack, goal, done, False{}}
508 case False{}:
509 err.st(goal)
510
511# a word's first character. The buffer is that character, not the array behind it
512def look.word(+ch: Char, stack: List<&2, Frame>, goal: Goal, done: Bool, ok: Bool) -> St:
513 match ok:
514 case True{}:
515 S{MWord{ch <> [], 1, V.num.ok.step(V.NStart{}, Char.to_u32(ch))},
516 stack, goal, done, False{}}
517 case False{}:
518 err.st(goal)
519
520# one character between tokens. The suffix stays with the stepper: a bracket
521# closes on it, a comma does not keep it, and a word keeps only its own char
522def look.plan(
523 ch: Char,
524 +stack: List<&2, Frame>,
525 +goal: Goal,
526 +done: Bool,
527 cls: Lex.Class
528) -> St:
529 match cls:
530 case Lex.CSpace{}:
531 idle.space(stack, goal, done)
532 case Lex.CPunct{tok}:
533 punct.on(tok, stack, goal, done)
534 case Lex.CQuote{}:
535 look.quote(stack, goal, done, expect.str.for(stack, goal, done))
536 case Lex.CBack{}:
537 err.st(goal)
538 case Lex.COther{}:
539 look.word(ch, stack, goal, done, expect.val(stack, done))
540
541# the end of the text, between tokens
542def idle.eof(stack: List<&2, Frame>, goal: Goal, done: Bool, bad: Bool) -> (Stop & Cur):
543 match goal done bad:
544 case GNext{} True{} False{}:
545 (SEv{EEnd{}}, Cur{"", stack, True{}, False{}})
546 case g d b:
547 bad.cur()
548
549# a number, when the word is not null, true, or false
550def word.kind.num(ok: Bool) -> Atom:
551 match ok:
552 case True{}:
553 ANum{}
554 case False{}:
555 ABad{}
556
557# null, true, false, or a number that kept a legal spelling
558def word.kind.go(num: V.Num, is_null: Bool, is_true: Bool, is_false: Bool) -> Atom:
559 match is_null is_true is_false:
560 case True{} x y:
561 ANull{}
562 case False{} True{} y:
563 ATrue{}
564 case False{} False{} True{}:
565 AFalse{}
566 case False{} False{} False{}:
567 word.kind.num(V.num.ok.done(num))
568
569# what a bare word is
570def word.kind(+cut: String, +nn: U32, num: V.Num) -> Atom:
571 word.kind.go(num, V.span.eq(cut, nn, "null"), V.span.eq(cut, nn, "true"),
572 V.span.eq(cut, nn, "false"))
573
574# a word that ended at the end of the text
575def word.eof.place(
576 atom: Atom,
577 cut: String,
578 nn: U32,
579 stack: List<&2, Frame>,
580 goal: Goal
581) -> St:
582 match atom:
583 case ABad{}:
584 err.st(goal)
585 case ANull{}:
586 place.val(ENull{}, "", stack, goal)
587 case ATrue{}:
588 place.val(EBool{True{}}, "", stack, goal)
589 case AFalse{}:
590 place.val(EBool{False{}}, "", stack, goal)
591 case ANum{}:
592 place.val(ENum{cut, nn}, "", stack, goal)
593
594# the delimiter of a skipped word, once the value is inside the container.
595# The delimiter is consumed. The caller resumes on the suffix after it
596def lay.plan(stack: List<&2, Frame>, goal: Goal, cls: Lex.Class) -> St:
597 match cls:
598 case Lex.CSpace{}:
599 S{MIdle{}, stack, goal, False{}, False{}}
600 case Lex.CPunct{tok}:
601 punct.on(tok, stack, goal, False{})
602 case other:
603 err.st(goal)
604
605# place the word, then take the delimiter, because the skip is still open
606def see.more.on(goal: Goal, cls: Lex.Class, spot: Spot) -> St:
607 match spot:
608 case SpotBad{}:
609 err.st(goal)
610 case Spot{stack, root}:
611 lay.plan(stack, goal, cls)
612
613# place the word, then take the delimiter
614def see.more(stack: List<&2, Frame>, goal: Goal, cls: Lex.Class) -> St:
615 see.more.on(goal, cls, place.spot(stack))
616
617# a code point that ends a bare word: the same bytes `classify` calls
618# punctuation, a quote, a backslash, or whitespace. A form feed stays in the word
619def word.special(+cp: U32) -> Bool:
620 Bool.or(U32.is_eq(cp, 9),
621 Bool.or(U32.is_eq(cp, 10),
622 Bool.or(U32.is_eq(cp, 13),
623 Bool.or(U32.is_eq(cp, 32),
624 Bool.or(U32.is_eq(cp, 34),
625 Bool.or(U32.is_eq(cp, 44),
626 Bool.or(U32.is_eq(cp, 58),
627 Bool.or(U32.is_eq(cp, 91),
628 Bool.or(U32.is_eq(cp, 92),
629 Bool.or(U32.is_eq(cp, 93),
630 Bool.or(U32.is_eq(cp, 123), U32.is_eq(cp, 125))))))))))))
631
632# where a plain word run stopped. A stop keeps the delimiter, already consumed
633type Wst is Data:
634 WMore{buf: List<&2, Char>, nn: U32, num: V.Num}
635 WStop{buf: List<&2, Char>, nn: U32, num: V.Num, ch: Char}
636 WEof{buf: List<&2, Char>, nn: U32, num: V.Num}
637
638# one more plain byte of a word. The buffer grows by that character
639def word.step(ch: Char, +cp: U32, buf: List<&2, Char>, nn: U32, num: V.Num) -> Wst:
640 WMore{ch <> buf, U32.add(nn, 1), V.num.ok.step(num, cp)}
641
642# keep the byte when it is not a delimiter
643def word.keep.go(
644 ch: Char,
645 cp: U32,
646 buf: List<&2, Char>,
647 nn: U32,
648 num: V.Num,
649 keep: Bool
650) -> Wst:
651 match keep:
652 case True{}:
653 word.step(ch, cp, buf, nn, num)
654 case False{}:
655 WStop{buf, nn, num, ch}
656
657# a digit is never a delimiter, so a long number does not walk `classify`
658def word.digit.go(
659 ch: Char,
660 +cp: U32,
661 buf: List<&2, Char>,
662 nn: U32,
663 num: V.Num,
664 dig: Bool
665) -> Wst:
666 match dig:
667 case True{}:
668 word.step(ch, cp, buf, nn, num)
669 case False{}:
670 word.keep.go(ch, cp, buf, nn, num, Bool.not(word.special(cp)))
671
672# one byte of a word: another plain byte, or the delimiter that ends it
673def word.decide(ch: Char, +cp: U32, buf: List<&2, Char>, nn: U32, num: V.Num) -> Wst:
674 word.digit.go(ch, cp, buf, nn, num, V.num.ok.digit(cp))
675
676# the suffix after a plain word. Tail call, so a long number is one loop
677def word.run(txt: String, st: Wst) -> (String & Wst):
678 match txt st:
679 case rest WStop{buf, nn, num, ch}:
680 (rest, WStop{buf, nn, num, ch})
681 case rest WEof{buf, nn, num}:
682 (rest, WEof{buf, nn, num})
683 case SNil{} WMore{buf, nn, num}:
684 ("", WEof{buf, nn, num})
685 case SCon{+c, t} WMore{buf, nn, num}:
686 word.run(t, word.decide(c, Char.to_u32(c), buf, nn, num))
687
688# drop an unread suffix. Tail call, so a failed long value does not stack
689def word.drop(txt: String) -> Bool:
690 match txt:
691 case SNil{}:
692 True{}
693 case SCon{c, t}:
694 word.drop(t)
695
696# drop a word's characters. Tail call, so a long spelling does not stack
697def word.forget(buf: List<&2, Char>) -> Bool:
698 match buf:
699 case Nil{}:
700 True{}
701 case Con{c, t}:
702 word.forget(t)
703
704# a failed word, once its buffer and the unread suffix are gone
705def word.burst.err(goal: Goal, dropped: Bool) -> St:
706 match dropped:
707 case d:
708 err.st(goal)
709
710# a failed word drops the buffer after the unread suffix
711def word.burst.dead(goal: Goal, dropped: Bool, buf: List<&2, Char>) -> St:
712 match dropped:
713 case d:
714 word.burst.err(goal, word.forget(buf))
715
716# a finished event already carries its suffix. Anything else continues on `rest`
717def jump.keep(dropped: Bool, st: St) -> St:
718 match dropped st:
719 case d S{mode, stack, goal, done, bad}:
720 S{mode, stack, goal, done, bad}
721
722# a state that already stopped drops `rest`. Anything else resumes on it
723def jump.unless.done(+rest: String, st: St) -> St:
724 match st:
725 case S{MDone{stop, saved}, stack, goal, done, bad}:
726 jump.keep(word.drop(rest), S{MDone{stop, saved}, stack, goal, done, bad})
727 case S{mode, stack, goal, done, bad}:
728 S{MJump{rest, mode}, stack, goal, done, bad}
729
730# a word that ended because the text did, once its spelling is in hand
731def word.burst.eof.raw(
732 +raw: String,
733 +nn: U32,
734 num: V.Num,
735 stack: List<&2, Frame>,
736 goal: Goal
737) -> St:
738 match goal:
739 case GSkip{1n+left}:
740 word.burst.err(goal, word.drop(raw))
741 case GNext{}:
742 word.eof.place(word.kind(raw, nn, num), raw, nn, stack, goal)
743 case GSkip{0n}:
744 word.eof.place(word.kind(raw, nn, num), raw, nn, stack, goal)
745
746# the empty suffix of an eof is dropped, then the word is placed
747def word.burst.eof.at(
748 dropped: Bool,
749 buf: List<&2, Char>,
750 nn: U32,
751 num: V.Num,
752 stack: List<&2, Frame>,
753 goal: Goal
754) -> St:
755 match dropped:
756 case d:
757 word.burst.eof.raw(Lex.text(buf), nn, num, stack, goal)
758
759# the skip is still inside a container, so the delimiter is taken and the
760# stepper continues. Only this path classifies that byte
761def word.burst.skip(rest: String, +ch: Char, stack: List<&2, Frame>, goal: Goal) -> St:
762 jump.unless.done(rest, see.more(stack, goal, Lex.classify(ch)))
763
764# a value landed on the delimiter. The suffix is consed back once, not copied.
765# The number keeps its own spelling, so it does not hold the rest of the array
766def word.burst.goal(
767 ev: Ev,
768 ch: Char,
769 rest: String,
770 stack: List<&2, Frame>,
771 goal: Goal
772) -> St:
773 match goal:
774 case GNext{}:
775 place.val(ev, SCon{ch, rest}, stack, goal)
776 case GSkip{0n}:
777 place.val(ev, SCon{ch, rest}, stack, goal)
778 case GSkip{1n+left}:
779 word.burst.skip(rest, ch, stack, goal)
780
781# both the spelling and the unread suffix are dropped when the word is not one
782def word.burst.junk(goal: Goal, dropped: Bool, rest: String) -> St:
783 match dropped:
784 case d:
785 word.burst.err(goal, word.drop(rest))
786
787# place the word. A number keeps its own spelling; the delimiter stays unread
788def word.burst.atom(
789 rest: String,
790 ch: Char,
791 raw: String,
792 nn: U32,
793 stack: List<&2, Frame>,
794 goal: Goal,
795 atom: Atom
796) -> St:
797 match atom:
798 case ABad{}:
799 word.burst.junk(goal, word.drop(raw), SCon{ch, rest})
800 case ANull{}:
801 word.burst.goal(ENull{}, ch, rest, stack, goal)
802 case ATrue{}:
803 word.burst.goal(EBool{True{}}, ch, rest, stack, goal)
804 case AFalse{}:
805 word.burst.goal(EBool{False{}}, ch, rest, stack, goal)
806 case ANum{}:
807 word.burst.goal(ENum{raw, nn}, ch, rest, stack, goal)
808
809# the delimiter is classified once, and only when a skip must consume it.
810# The plain bytes never were
811def word.burst.raw(
812 rest: String,
813 ch: Char,
814 +raw: String,
815 +nn: U32,
816 num: V.Num,
817 stack: List<&2, Frame>,
818 goal: Goal
819) -> St:
820 word.burst.atom(rest, ch, raw, nn, stack, goal, word.kind(raw, nn, num))
821
822# a word ended on a delimiter. Its spelling is its own text
823def word.burst.stop(
824 rest: String,
825 ch: Char,
826 buf: List<&2, Char>,
827 nn: U32,
828 num: V.Num,
829 stack: List<&2, Frame>,
830 goal: Goal
831) -> St:
832 word.burst.raw(rest, ch, Lex.text(buf), nn, num, stack, goal)
833
834# both the unread suffix and the buffer are dropped when the word cannot finish
835def word.burst.fail(goal: Goal, dropped: Bool, buf: List<&2, Char>) -> St:
836 match dropped:
837 case d:
838 word.burst.dead(goal, True{}, buf)
839
840# `got` is the suffix after the run, and why it stopped
841def word.burst.at(got: (String & Wst), stack: List<&2, Frame>, goal: Goal) -> St:
842 (rest, wst) = got
843 match wst:
844 case WEof{buf, nn, num}:
845 word.burst.eof.at(word.drop(rest), buf, nn, num, stack, goal)
846 case WStop{buf, nn, num, ch}:
847 word.burst.stop(rest, ch, buf, nn, num, stack, goal)
848 case WMore{buf, nn, num}:
849 word.burst.fail(goal, word.drop(rest), buf)
850
851# plain digits and letters in one loop. The delimiter is consed back once
852def word.burst(
853 txt: String,
854 buf: List<&2, Char>,
855 nn: U32,
856 num: V.Num,
857 stack: List<&2, Frame>,
858 goal: Goal,
859 bad: Bool
860) -> St:
861 match bad:
862 case True{}:
863 word.burst.dead(goal, word.drop(txt), buf)
864 case False{}:
865 word.burst.at(word.run(txt, WMore{buf, nn, num}), stack, goal)
866
867# a backslash in a span: copy the prefix, or drop it when skipping
868def span.esc(
869 +cut: String,
870 +nn: U32,
871 stack: List<&2, Frame>,
872 goal: Goal,
873 done: Bool
874) -> St:
875 match goal:
876 case GNext{}:
877 S{MEsc{copy.rev(cut, nn, U32.is_zero(nn), [])}, stack, goal, done, False{}}
878 case GSkip{depth}:
879 S{MSkEsc{}, stack, goal, done, False{}}
880
881# the next state after one character of a span. The unread tail stays with
882# the stepper, so the usual character does not keep it
883def span.advance(
884 key: Skind,
885 cut: String,
886 nn: U32,
887 stack: List<&2, Frame>,
888 goal: Goal,
889 done: Bool
890) -> St:
891 match key:
892 case KGo{}:
893 S{MSpan{cut, U32.add(nn, 1)}, stack, goal, done, False{}}
894 case KEnd{}:
895 str.place.here(PSpan{cut, nn}, stack, goal)
896 case KEsc{}:
897 span.esc(cut, nn, stack, goal, done)
898 case KBad{}:
899 err.st(goal)
900
901# the first character inside a string. A normal one opens the span on this
902# suffix; the stepper then walks the tail, so the keep is once per string
903def quote.go(
904 key: Skind,
905 ch: Char,
906 tl: String,
907 stack: List<&2, Frame>,
908 goal: Goal,
909 done: Bool
910) -> St:
911 match key:
912 case KGo{}:
913 S{MSpan{SCon{ch, tl}, 1}, stack, goal, done, False{}}
914 case KEnd{}:
915 str.place.here(PSpan{"", 0}, stack, goal)
916 case KEsc{}:
917 span.esc("", 0, stack, goal, done)
918 case KBad{}:
919 err.st(goal)
920
921# the first character inside a string
922def quote.first(
923 +ch: Char,
924 tl: String,
925 stack: List<&2, Frame>,
926 goal: Goal,
927 done: Bool
928) -> St:
929 quote.go(span.kind.of(ch), ch, tl, stack, goal, done)
930
931# the next state after one character of a string being skipped
932def sk.advance(key: Skind, stack: List<&2, Frame>, goal: Goal, done: Bool) -> St:
933 match key:
934 case KGo{}:
935 S{MSkStr{}, stack, goal, done, False{}}
936 case KEnd{}:
937 str.skip.here(stack, goal)
938 case KEsc{}:
939 S{MSkEsc{}, stack, goal, done, False{}}
940 case KBad{}:
941 err.st(goal)
942
943# one character of a string that is being decoded
944def copy.hit.go(
945 +ch: Char,
946 +tl: String,
947 buf: List<&2, Char>,
948 stack: List<&2, Frame>,
949 goal: Goal,
950 done: Bool,
951 cls: Lex.Class,
952 ctrl: Bool
953) -> St:
954 match cls ctrl:
955 case Lex.CQuote{} c:
956 str.place(POwn{Lex.text(buf)}, tl, stack, goal)
957 case Lex.CBack{} False{}:
958 S{MEsc{buf}, stack, goal, done, False{}}
959 case k True{}:
960 err.st(goal)
961 case k False{}:
962 S{MCopy{ch <> buf}, stack, goal, done, False{}}
963
964# one character of a string that is being decoded
965def copy.hit(
966 +ch: Char,
967 +tl: String,
968 buf: List<&2, Char>,
969 stack: List<&2, Frame>,
970 goal: Goal,
971 done: Bool,
972 bad: Bool,
973 cls: Lex.Class,
974 ctrl: Bool
975) -> St:
976 match bad:
977 case True{}:
978 err.st(goal)
979 case False{}:
980 copy.hit.go(ch, tl, buf, stack, goal, done, cls, ctrl)
981
982# the character after a backslash, once `u` has been noticed
983def esc.put(
984 _tl: String,
985 buf: List<&2, Char>,
986 stack: List<&2, Frame>,
987 goal: Goal,
988 done: Bool,
989 got: Maybe<&2, Char>
990) -> St:
991 match got:
992 case Some{ch}:
993 S{MCopy{ch <> buf}, stack, goal, done, False{}}
994 case None{}:
995 err.st(goal)
996
997# the character after a backslash
998def esc.hit.go(
999 +ch: Char,
1000 +tl: String,
1001 buf: List<&2, Char>,
1002 stack: List<&2, Frame>,
1003 goal: Goal,
1004 done: Bool,
1005 is_u: Bool
1006) -> St:
1007 match is_u:
1008 case True{}:
1009 S{MUni{3n, 0, buf}, stack, goal, done, False{}}
1010 case False{}:
1011 esc.put(tl, buf, stack, goal, done, Lex.unescape(ch))
1012
1013# the character after a backslash
1014def esc.hit(
1015 +ch: Char,
1016 +tl: String,
1017 buf: List<&2, Char>,
1018 stack: List<&2, Frame>,
1019 goal: Goal,
1020 done: Bool,
1021 bad: Bool
1022) -> St:
1023 match bad:
1024 case True{}:
1025 err.st(goal)
1026 case False{}:
1027 esc.hit.go(ch, tl, buf, stack, goal, done, U32.is_eq(Char.to_u32(ch), 117))
1028
1029# a `\u` code point that is not a surrogate
1030def uni.end.go(
1031 _tl: String,
1032 cp: U32,
1033 buf: List<&2, Char>,
1034 stack: List<&2, Frame>,
1035 goal: Goal,
1036 done: Bool,
1037 sur: Lex.Sur
1038) -> St:
1039 match sur:
1040 case Lex.SOk{}:
1041 S{MCopy{Char.from_u32(cp) <> buf}, stack, goal, done, False{}}
1042 case Lex.SHi{}:
1043 S{MHi{cp, buf}, stack, goal, done, False{}}
1044 case Lex.SLo{}:
1045 err.st(goal)
1046
1047# four hex digits are in
1048def uni.end(
1049 tl: String,
1050 +cp: U32,
1051 buf: List<&2, Char>,
1052 stack: List<&2, Frame>,
1053 goal: Goal,
1054 done: Bool
1055) -> St:
1056 uni.end.go(tl, cp, buf, stack, goal, done, Lex.unicode.sur(cp))
1057
1058# one more hex digit, or the code point
1059def uni.put(
1060 tl: String,
1061 left: Nat,
1062 acc: U32,
1063 dig: U32,
1064 buf: List<&2, Char>,
1065 stack: List<&2, Frame>,
1066 goal: Goal,
1067 done: Bool
1068) -> St:
1069 match left:
1070 case 0n:
1071 uni.end(tl, (acc * 16 + dig : U32), buf, stack, goal, done)
1072 case 1n+p:
1073 S{MUni{p, (acc * 16 + dig : U32), buf}, stack, goal, done, False{}}
1074
1075# one hex digit of a `\u` escape
1076def uni.hit.go(
1077 +ch: Char,
1078 +tl: String,
1079 left: Nat,
1080 acc: U32,
1081 buf: List<&2, Char>,
1082 stack: List<&2, Frame>,
1083 goal: Goal,
1084 done: Bool,
1085 ok: Bool
1086) -> St:
1087 match ok:
1088 case True{}:
1089 uni.put(tl, left, acc, Lex.hex(ch), buf, stack, goal, done)
1090 case False{}:
1091 err.st(goal)
1092
1093# one hex digit of a `\u` escape
1094def uni.hit(
1095 +ch: Char,
1096 +tl: String,
1097 left: Nat,
1098 acc: U32,
1099 buf: List<&2, Char>,
1100 stack: List<&2, Frame>,
1101 goal: Goal,
1102 done: Bool,
1103 bad: Bool
1104) -> St:
1105 match bad:
1106 case True{}:
1107 err.st(goal)
1108 case False{}:
1109 uni.hit.go(ch, tl, left, acc, buf, stack, goal, done, Lex.hex.ok(Char.to_u32(ch)))
1110
1111# the character after a high surrogate, on a live cursor
1112def hi.hit.go(
1113 _tl: String,
1114 hi: U32,
1115 buf: List<&2, Char>,
1116 stack: List<&2, Frame>,
1117 goal: Goal,
1118 done: Bool,
1119 cls: Lex.Class
1120) -> St:
1121 match cls:
1122 case Lex.CBack{}:
1123 S{MHiEsc{hi, buf}, stack, goal, done, False{}}
1124 case other:
1125 err.st(goal)
1126
1127# the character after a high surrogate: it has to be a backslash
1128def hi.hit(
1129 tl: String,
1130 hi: U32,
1131 buf: List<&2, Char>,
1132 stack: List<&2, Frame>,
1133 goal: Goal,
1134 done: Bool,
1135 bad: Bool,
1136 cls: Lex.Class
1137) -> St:
1138 match bad:
1139 case True{}:
1140 err.st(goal)
1141 case False{}:
1142 hi.hit.go(tl, hi, buf, stack, goal, done, cls)
1143
1144# the character after the backslash of a pair: it has to be `u`
1145def hiesc.hit.go(
1146 _tl: String,
1147 hi: U32,
1148 buf: List<&2, Char>,
1149 stack: List<&2, Frame>,
1150 goal: Goal,
1151 done: Bool,
1152 is_u: Bool
1153) -> St:
1154 match is_u:
1155 case True{}:
1156 S{MLo{3n, 0, hi, buf}, stack, goal, done, False{}}
1157 case False{}:
1158 err.st(goal)
1159
1160# the character after the backslash of a pair
1161def hiesc.hit(
1162 +ch: Char,
1163 tl: String,
1164 hi: U32,
1165 buf: List<&2, Char>,
1166 stack: List<&2, Frame>,
1167 goal: Goal,
1168 done: Bool,
1169 bad: Bool
1170) -> St:
1171 match bad:
1172 case True{}:
1173 err.st(goal)
1174 case False{}:
1175 hiesc.hit.go(tl, hi, buf, stack, goal, done, U32.is_eq(Char.to_u32(ch), 117))
1176
1177# the low surrogate is in, or it is not one
1178def lo.end.go(
1179 _tl: String,
1180 lo: U32,
1181 hi: U32,
1182 buf: List<&2, Char>,
1183 stack: List<&2, Frame>,
1184 goal: Goal,
1185 done: Bool,
1186 sur: Lex.Sur
1187) -> St:
1188 match sur:
1189 case Lex.SLo{}:
1190 S{MCopy{Char.from_u32(Lex.unicode.pair(hi, lo)) <> buf}, stack, goal, done, False{}}
1191 case other:
1192 err.st(goal)
1193
1194# four hex digits of the low surrogate are in
1195def lo.end(
1196 tl: String,
1197 +lo: U32,
1198 hi: U32,
1199 buf: List<&2, Char>,
1200 stack: List<&2, Frame>,
1201 goal: Goal,
1202 done: Bool
1203) -> St:
1204 lo.end.go(tl, lo, hi, buf, stack, goal, done, Lex.unicode.sur(lo))
1205
1206# one more hex digit of the low surrogate, or the code point
1207def lo.put(
1208 tl: String,
1209 left: Nat,
1210 acc: U32,
1211 hi: U32,
1212 dig: U32,
1213 buf: List<&2, Char>,
1214 stack: List<&2, Frame>,
1215 goal: Goal,
1216 done: Bool
1217) -> St:
1218 match left:
1219 case 0n:
1220 lo.end(tl, (acc * 16 + dig : U32), hi, buf, stack, goal, done)
1221 case 1n+p:
1222 S{MLo{p, (acc * 16 + dig : U32), hi, buf}, stack, goal, done, False{}}
1223
1224# one hex digit of the low surrogate
1225def lo.hit.go(
1226 +ch: Char,
1227 +tl: String,
1228 left: Nat,
1229 acc: U32,
1230 hi: U32,
1231 buf: List<&2, Char>,
1232 stack: List<&2, Frame>,
1233 goal: Goal,
1234 done: Bool,
1235 ok: Bool
1236) -> St:
1237 match ok:
1238 case True{}:
1239 lo.put(tl, left, acc, hi, Lex.hex(ch), buf, stack, goal, done)
1240 case False{}:
1241 err.st(goal)
1242
1243# one hex digit of the low surrogate
1244def lo.hit(
1245 +ch: Char,
1246 +tl: String,
1247 left: Nat,
1248 acc: U32,
1249 hi: U32,
1250 buf: List<&2, Char>,
1251 stack: List<&2, Frame>,
1252 goal: Goal,
1253 done: Bool,
1254 bad: Bool
1255) -> St:
1256 match bad:
1257 case True{}:
1258 err.st(goal)
1259 case False{}:
1260 lo.hit.go(ch, tl, left, acc, hi, buf, stack, goal, done, Lex.hex.ok(Char.to_u32(ch)))
1261
1262# a decoded short escape puts the skip back in the string
1263def sk.esc.put(
1264 _tl: String,
1265 stack: List<&2, Frame>,
1266 goal: Goal,
1267 done: Bool,
1268 got: Maybe<&2, Char>
1269) -> St:
1270 match got:
1271 case Some{ch}:
1272 S{MSkStr{}, stack, goal, done, False{}}
1273 case None{}:
1274 err.st(goal)
1275
1276# a short escape, or the start of `\u`, inside a skipped string
1277def sk.esc.go(
1278 +ch: Char,
1279 tl: String,
1280 stack: List<&2, Frame>,
1281 goal: Goal,
1282 done: Bool,
1283 is_u: Bool
1284) -> St:
1285 match is_u:
1286 case True{}:
1287 S{MSkUni{3n, 0}, stack, goal, done, False{}}
1288 case False{}:
1289 sk.esc.put(tl, stack, goal, done, Lex.unescape(ch))
1290
1291# the character after a backslash in a skipped string
1292def sk.esc(
1293 +ch: Char,
1294 tl: String,
1295 stack: List<&2, Frame>,
1296 goal: Goal,
1297 done: Bool,
1298 bad: Bool
1299) -> St:
1300 match bad:
1301 case True{}:
1302 err.st(goal)
1303 case False{}:
1304 sk.esc.go(ch, tl, stack, goal, done, U32.is_eq(Char.to_u32(ch), 117))
1305
1306# a skipped `\u` that is not a surrogate
1307def sk.uni.end.go(
1308 _tl: String,
1309 stack: List<&2, Frame>,
1310 goal: Goal,
1311 done: Bool,
1312 sur: Lex.Sur,
1313 cp: U32
1314) -> St:
1315 match sur:
1316 case Lex.SOk{}:
1317 S{MSkStr{}, stack, goal, done, False{}}
1318 case Lex.SHi{}:
1319 S{MSkHi{cp}, stack, goal, done, False{}}
1320 case Lex.SLo{}:
1321 err.st(goal)
1322
1323# four hex digits of a skipped `\u` are in
1324def sk.uni.end(
1325 tl: String,
1326 +cp: U32,
1327 stack: List<&2, Frame>,
1328 goal: Goal,
1329 done: Bool
1330) -> St:
1331 sk.uni.end.go(tl, stack, goal, done, Lex.unicode.sur(cp), cp)
1332
1333# one more skipped hex digit, or the code point
1334def sk.uni.put(
1335 tl: String,
1336 left: Nat,
1337 acc: U32,
1338 dig: U32,
1339 stack: List<&2, Frame>,
1340 goal: Goal,
1341 done: Bool
1342) -> St:
1343 match left:
1344 case 0n:
1345 sk.uni.end(tl, (acc * 16 + dig : U32), stack, goal, done)
1346 case 1n+p:
1347 S{MSkUni{p, (acc * 16 + dig : U32)}, stack, goal, done, False{}}
1348
1349# one hex digit of a skipped `\u`
1350def sk.uni.go(
1351 +ch: Char,
1352 tl: String,
1353 left: Nat,
1354 acc: U32,
1355 stack: List<&2, Frame>,
1356 goal: Goal,
1357 done: Bool,
1358 ok: Bool
1359) -> St:
1360 match ok:
1361 case True{}:
1362 sk.uni.put(tl, left, acc, Lex.hex(ch), stack, goal, done)
1363 case False{}:
1364 err.st(goal)
1365
1366# one hex digit of a skipped `\u`
1367def sk.uni(
1368 +ch: Char,
1369 tl: String,
1370 left: Nat,
1371 acc: U32,
1372 stack: List<&2, Frame>,
1373 goal: Goal,
1374 done: Bool,
1375 bad: Bool
1376) -> St:
1377 match bad:
1378 case True{}:
1379 err.st(goal)
1380 case False{}:
1381 sk.uni.go(ch, tl, left, acc, stack, goal, done, Lex.hex.ok(Char.to_u32(ch)))
1382
1383# the character after a skipped high surrogate
1384def sk.hi(stack: List<&2, Frame>, goal: Goal, done: Bool, hi: U32, cls: Lex.Class) -> St:
1385 match cls:
1386 case Lex.CBack{}:
1387 S{MSkHiEsc{hi}, stack, goal, done, False{}}
1388 case other:
1389 err.st(goal)
1390
1391# `u` after the backslash of a skipped pair
1392def sk.hiesc(stack: List<&2, Frame>, goal: Goal, done: Bool, hi: U32, is_u: Bool) -> St:
1393 match is_u:
1394 case True{}:
1395 S{MSkLo{3n, 0, hi}, stack, goal, done, False{}}
1396 case False{}:
1397 err.st(goal)
1398
1399# the low half of a skipped pair is in, or it is not one
1400def sk.lo.end(_tl: String, stack: List<&2, Frame>, goal: Goal, done: Bool, sur: Lex.Sur) -> St:
1401 match sur:
1402 case Lex.SLo{}:
1403 S{MSkStr{}, stack, goal, done, False{}}
1404 case other:
1405 err.st(goal)
1406
1407# one more hex digit of a skipped low surrogate, or the code point
1408def sk.lo.put(
1409 tl: String,
1410 left: Nat,
1411 acc: U32,
1412 hi: U32,
1413 dig: U32,
1414 stack: List<&2, Frame>,
1415 goal: Goal,
1416 done: Bool
1417) -> St:
1418 match left:
1419 case 0n:
1420 sk.lo.end(tl, stack, goal, done, Lex.unicode.sur((acc * 16 + dig : U32)))
1421 case 1n+p:
1422 S{MSkLo{p, (acc * 16 + dig : U32), hi}, stack, goal, done, False{}}
1423
1424# one hex digit of a skipped low surrogate
1425def sk.lo.go(
1426 +ch: Char,
1427 tl: String,
1428 left: Nat,
1429 acc: U32,
1430 hi: U32,
1431 stack: List<&2, Frame>,
1432 goal: Goal,
1433 done: Bool,
1434 ok: Bool
1435) -> St:
1436 match ok:
1437 case True{}:
1438 sk.lo.put(tl, left, acc, hi, Lex.hex(ch), stack, goal, done)
1439 case False{}:
1440 err.st(goal)
1441
1442# the levels of the walk's budget: next.grow runs levels 0 to 32 of leaves of
1443# next.leaf() jumps, so a walk may jump back from a plain word 4 (2^33 - 1)
1444# times, more than the 2^32 - 1 it had. A byte step keeps the count
1445# and shrinks the suffix instead, so a long string does not burn it. The
1446# levels, not the count, are the literal, so a proof about `next` never
1447# expands four billion successors
1448def next.levels() -> Nat:
1449 33n
1450
1451# one character, between events. A plain word is one tail loop. A finished
1452# event returns; everything else tail-calls on the shorter suffix. A string
1453# character and a bracket are not handed to the step twice, so a long row or
1454# a long array moves the suffix instead of keeping it
1455def next.go(fuel: Nat, txt: String, st: St) -> Out:
1456 match fuel txt st:
1457 case n any S{MDone{stop, saved}, stack, goal, done, bad}:
1458 Fin{(stop, Cur{saved, stack, done, bad})}
1459 case n txt S{MRest{stop}, stack, goal, done, bad}:
1460 Fin{(stop, Cur{txt, stack, done, bad})}
1461 case 0n any S{mode, stack, goal, done, bad}:
1462 More{any, S{mode, stack, goal, done, bad}}
1463 case 1n+left any S{MJump{rest, mode}, stack, goal, done, bad}:
1464 next.go(left, rest, S{mode, stack, goal, done, bad})
1465 case 1n+left SNil{} S{MQuote{}, stack, goal, done, bad}:
1466 Fin{bad.cur()}
1467 case 1n+left SCon{ch, tl} S{MQuote{}, stack, goal, done, True{}}:
1468 next.go(1n+left, tl, err.st(goal))
1469 case 1n+left SCon{ch, +tl} S{MQuote{}, stack, goal, done, False{}}:
1470 next.go(1n+left, tl, quote.first(ch, tl, stack, goal, done))
1471 case 1n+left SNil{} S{MIdle{}, stack, goal, done, bad}:
1472 Fin{idle.eof(stack, goal, done, bad)}
1473 case 1n+left SCon{ch, tl} S{MIdle{}, stack, goal, done, True{}}:
1474 next.go(1n+left, tl, err.st(goal))
1475 case 1n+left SCon{ch, tl} S{MIdle{}, stack, goal, done, False{}}:
1476 next.go(1n+left, tl, look.plan(ch, stack, goal, done, Lex.classify(ch)))
1477 case 1n+left txt S{MWord{buf, nn, num}, stack, goal, done, bad}:
1478 next.go(left, "", word.burst(txt, buf, nn, num, stack, goal, bad))
1479 case 1n+left SNil{} S{MSpan{cut, nn}, stack, goal, done, bad}:
1480 Fin{bad.cur()}
1481 case 1n+left SCon{ch, tl} S{MSpan{cut, nn}, stack, goal, done, True{}}:
1482 next.go(1n+left, tl, err.st(goal))
1483 case 1n+left SCon{ch, tl} S{MSpan{cut, nn}, stack, goal, done, False{}}:
1484 next.go(1n+left, tl, span.advance(span.kind.of(ch), cut, nn, stack, goal, done))
1485 case 1n+left SNil{} S{MCopy{buf}, stack, goal, done, bad}:
1486 Fin{bad.cur()}
1487 case 1n+left SCon{+ch, +tl} S{MCopy{buf}, stack, goal, done, bad}:
1488 next.go(1n+left, tl, copy.hit(ch, tl, buf, stack, goal, done, bad, Lex.classify(ch),
1489 U32.is_lt(Char.to_u32(ch), 32)))
1490 case 1n+left SNil{} S{MEsc{buf}, stack, goal, done, bad}:
1491 Fin{bad.cur()}
1492 case 1n+left SCon{+ch, +tl} S{MEsc{buf}, stack, goal, done, bad}:
1493 next.go(1n+left, tl, esc.hit(ch, tl, buf, stack, goal, done, bad))
1494 case 1n+left SNil{} S{MUni{left2, acc, buf}, stack, goal, done, bad}:
1495 Fin{bad.cur()}
1496 case 1n+left SCon{+ch, +tl} S{MUni{left2, acc, buf}, stack, goal, done, bad}:
1497 next.go(1n+left, tl, uni.hit(ch, tl, left2, acc, buf, stack, goal, done, bad))
1498 case 1n+left SNil{} S{MHi{hi, buf}, stack, goal, done, bad}:
1499 Fin{bad.cur()}
1500 case 1n+left SCon{ch, tl} S{MHi{hi, buf}, stack, goal, done, bad}:
1501 next.go(1n+left, tl, hi.hit(tl, hi, buf, stack, goal, done, bad, Lex.classify(ch)))
1502 case 1n+left SNil{} S{MHiEsc{hi, buf}, stack, goal, done, bad}:
1503 Fin{bad.cur()}
1504 case 1n+left SCon{+ch, tl} S{MHiEsc{hi, buf}, stack, goal, done, bad}:
1505 next.go(1n+left, tl, hiesc.hit(ch, tl, hi, buf, stack, goal, done, bad))
1506 case 1n+left SNil{} S{MLo{left2, acc, hi, buf}, stack, goal, done, bad}:
1507 Fin{bad.cur()}
1508 case 1n+left SCon{+ch, +tl} S{MLo{left2, acc, hi, buf}, stack, goal, done, bad}:
1509 next.go(1n+left, tl, lo.hit(ch, tl, left2, acc, hi, buf, stack, goal, done, bad))
1510 case 1n+left SNil{} S{MSkStr{}, stack, goal, done, bad}:
1511 Fin{bad.cur()}
1512 case 1n+left SCon{ch, tl} S{MSkStr{}, stack, goal, done, True{}}:
1513 next.go(1n+left, tl, err.st(goal))
1514 case 1n+left SCon{ch, tl} S{MSkStr{}, stack, goal, done, False{}}:
1515 next.go(1n+left, tl, sk.advance(span.kind.of(ch), stack, goal, done))
1516 case 1n+left SNil{} S{MSkEsc{}, stack, goal, done, bad}:
1517 Fin{bad.cur()}
1518 case 1n+left SCon{+ch, tl} S{MSkEsc{}, stack, goal, done, bad}:
1519 next.go(1n+left, tl, sk.esc(ch, tl, stack, goal, done, bad))
1520 case 1n+left SNil{} S{MSkUni{left2, acc}, stack, goal, done, bad}:
1521 Fin{bad.cur()}
1522 case 1n+left SCon{+ch, tl} S{MSkUni{left2, acc}, stack, goal, done, bad}:
1523 next.go(1n+left, tl, sk.uni(ch, tl, left2, acc, stack, goal, done, bad))
1524 case 1n+left SNil{} S{MSkHi{hi}, stack, goal, done, bad}:
1525 Fin{bad.cur()}
1526 case 1n+left SCon{ch, tl} S{MSkHi{hi}, stack, goal, done, bad}:
1527 next.go(1n+left, tl, sk.hi(stack, goal, done, hi, Lex.classify(ch)))
1528 case 1n+left SNil{} S{MSkHiEsc{hi}, stack, goal, done, bad}:
1529 Fin{bad.cur()}
1530 case 1n+left SCon{+ch, tl} S{MSkHiEsc{hi}, stack, goal, done, bad}:
1531 next.go(1n+left, tl, sk.hiesc(stack, goal, done, hi, U32.is_eq(Char.to_u32(ch), 117)))
1532 case 1n+left SNil{} S{MSkLo{left2, acc, hi}, stack, goal, done, bad}:
1533 Fin{bad.cur()}
1534 case 1n+left SCon{+ch, tl} S{MSkLo{left2, acc, hi}, stack, goal, done, bad}:
1535 next.go(1n+left, tl, sk.lo.go(ch, tl, left2, acc, hi, stack, goal, done,
1536 Lex.hex.ok(Char.to_u32(ch))))
1537
1538# the jumps one leaf of the budget allows. A plain word takes two (the burst
1539# and the jump back), so most events finish in one leaf. A leaf that runs out
1540# is not cut short: next.run resumes it in the next leaf
1541def next.leaf() -> Nat:
1542 4n
1543
1544# a walk with a budget of 2^lvl jumps: one step's budget at depth 0, and
1545# the budget below it twice otherwise. A finished step passes through
1546def next.run(lvl: Nat, res: Out) -> Out:
1547 match lvl res:
1548 case 0n More{txt, st}:
1549 next.go(next.leaf(), txt, st)
1550 case 1n+(+pp) More{txt, st}:
1551 next.run(pp, next.run(pp, More{txt, st}))
1552 case _ Fin{got}:
1553 Fin{got}
1554
1555# a walk that runs level `lvl`, then the next level up, until it finishes or
1556# `left` levels are done. Most events finish at level 0, one step's budget, so
1557# a walk pays for the levels only when it jumps many times
1558def next.grow(left: Nat, +lvl: Nat, res: Out) -> Out:
1559 match left res:
1560 case 1n+pp More{txt, st}:
1561 next.grow(pp, 1n+lvl, next.run(lvl, More{txt, st}))
1562 case _ More{txt, st}:
1563 More{txt, st}
1564 case _ Fin{got}:
1565 Fin{got}
1566
1567# a step's answer. A budget used up is a failure, as it always was
1568def next.fin(res: Out) -> (Stop & Cur):
1569 match res:
1570 case Fin{got}:
1571 got
1572 case More{_, _}:
1573 bad.cur()
1574
1575# a walk from the start of `txt` in state `st`, with the whole budget
1576def next.walk(txt: String, st: St) -> (Stop & Cur):
1577 next.fin(next.grow(next.levels(), 0n, More{txt, st}))
1578
1579# the cursor after an end: nothing left to read, and the root is over
1580def next.ended(cur: Cur) -> Cur:
1581 Cur{_, stack, _, _} = cur
1582 Cur{"", stack, True{}, False{}}
1583
1584# an event from a step. A skip that surfaces here is a broken reader. An
1585# error always comes with the failed cursor, and an end with an ended one, so
1586# both stick: the steps build them that way, and this says so where it is used
1587def next.use(got: (Stop & Cur)) -> (Ev & Cur):
1588 (stop, cur) = got
1589 match stop:
1590 case SEv{EErr{}}:
1591 (EErr{}, Cur{"", Nil{}, False{}, True{}})
1592 case SEv{EEnd{}}:
1593 (EEnd{}, next.ended(cur))
1594 case SEv{ev}:
1595 (ev, cur)
1596 case SSkip{}:
1597 (EErr{}, Cur{"", Nil{}, False{}, True{}})
1598
1599# the next event, unless the cursor has already failed
1600def next.open(rest: String, stack: List<&2, Frame>, done: Bool, bad: Bool) -> (Ev & Cur):
1601 match bad:
1602 case True{}:
1603 (EErr{}, Cur{"", Nil{}, False{}, True{}})
1604 case False{}:
1605 next.use(next.walk(rest, S{MIdle{}, stack, GNext{}, done, False{}}))
1606
1607# the cursor after a skip. An event other than a failure means the skip
1608# stopped on something that was not one value
1609def skip.use(got: (Stop & Cur)) -> Cur:
1610 (stop, cur) = got
1611 match stop:
1612 case SSkip{}:
1613 cur
1614 case SEv{EErr{}}:
1615 cur
1616 case SEv{ev}:
1617 Cur{"", Nil{}, False{}, True{}}
1618
1619# drop one value, unless the cursor has already failed or the root is over
1620def skip.open(rest: String, stack: List<&2, Frame>, done: Bool, bad: Bool) -> Cur:
1621 match done bad:
1622 case d True{}:
1623 Cur{"", Nil{}, False{}, True{}}
1624 case True{} False{}:
1625 Cur{"", Nil{}, False{}, True{}}
1626 case False{} False{}:
1627 skip.use(next.walk(rest, S{MIdle{}, stack, GSkip{0n}, False{}, False{}}))
1628
1629# a cursor at the start of `src`. It holds that string as its unread suffix,
1630# so dropping the caller's own variable does not drop the cursor, and
1631# dropping the cursor drops what it has not read yet
1632def cursor(src: String) -> Cur:
1633 Cur{src, Nil{}, False{}, False{}}
1634
1635# the next event. A comma or a colon is consumed on the way to the event
1636# after it. After the root value, further calls are the end, or an error
1637# when another value follows
1638def next(cur: Cur) -> (Ev & Cur):
1639 Cur{rest, stack, done, bad} = cur
1640 next.open(rest, stack, done, bad)
1641
1642# drop the next value: one scalar, or one array or object with everything
1643# inside it. The cursor is left where that value ended. A string with no
1644# escapes is not copied; a string being skipped is not decoded into a buffer
1645def skip(cur: Cur) -> Cur:
1646 Cur{rest, stack, done, bad} = cur
1647 skip.open(rest, stack, done, bad)
1648
1649# the text of a string, a key, or a number. A span is copied here. None when
1650# the event is not one of those
1651def text(ev: Ev) -> Maybe<&2, String>:
1652 match ev:
1653 case EStr{body}:
1654 Some{body}
1655 case EKey{body}:
1656 Some{body}
1657 case EStrS{src, +nn}:
1658 Some{V.span.str(src, nn, U32.is_zero(nn))}
1659 case EKeyS{src, +nn}:
1660 Some{V.span.str(src, nn, U32.is_zero(nn))}
1661 case ENum{src, +nn}:
1662 Some{V.span.str(src, nn, U32.is_zero(nn))}
1663 case other:
1664 None{}