scan.bend
yaml-for-bend@0.1.0.0 · 1,484 lines · 0 definitions
1# The scanner: YAML text to tokens.
2#
3# It follows the design of libyaml's scanner. One step fetches the next
4# token. Block collections open and close by indentation, so the scanner
5# keeps a stack of indentations and adds BSeqStart, BMapStart and BEnd
6# tokens as they change. A "simple key" (a key without "?") is only known to
7# be a key when its ":" arrives, so the scanner remembers where each
8# possible key started and, at the ":", inserts a KeyTok (and BMapStart)
9# before the key's first token.
10#
11# It reads YAML 1.2 as the YAML test suite has it: tabs separate tokens
12# but never indent, a flow collection's lines and a quoted scalar's indent
13# past the block around them, a comment follows a blank, anchors take any
14# character but blanks and flow indicators, ":" is a value indicator right
15# after a JSON-like node in flow, and reserved directives are skipped.
16#
17# An error ends the tokens with a Broken token, so the parser still sees
18# the tokens before it.
19
20import Base
21
22import yaml-for-bend@0.1.0.0/token.bend as T
23
24# The text left to read, where it starts, and how many characters are left.
25type Cur is Data:
26 Cur{s: String, at: T.Mark, left: Nat}
27
28# A place a simple key may start: whether one still may, whether it must (a
29# block key at the indentation), the number its first token has, and where.
30type SKey is Data:
31 SKey{possible: Bool, required: Bool, number: Nat, at: T.Mark, tab: Bool}
32
33# The scanner: the cursor, the block indentation plus one (0 outside any
34# block collection) and the ones it returns to, how deep in flow collections
35# it is, whether a simple key may start here, one possible key per flow
36# level (the block level last), the tokens so far (newest first) and their
37# count.
38type Sc is Data:
39 Sc{cur: Cur, ind: Nat, inds: List<&2, Nat>, flow: Nat, allowed: Bool, tab: Bool, keys: List<&2, SKey>, out: List<&2, T.Token>, n: Nat}
40
41# A step goes on, or stops: at the stream's end or at an error, as the last
42# token says.
43type Step is Data:
44 Go{st: Sc}
45 Stop{st: Sc}
46
47type Lay is Data:
48 Lay{ind: Nat, inds: List<&2, Nat>, out: List<&2, T.Token>, n: Nat}
49
50# Between tokens
51# --------------
52
53type Skipped is Data:
54 Skipped{cur: Cur, allowed: Bool, tab: Bool, nl: Bool, sp: Nat, bad: Bool}
55
56# Plain scalars
57# -------------
58
59# Reading a plain scalar: the cursor, the text so far (reversed), where the
60# last word ended, the blanks after it (reversed), whether a line break came
61# since, how many more breaks did, and whether a word is being read.
62type Pl is Data:
63 Pl{cur: Cur, acc: String, end: T.Mark, ws: String, lead: Bool, brks: Nat, word: Bool, bad: Bool}
64 PlDone{cur: Cur, acc: String, end: T.Mark, lead: Bool}
65 PlBad{error: T.Error}
66
67# Quoted scalars
68# --------------
69
70# Reading a quoted scalar: the cursor, the text so far (reversed), the
71# blanks waiting (reversed), whether a line break came since the last text
72# and how many more did, whether that break was escaped, and whether text
73# (not blanks) is being read.
74type Qs is Data:
75 Qs{cur: Cur, acc: String, ws: String, lead: Bool, brks: Nat, esc: Bool, text: Bool, bad: Bool}
76 QsDone{cur: Cur, acc: String}
77 QsBad{error: T.Error}
78
79# Block scalars
80# -------------
81
82type Chomp is Data:
83 Clip{}
84 Strip{}
85 Keep{}
86
87# A block scalar's header: how to chomp, the indentation indicator (0 when
88# there's none), and the cursor after it; or an error.
89type Head is Data:
90 Head{chomp: Chomp, inc: Nat, cur: Cur}
91 HeadBad{why: String}
92
93type BPhase is Data:
94 Breaks{}
95 LineStart{}
96 Content{}
97
98# Reading a block scalar: the cursor, the text so far (reversed), the
99# content's indentation plus one (0 until known), the deepest blank line, whether a
100# line break waits, whether the last line started with a blank, how many
101# more breaks wait, and the phase: blank lines, a line's start, or content.
102type Bs is Data:
103 Bs{cur: Cur, acc: String, ind: Nat, max: Nat, lead: Bool, lblank: Bool, brks: Nat, phase: BPhase}
104 BsDone{acc: String, lead: Bool, brks: Nat, cur: Cur}
105 BsBad{error: T.Error}
106
107# Anchors, aliases and tags
108# -------------------------
109
110# What a scan took: the text it read and how many characters that was.
111type Taken is Data:
112 Taken{text: String, n: Nat}
113
114# The step after +v+, when it goes on.
115def then(v: Step, f: Sc -> Step) -> Step:
116 match v:
117 case Go{st}:
118 f(st)
119 case Stop{st}:
120 Stop{st}
121
122# +yes+ when +hit+, else +no+; only the one chosen runs.
123def when(-A: Type, hit: Bool, yes: Unit -> A, no: Unit -> A) -> A:
124 match hit:
125 case True{}:
126 yes(Unit{})
127 case False{}:
128 no(Unit{})
129
130# Characters
131# ----------
132
133def code(c: Char) -> U32:
134 match c:
135 case Chr{x}:
136 x
137
138# The character +n+ places on, or NUL past the end.
139def peek(s: String, n: Nat) -> Char:
140 match s:
141 case SNil{}:
142 Chr{0}
143 case SCon{c, t}:
144 match n:
145 case 0n:
146 c
147 case 1n+k:
148 peek(t, k)
149
150def is_blank(c: Char) -> Bool:
151 match c:
152 case ' ':
153 True{}
154 case '\t':
155 True{}
156 case _:
157 False{}
158
159def is_break(c: Char) -> Bool:
160 Char.is_eq(c, '\n')
161
162def is_end(c: Char) -> Bool:
163 U32.is_eq(code(c), 0)
164
165def is_blankz(+c: Char) -> Bool:
166 Bool.or(is_blank(c), Bool.or(is_break(c), is_end(c)))
167
168def is_breakz(+c: Char) -> Bool:
169 Bool.or(is_break(c), is_end(c))
170
171def is_flow.ind(c: Char) -> Bool:
172 match c:
173 case ',':
174 True{}
175 case '[':
176 True{}
177 case ']':
178 True{}
179 case '{':
180 True{}
181 case '}':
182 True{}
183 case _:
184 False{}
185
186def between(+x: U32, lo: U32, hi: U32) -> Bool:
187 Bool.and(U32.is_ge(x, lo), U32.is_le(x, hi))
188
189def is_digit(c: Char) -> Bool:
190 Char.is_digit(c)
191
192def is_hex(+c: Char) -> Bool:
193 Bool.or(Char.is_digit(c), Bool.or(between(code(c), 65, 70), between(code(c), 97, 102)))
194
195# A letter, digit or "-": what a tag handle's name is made of.
196def is_word(+c: Char) -> Bool:
197 Bool.or(Char.is_alpha(c), Bool.or(Char.is_digit(c), Char.is_eq(c, '-')))
198
199def is_in(c: Char, s: String) -> Bool:
200 String.contains(s, SCon{c, SNil{}})
201
202# A URI character other than "%" (which starts an escape).
203def is_uri(+c: Char) -> Bool:
204 Bool.or(is_word(c), is_in(c, "#;/?:@&=+$,_.!~*'()[]"))
205
206# A tag character: a URI character but "!" and the flow indicators.
207def is_tag(+c: Char) -> Bool:
208 Bool.or(is_word(c), is_in(c, "#;/?:@&=+$_.~*'()"))
209
210# What YAML allows in a document: tab, line feed, and printable characters.
211def is_printable(c: Char) -> Bool:
212 match c:
213 case Chr{+x}:
214 Bool.or(Bool.or(U32.is_eq(x, 9), U32.is_eq(x, 10)),
215 Bool.or(between(x, 32, 126), Bool.or(U32.is_eq(x, 133), Bool.or(between(x, 160, 55295),
216 Bool.or(between(x, 57344, 65533), between(x, 65536, 1114111))))))
217
218def hex.digit(c: Char) -> U32:
219 +x = code(c)
220 U32.sub(x, Bool.pick(U32, U32.is_ge(x, 97), 87, Bool.pick(U32, U32.is_ge(x, 65), 55, 48)))
221
222def hex.value(s: String, acc: U32) -> U32:
223 match s:
224 case SNil{}:
225 acc
226 case SCon{c, t}:
227 hex.value(t, U32.add(U32.mul(acc, 16), hex.digit(c)))
228
229def all.hex(s: String) -> Bool:
230 match s:
231 case SNil{}:
232 True{}
233 case SCon{+c, t}:
234 Bool.and(is_hex(c), all.hex(t))
235
236# "---" or "..." (+c+ thrice) and then a blank, a break or the end.
237def marker(+s: String, +c: Char) -> Bool:
238 Bool.and(Bool.and(Char.is_eq(peek(s, 0n), c), Char.is_eq(peek(s, 1n), c)), Bool.and(Char.is_eq(peek(s, 2n), c), is_blankz(peek(s, 3n))))
239
240def is_marker(+s: String) -> Bool:
241 Bool.or(marker(s, '-'), marker(s, '.'))
242
243# Line breaks as "\n": "\r\n" and a lone "\r" both become one, as YAML reads
244# them.
245def lf(s: String) -> String:
246 match s:
247 case SNil{}:
248 SNil{}
249 case SCon{c, t}:
250 match c:
251 case '\r':
252 match t:
253 case SCon{'\n', u}:
254 SCon{'\n', lf(u)}
255 case _:
256 SCon{'\n', lf(t)}
257 case _:
258 SCon{c, lf(t)}
259
260def strip.bom(s: String) -> String:
261 match s:
262 case SCon{c, +t}:
263 match c:
264 case Chr{+x}:
265 Bool.pick(String, U32.is_eq(x, 65279), t, SCon{Chr{x}, t})
266 case SNil{}:
267 SNil{}
268
269# The cursor
270# ----------
271
272def past(at: T.Mark, c: Char) -> T.Mark:
273 match at:
274 case T.Mark{line, col, idx}:
275 match c:
276 case '\n':
277 T.Mark{1n+line, 0n, 1n+idx}
278 case _:
279 T.Mark{line, 1n+col, 1n+idx}
280
281def adv(c: Cur) -> Cur:
282 match c:
283 case Cur{s, at, left}:
284 match s:
285 case SNil{}:
286 Cur{SNil{}, at, left}
287 case SCon{h, t}:
288 Cur{t, past(at, h), Nat.sub(left, 1n)}
289
290def adv.n(n: Nat, c: Cur) -> Cur:
291 match n:
292 case 0n:
293 c
294 case 1n+k:
295 adv.n(k, adv(c))
296
297def cur.s(c: Cur) -> String:
298 match c:
299 case Cur{s, at, left}:
300 s
301
302def cur.at(c: Cur) -> T.Mark:
303 match c:
304 case Cur{s, at, left}:
305 at
306
307def cur.left(c: Cur) -> Nat:
308 match c:
309 case Cur{s, at, left}:
310 left
311
312def cur.col(c: Cur) -> Nat:
313 T.mark.col(cur.at(c))
314
315# The scanner's state
316# -------------------
317
318def sc.cur(st: Sc) -> Cur:
319 match st:
320 case Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}:
321 cur
322
323def sc.s(st: Sc) -> String:
324 cur.s(sc.cur(st))
325
326def sc.at(st: Sc) -> T.Mark:
327 cur.at(sc.cur(st))
328
329def sc.col(st: Sc) -> Nat:
330 cur.col(sc.cur(st))
331
332def sc.ind(st: Sc) -> Nat:
333 match st:
334 case Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}:
335 ind
336
337def sc.flow(st: Sc) -> Nat:
338 match st:
339 case Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}:
340 flow
341
342def sc.in_flow(st: Sc) -> Bool:
343 Nat.is_gt(sc.flow(st), 0n)
344
345def sc.tab(st: Sc) -> Bool:
346 match st:
347 case Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}:
348 tab
349
350def sc.allowed(st: Sc) -> Bool:
351 match st:
352 case Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}:
353 allowed
354
355def sc.keys(st: Sc) -> List<&2, SKey>:
356 match st:
357 case Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}:
358 keys
359
360def sc.out(st: Sc) -> List<&2, T.Token>:
361 match st:
362 case Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}:
363 out
364
365def sc.with.cur(st: Sc, c: Cur) -> Sc:
366 match st:
367 case Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}:
368 Sc{c, ind, inds, flow, allowed, tab, keys, out, n}
369
370def sc.allow(st: Sc, b: Bool) -> Sc:
371 match st:
372 case Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}:
373 Sc{cur, ind, inds, flow, b, tab, keys, out, n}
374
375def sc.with.keys(st: Sc, ks: List<&2, SKey>) -> Sc:
376 match st:
377 case Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}:
378 Sc{cur, ind, inds, flow, allowed, tab, ks, out, n}
379
380def skip(+st: Sc, k: Nat) -> Sc:
381 sc.with.cur(st, adv.n(k, sc.cur(st)))
382
383# A token from +start+ to +end+.
384def emit.span(st: Sc, tok: T.Tok, start: T.Mark, end: T.Mark) -> Sc:
385 match st:
386 case Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}:
387 Sc{cur, ind, inds, flow, allowed, tab, keys, T.Token{tok, start, end} <> out, 1n+n}
388
389# A token from +start+ to here.
390def emit(+st: Sc, tok: T.Tok, start: T.Mark) -> Sc:
391 emit.span(st, tok, start, sc.at(st))
392
393# A one-character token here.
394def single(+st: Sc, tok: T.Tok) -> Sc:
395 emit(skip(st, 1n), tok, sc.at(st))
396
397def fail(+st: Sc, why: String, +at: T.Mark) -> Step:
398 Stop{emit(st, T.Broken{T.Error{why, at, T.Syntax{}}}, at)}
399
400def fail.with(+st: Sc, e: T.Error) -> Step:
401 match e:
402 case T.Error{why, at, cause}:
403 fail(st, why, at)
404
405# "found character that cannot start any token"
406def cannot(+st: Sc) -> Step:
407 fail(st, "found character that cannot start any token while scanning for the next token", sc.at(st))
408
409# The token before this one: whether it was a JSON-like node (a quoted
410# scalar or a flow collection), after which ":" is a value indicator even
411# without a space.
412def json.tok(t: T.Tok) -> Bool:
413 match t:
414 case T.ScalarTok{text, style}:
415 match style:
416 case T.Single{}:
417 True{}
418 case T.Double{}:
419 True{}
420 case _:
421 False{}
422 case T.FSeqEnd{}:
423 True{}
424 case T.FMapEnd{}:
425 True{}
426 case _:
427 False{}
428
429def json.before(out: List<&2, T.Token>) -> Bool:
430 match out:
431 case Nil{}:
432 False{}
433 case Con{t, rest}:
434 match t:
435 case T.Token{tok, start, end}:
436 json.tok(tok)
437
438# Simple keys
439# -----------
440
441# The possible key on top, which a ":" would make a key.
442def key.top(keys: List<&2, SKey>) -> SKey:
443 match keys:
444 case Nil{}:
445 SKey{False{}, False{}, 0n, T.Mark{0n, 0n, 0n}, False{}}
446 case Con{k, rest}:
447 k
448
449def key.or(r: Result<&2, &2, T.Error, List<&2, SKey>>, +st: Sc) -> Step:
450 match r:
451 case Fail{e}:
452 fail.with(st, e)
453 case Done{ks}:
454 Go{sc.with.keys(st, ks)}
455
456def key.removed(keys: List<&2, SKey>) -> Result<&2, &2, T.Error, List<&2, SKey>>:
457 match keys:
458 case Nil{}:
459 Done{[]}
460 case Con{k, rest}:
461 match k:
462 case SKey{+possible, +required, number, +at, tab}:
463 when(Result<&2, &2, T.Error, List<&2, SKey>>, Bool.and(possible, required),
464 u => Fail{T.Error{"could not find expected ':' while scanning a simple key", at, T.Syntax{}}},
465 u => Done{SKey{False{}, required, number, at, tab} <> rest})
466
467# The key on top can no longer be; an error if it had to be.
468def remove.key(+st: Sc) -> Step:
469 key.or(key.removed(sc.keys(st)), st)
470
471def key.put(st: Sc, required: Bool) -> Sc:
472 match st:
473 case Sc{+cur, ind, inds, flow, allowed, +tab, keys, out, +n}:
474 Sc{cur, ind, inds, flow, allowed, tab, SKey{True{}, required, n, cur.at(cur), tab} <> List.drop(&2, SKey, keys, 1n), out, n}
475
476def key.saved(r: Step, +required: Bool) -> Step:
477 match r:
478 case Go{st}:
479 Go{key.put(st, required)}
480 case Stop{st}:
481 Stop{st}
482
483# A simple key may start here. It must be one (no "?" can come later) when
484# it starts a block line at the indentation.
485def save.key(+st: Sc) -> Step:
486 +required = Bool.and(Nat.is_eq(sc.flow(st), 0n), Nat.is_eq(sc.ind(st), 1n+sc.col(st)))
487 when(Step, sc.allowed(st), u => key.saved(remove.key(st), required), u => Go{st})
488
489def stale.cons(k: SKey, r: Result<&2, &2, T.Error, List<&2, SKey>>) -> Result<&2, &2, T.Error, List<&2, SKey>>:
490 match r:
491 case Fail{e}:
492 Fail{e}
493 case Done{ks}:
494 Done{k <> ks}
495
496# Keys left behind: a simple key stays on one line and within 1024
497# characters.
498def stale.go(keys: List<&2, SKey>, +at: T.Mark) -> Result<&2, &2, T.Error, List<&2, SKey>>:
499 match keys:
500 case Nil{}:
501 Done{[]}
502 case Con{k, rest}:
503 match k:
504 case SKey{+possible, +required, number, +kat, tab}:
505 +old = Bool.and(possible, Bool.or(Nat.is_lt(T.mark.line(kat), T.mark.line(at)), Nat.is_lt(Nat.add(T.mark.idx(kat), 1024n), T.mark.idx(at))))
506 when(Result<&2, &2, T.Error, List<&2, SKey>>, Bool.and(old, required),
507 u => Fail{T.Error{"could not find expected ':' while scanning a simple key", kat, T.Syntax{}}},
508 u => stale.cons(SKey{Bool.and(possible, Bool.not(old)), required, number, kat, tab}, stale.go(rest, at)))
509
510def stale(+st: Sc) -> Step:
511 key.or(stale.go(sc.keys(st), sc.at(st)), st)
512
513def key.cleared(keys: List<&2, SKey>) -> List<&2, SKey>:
514 match keys:
515 case Nil{}:
516 []
517 case Con{k, rest}:
518 match k:
519 case SKey{possible, required, number, at, tab}:
520 SKey{False{}, required, number, at, tab} <> rest
521
522def flow.push(st: Sc) -> Sc:
523 match st:
524 case Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}:
525 Sc{cur, ind, inds, 1n+flow, allowed, tab, SKey{False{}, False{}, 0n, T.Mark{0n, 0n, 0n}, False{}} <> keys, out, n}
526
527def flow.pop(st: Sc) -> Sc:
528 match st:
529 case Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}:
530 match flow:
531 case 0n:
532 Sc{cur, ind, inds, 0n, allowed, tab, keys, out, n}
533 case 1n+f:
534 Sc{cur, ind, inds, f, allowed, tab, List.drop(&2, SKey, keys, 1n), out, n}
535
536# Indentation
537# -----------
538
539def insert.at(d: Nat, out: List<&2, T.Token>, t: T.Token) -> List<&2, T.Token>:
540 match d:
541 case 0n:
542 t <> out
543 case 1n+k:
544 match out:
545 case Nil{}:
546 [t]
547 case Con{h, rest}:
548 h <> insert.at(k, rest, t)
549
550# A token at the end, or (+number+) before the token with that number.
551def insert(out: List<&2, T.Token>, n: Nat, number: Maybe<&2, Nat>, t: T.Token) -> List<&2, T.Token>:
552 match number:
553 case None{}:
554 t <> out
555 case Some{k}:
556 insert.at(Nat.sub(n, k), out, t)
557
558def roll.do(st: Sc, col: Nat, number: Maybe<&2, Nat>, tok: T.Tok, +at: T.Mark) -> Sc:
559 match st:
560 case Sc{cur, ind, inds, flow, allowed, tab, keys, out, +n}:
561 Sc{cur, 1n+col, ind <> inds, flow, allowed, tab, keys, insert(out, n, number, T.Token{tok, at, at}), 1n+n}
562
563# A block collection starts at +col+ when that's deeper than the indentation.
564def roll(+st: Sc, +col: Nat, number: Maybe<&2, Nat>, tok: T.Tok, at: T.Mark) -> Sc:
565 when(Sc, Bool.and(Nat.is_eq(sc.flow(st), 0n), Nat.is_lt(sc.ind(st), 1n+col)), u => roll.do(st, col, number, tok, at), u => st)
566
567def unroll.go(inds: List<&2, Nat>, +ind: Nat, +lim: Nat, +at: T.Mark, +out: List<&2, T.Token>, +n: Nat) -> Lay:
568 match inds:
569 case Nil{}:
570 Lay{ind, [], out, n}
571 case Con{+h, +t}:
572 when(Lay, Nat.is_gt(ind, lim), u => unroll.go(t, h, lim, at, T.Token{T.BEnd{}, at, at} <> out, 1n+n), u => Lay{ind, h <> t, out, n})
573
574def unrolled(l: Lay, cur: Cur, flow: Nat, allowed: Bool, tab: Bool, keys: List<&2, SKey>) -> Sc:
575 match l:
576 case Lay{ind, inds, out, n}:
577 Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}
578
579def unroll.do(st: Sc, +lim: Nat) -> Sc:
580 match st:
581 case Sc{+cur, ind, inds, flow, allowed, tab, keys, out, n}:
582 unrolled(unroll.go(inds, ind, lim, cur.at(cur), out, n), cur, flow, allowed, tab, keys)
583
584# Block collections deeper than +lim+ (a column plus one) end.
585def unroll(+st: Sc, +lim: Nat) -> Sc:
586 when(Sc, sc.in_flow(st), u => st, u => unroll.do(st, lim))
587
588def skip.or(eat: Bool, more: Unit -> Skipped, stop: Skipped) -> Skipped:
589 match eat:
590 case True{}:
591 more(Unit{})
592 case False{}:
593 stop
594
595# Blanks, comments and line breaks before a token. A "#" starts a comment
596# after a blank or at a line's start (+gap+). A line break in block context
597# lets a simple key start. Along the way: whether a tab came on the token's
598# line (+tab+), whether a line break came (+nl+), the spaces that start the
599# token's line (+sp+, while +lead+), and whether a tab stood where block
600# indentation belongs (+bad+).
601def skip.go(s: String, +at: T.Mark, +left: Nat, +flow: Bool, +ind: Nat, +allowed: Bool, +comment: Bool, +gap: Bool,
602 +tab: Bool, +nl: Bool, +lead: Bool, +sp: Nat, +bad: Bool) -> Skipped:
603 match s:
604 case SNil{}:
605 Skipped{Cur{SNil{}, at, left}, allowed, tab, nl, sp, False{}}
606 case SCon{+c, +t}:
607 +brk = is_break(c)
608 +space = Char.is_eq(c, ' ')
609 +htab = Char.is_eq(c, '\t')
610 +eat = Bool.or(Bool.or(brk, comment), Bool.or(Bool.or(space, htab), Bool.and(Char.is_eq(c, '#'), gap)))
611 skip.or(eat,
612 u => skip.go(t, past(at, c), Nat.sub(left, 1n), flow, ind, Bool.or(allowed, Bool.and(brk, Bool.not(flow))),
613 Bool.and(Bool.not(brk), Bool.or(comment, Char.is_eq(c, '#'))), True{},
614 Bool.and(Bool.not(brk), Bool.or(tab, htab)), Bool.or(nl, brk), Bool.or(brk, Bool.and(lead, Bool.or(space, htab))),
615 Bool.pick(Nat, brk, 0n, Bool.pick(Nat, Bool.and(Bool.and(lead, space), Bool.not(tab)), 1n+sp, sp)),
616 Bool.and(Bool.not(brk), Bool.or(bad, Bool.and(Bool.and(htab, lead), Bool.and(Bool.not(flow), Nat.is_lt(1n+sp, ind)))))),
617 Skipped{Cur{SCon{c, t}, at, left}, allowed, tab, nl, sp, bad})
618
619def skipped(k: Skipped, st: Sc) -> Sc:
620 match k:
621 case Skipped{cur, allowed, tab, nl, sp, bad}:
622 match st:
623 case Sc{c, ind, inds, flow, a, b, keys, out, n}:
624 Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}
625
626# Whether the last token was a plain or block scalar, which end past the
627# blanks after them, so a "#" here follows a blank.
628def last.plain(out: List<&2, T.Token>) -> Bool:
629 match out:
630 case Nil{}:
631 False{}
632 case Con{t, rest}:
633 match t:
634 case T.Token{tok, start, end}:
635 match tok:
636 case T.ScalarTok{text, style}:
637 match style:
638 case T.Plain{}:
639 True{}
640 case T.Literal{}:
641 True{}
642 case T.Folded{}:
643 True{}
644 case _:
645 False{}
646 case _:
647 False{}
648
649# What the skipped whitespace allows: no tab where block indentation
650# belongs, and in flow, lines indented past the block around them.
651def skip.check(k: Skipped, +st: Sc) -> Step:
652 match k:
653 case Skipped{+cur, +allowed, +tab, +nl, +sp, +bad}:
654 +content = Bool.not(is_end(peek(cur.s(cur), 0n)))
655 +at = cur.at(cur)
656 when(Step, Bool.and(bad, content), u => fail(st, "found a tab character where an indentation space is expected", at),
657 u => when(Step, Bool.and(Bool.and(sc.in_flow(st), nl), Bool.and(content, Nat.is_lt(sp, sc.ind(st)))),
658 u => fail(st, "did not find expected indentation in a flow collection", at),
659 u => Go{skipped(Skipped{cur, allowed, tab, nl, sp, bad}, st)}))
660
661def skip.ws.at(c: Cur, +st: Sc) -> Step:
662 match c:
663 case Cur{s, +at, left}:
664 +lead = Nat.is_eq(T.mark.col(at), 0n)
665 skip.check(skip.go(s, at, left, sc.in_flow(st), sc.ind(st), sc.allowed(st), False{}, Bool.or(lead, last.plain(sc.out(st))),
666 False{}, False{}, lead, 0n, False{}), st)
667
668def skip.ws(+st: Sc) -> Step:
669 skip.ws.at(sc.cur(st), st)
670
671# Folded line breaks: one break is a space, more are newlines (less one).
672def folded.breaks(+brks: Nat) -> String:
673 Bool.pick(String, Nat.is_eq(brks, 0n), " ", String.repeat("\n", brks))
674
675# The blanks or breaks before a word join the text.
676def flush(acc: String, ws: String, lead: Bool, +brks: Nat) -> String:
677 match lead:
678 case True{}:
679 folded.breaks(brks) ++ acc
680 case False{}:
681 ws ++ acc
682
683# In a word: a blank ends it; ": " or a flow indicator in flow ends the scalar.
684def pl.word(cur: Cur, +acc: String, +end: T.Mark, +ws: String, +lead: Bool, +brks: Nat, +flow: Bool) -> Pl:
685 match cur:
686 case Cur{+s, +at, +left}:
687 +c = peek(s, 0n)
688 +c1 = peek(s, 1n)
689 +stop = Bool.or(Bool.and(Char.is_eq(c, ':'), Bool.or(is_blankz(c1), Bool.and(flow, is_flow.ind(c1)))), Bool.and(flow, is_flow.ind(c)))
690 Bool.pick(Pl, is_blankz(c), Pl{Cur{s, at, left}, acc, end, ws, lead, brks, False{}, False{}},
691 Bool.pick(Pl, stop, PlDone{Cur{s, at, left}, acc, end, lead},
692 Pl{Cur{String.drop(s, 1n), past(at, c), Nat.sub(left, 1n)}, SCon{c, flush(acc, ws, lead, brks)}, past(at, c), "", False{}, 0n, True{}, False{}}))
693
694# Between words: blanks and breaks wait to join the text; then the next word
695# starts, unless the scalar ended (less indented, a document marker, a
696# comment, or the end). A tab may not stand where indentation belongs on a
697# line that goes on.
698def pl.blank(cur: Cur, +acc: String, +end: T.Mark, +ws: String, +lead: Bool, +brks: Nat, +bad: Bool, +ind: Nat, +flow: Bool, +start: T.Mark) -> Pl:
699 match cur:
700 case Cur{+s, +at, +left}:
701 +c = peek(s, 0n)
702 +next = {Cur{String.drop(s, 1n), past(at, c), Nat.sub(left, 1n)} : Cur}
703 +here = {Cur{s, at, left} : Cur}
704 +col = T.mark.col(at)
705 +done = Bool.or(Bool.or(is_end(c), Bool.and(Bool.not(flow), Nat.is_lt(col, ind))),
706 Bool.or(Bool.and(Nat.is_eq(col, 0n), is_marker(s)), Char.is_eq(c, '#')))
707 +tabbed = Bool.and(Bool.and(lead, Char.is_eq(c, '\t')), Nat.is_lt(col, ind))
708 Bool.pick(Pl, is_blank(c), Pl{next, acc, end, Bool.pick(String, lead, ws, SCon{c, ws}), lead, brks, False{}, Bool.or(bad, tabbed)},
709 Bool.pick(Pl, is_break(c), Pl{next, acc, end, Bool.pick(String, lead, ws, ""), True{}, Bool.pick(Nat, lead, 1n+brks, brks), False{}, False{}},
710 Bool.pick(Pl, done, PlDone{here, acc, end, lead},
711 Bool.pick(Pl, bad, PlBad{T.Error{"found a tab character that violates indentation while scanning a plain scalar", start, T.Syntax{}}},
712 Pl{here, acc, end, ws, lead, brks, True{}, False{}}))))
713
714def pl.step(p: Pl, +ind: Nat, +flow: Bool, +start: T.Mark) -> Pl:
715 match p:
716 case Pl{cur, acc, end, ws, lead, brks, word, bad}:
717 match word:
718 case True{}:
719 pl.word(cur, acc, end, ws, lead, brks, flow)
720 case False{}:
721 pl.blank(cur, acc, end, ws, lead, brks, bad, ind, flow, start)
722 case _:
723 p
724
725def pl.run(fuel: Nat, +ind: Nat, +flow: Bool, +start: T.Mark, p: Pl) -> Pl:
726 match fuel:
727 case 0n:
728 p
729 case 1n+k:
730 match p:
731 case Pl{cur, acc, end, ws, lead, brks, word, bad}:
732 pl.run(k, ind, flow, start, pl.step(Pl{cur, acc, end, ws, lead, brks, word, bad}, ind, flow, start))
733 case _:
734 p
735
736def plain.done(p: Pl, +st: Sc, start: T.Mark) -> Step:
737 match p:
738 case PlDone{cur, acc, end, lead}:
739 Go{sc.allow(emit.span(sc.with.cur(st, cur), T.ScalarTok{String.reverse(acc), T.Plain{}}, start, end), lead)}
740 case PlBad{e}:
741 fail.with(st, e)
742 case Pl{cur, acc, end, ws, lead, brks, word, bad}:
743 Go{sc.allow(emit.span(sc.with.cur(st, cur), T.ScalarTok{String.reverse(acc), T.Plain{}}, start, end), lead)}
744
745def plain.scan(+st: Sc) -> Step:
746 +cur = sc.cur(st)
747 +at = cur.at(cur)
748 plain.done(pl.run(Nat.add(Nat.mul(cur.left(cur), 2n), 4n), sc.ind(st), sc.in_flow(st), at, Pl{cur, "", at, "", False{}, 0n, True{}, False{}}), st, at)
749
750def plain(+st: Sc) -> Step:
751 then(save.key(st), st => plain.scan(sc.allow(st, False{})))
752
753def q.text(cur: Cur, acc: String) -> Qs:
754 Qs{cur, acc, "", False{}, 0n, False{}, True{}, False{}}
755
756def simple.escape(e: Char) -> Maybe<&2, Char>:
757 match e:
758 case '0':
759 Some{Chr{0}}
760 case 'a':
761 Some{Chr{7}}
762 case 'b':
763 Some{Chr{8}}
764 case 't':
765 Some{Chr{9}}
766 case '\t':
767 Some{Chr{9}}
768 case 'n':
769 Some{Chr{10}}
770 case 'v':
771 Some{Chr{11}}
772 case 'f':
773 Some{Chr{12}}
774 case 'r':
775 Some{Chr{13}}
776 case 'e':
777 Some{Chr{27}}
778 case ' ':
779 Some{' '}
780 case '"':
781 Some{'"'}
782 case '/':
783 Some{'/'}
784 case '\\':
785 Some{'\\'}
786 case 'N':
787 Some{Chr{133}}
788 case '_':
789 Some{Chr{160}}
790 case 'L':
791 Some{Chr{8232}}
792 case 'P':
793 Some{Chr{8233}}
794 case _:
795 None{}
796
797def q.simple(m: Maybe<&2, Char>, cur: Cur, acc: String, start: T.Mark) -> Qs:
798 match m:
799 case None{}:
800 QsBad{T.Error{"found unknown escape character while parsing a quoted scalar", start, T.Syntax{}}}
801 case Some{c}:
802 q.text(adv.n(2n, cur), SCon{c, acc})
803
804# A code point that is a character: not a surrogate, and at most 10FFFF.
805def is_scalar.value(+x: U32) -> Bool:
806 Bool.and(Bool.not(between(x, 55296, 57343)), U32.is_le(x, 1114111))
807
808# \x, \u or \U and +n+ hex digits.
809def q.hex(+n: Nat, +cur: Cur, acc: String, +start: T.Mark) -> Qs:
810 +digits = String.take(String.drop(cur.s(cur), 2n), n)
811 +x = hex.value(digits, 0)
812 Bool.pick(Qs, Bool.not(Bool.and(Nat.is_eq(String.length(digits), n), all.hex(digits))),
813 QsBad{T.Error{"did not find expected hexdecimal number while parsing a quoted scalar", start, T.Syntax{}}},
814 Bool.pick(Qs, Bool.not(is_scalar.value(x)),
815 QsBad{T.Error{"found invalid Unicode character escape code while parsing a quoted scalar", start, T.Syntax{}}},
816 q.text(adv.n(Nat.add(n, 2n), cur), SCon{Chr{x}, acc})))
817
818def q.escape(e: Char, cur: Cur, acc: String, start: T.Mark) -> Qs:
819 match e:
820 case '\n':
821 Qs{adv.n(2n, cur), acc, "", True{}, 0n, True{}, False{}, False{}}
822 case 'x':
823 q.hex(2n, cur, acc, start)
824 case 'u':
825 q.hex(4n, cur, acc, start)
826 case 'U':
827 q.hex(8n, cur, acc, start)
828 case _:
829 q.simple(simple.escape(e), cur, acc, start)
830
831def q.char(double: Bool, c: Char, +c1: Char, +cur: Cur, +acc: String, start: T.Mark) -> Qs:
832 match double:
833 case False{}:
834 match c:
835 case '\'':
836 Bool.pick(Qs, Char.is_eq(c1, '\''), q.text(adv.n(2n, cur), SCon{'\'', acc}), QsDone{adv(cur), acc})
837 case _:
838 q.text(adv(cur), SCon{c, acc})
839 case True{}:
840 match c:
841 case '"':
842 QsDone{adv(cur), acc}
843 case '\\':
844 q.escape(c1, cur, acc, start)
845 case _:
846 q.text(adv(cur), SCon{c, acc})
847
848def q.in.text(+double: Bool, +cur: Cur, +acc: String, +start: T.Mark) -> Qs:
849 +s = cur.s(cur)
850 +c = peek(s, 0n)
851 Bool.pick(Qs, Bool.and(Nat.is_eq(cur.col(cur), 0n), is_marker(s)),
852 QsBad{T.Error{"found unexpected document indicator while scanning a quoted scalar", start, T.Syntax{}}},
853 Bool.pick(Qs, is_end(c), QsBad{T.Error{"found unexpected end of stream while scanning a quoted scalar", start, T.Syntax{}}},
854 Bool.pick(Qs, Bool.or(is_blank(c), is_break(c)), Qs{cur, acc, "", False{}, 0n, False{}, False{}, False{}},
855 q.char(double, c, peek(s, 1n), cur, acc, start))))
856
857# The blanks and breaks between text: kept, or folded, once text resumes.
858def q.joined(+acc: String, ws: String, lead: Bool, +brks: Nat, esc: Bool) -> String:
859 match lead:
860 case False{}:
861 ws ++ acc
862 case True{}:
863 Bool.pick(String, esc, String.repeat("\n", brks), folded.breaks(brks)) ++ acc
864
865# Between text: blanks and breaks wait to join it. A line after a break
866# must be indented past the block around it (+ind+), with spaces.
867def q.in.blanks(+cur: Cur, +acc: String, +ws: String, +lead: Bool, +brks: Nat, +esc: Bool, +bad: Bool, +ind: Nat, +start: T.Mark) -> Qs:
868 +c = peek(cur.s(cur), 0n)
869 +shallow = Nat.is_lt(cur.col(cur), ind)
870 Bool.pick(Qs, is_blank(c), Qs{adv(cur), acc, Bool.pick(String, lead, ws, SCon{c, ws}), lead, brks, esc, False{}, Bool.or(bad, Bool.and(lead, Bool.and(shallow, Char.is_eq(c, '\t'))))},
871 Bool.pick(Qs, is_break(c), Qs{adv(cur), acc, "", True{}, Bool.pick(Nat, lead, 1n+brks, 0n), esc, False{}, False{}},
872 Bool.pick(Qs, Bool.and(lead, Bool.and(Bool.not(is_end(c)), Bool.or(bad, shallow))),
873 QsBad{T.Error{"found a line indented too little in a quoted scalar while scanning a quoted scalar", start, T.Syntax{}}},
874 q.text(cur, q.joined(acc, ws, lead, brks, esc)))))
875
876def q.step(q: Qs, +double: Bool, +ind: Nat, +start: T.Mark) -> Qs:
877 match q:
878 case Qs{cur, acc, ws, lead, brks, esc, text, bad}:
879 match text:
880 case True{}:
881 q.in.text(double, cur, acc, start)
882 case False{}:
883 q.in.blanks(cur, acc, ws, lead, brks, esc, bad, ind, start)
884 case _:
885 q
886
887def q.run(fuel: Nat, +double: Bool, +ind: Nat, +start: T.Mark, q: Qs) -> Qs:
888 match fuel:
889 case 0n:
890 q
891 case 1n+k:
892 match q:
893 case Qs{cur, acc, ws, lead, brks, esc, text, bad}:
894 q.run(k, double, ind, start, q.step(Qs{cur, acc, ws, lead, brks, esc, text, bad}, double, ind, start))
895 case _:
896 q
897
898def quoted.done(q: Qs, +st: Sc, +double: Bool, start: T.Mark) -> Step:
899 match q:
900 case QsDone{cur, acc}:
901 Go{emit(sc.with.cur(st, cur), T.ScalarTok{String.reverse(acc), Bool.pick(T.Style, double, T.Double{}, T.Single{})}, start)}
902 case QsBad{e}:
903 fail.with(st, e)
904 case _:
905 fail(st, "found unexpected end of stream while scanning a quoted scalar", start)
906
907def quoted.scan(+st: Sc, +double: Bool) -> Step:
908 +cur = sc.cur(st)
909 +at = cur.at(cur)
910 quoted.done(q.run(Nat.add(Nat.mul(cur.left(cur), 2n), 4n), double, sc.ind(st), at, q.text(adv(cur), "")), st, double, at)
911
912def quoted(+st: Sc, +double: Bool) -> Step:
913 then(save.key(st), st => quoted.scan(sc.allow(st, False{}), double))
914
915def digit.value(c: Char) -> Nat:
916 U32.to_nat(U32.sub(code(c), 48))
917
918def chomp.of(c: Char) -> Maybe<&2, Chomp>:
919 match c:
920 case '+':
921 Some{Keep{}}
922 case '-':
923 Some{Strip{}}
924 case _:
925 None{}
926
927def chomp.or(m: Maybe<&2, Chomp>, d: Chomp) -> Chomp:
928 match m:
929 case None{}:
930 d
931 case Some{c}:
932 c
933
934def is_some.chomp(m: Maybe<&2, Chomp>) -> Bool:
935 match m:
936 case None{}:
937 False{}
938 case Some{c}:
939 True{}
940
941# Past blanks and a comment (after a blank), to a line break or the end.
942def head.tail(s: String, +at: T.Mark, +left: Nat, +comment: Bool, +gap: Bool) -> Cur:
943 match s:
944 case SNil{}:
945 Cur{SNil{}, at, left}
946 case SCon{+c, +t}:
947 +hash = Bool.and(Char.is_eq(c, '#'), gap)
948 when(Cur, Bool.or(is_break(c), Bool.not(Bool.or(comment, Bool.or(is_blank(c), hash)))), u => Cur{SCon{c, t}, at, left},
949 u => head.tail(t, past(at, c), Nat.sub(left, 1n), Bool.or(comment, hash), is_blank(c)))
950
951def head.check(chomp: Chomp, +inc: Nat, +cur: Cur) -> Head:
952 Bool.pick(Head, is_breakz(peek(cur.s(cur), 0n)), Head{chomp, inc, adv(cur)}, HeadBad{"did not find expected comment or line break while scanning a block scalar"})
953
954def head.end(chomp: Chomp, +inc: Nat, cur: Cur) -> Head:
955 match cur:
956 case Cur{s, at, left}:
957 head.check(chomp, inc, head.tail(s, at, left, False{}, False{}))
958
959# The indicators after "|" or ">", in either order.
960def header(+cur: Cur) -> Head:
961 +s = cur.s(cur)
962 +c0 = peek(s, 0n)
963 +c1 = peek(s, 1n)
964 +ch0 = chomp.of(c0)
965 +ch1 = chomp.of(c1)
966 +d0 = Bool.and(Char.is_digit(c0), Bool.not(Char.is_eq(c0, '0')))
967 +d1 = Bool.and(Char.is_digit(c1), Bool.not(Char.is_eq(c1, '0')))
968 when(Head, Bool.or(Char.is_eq(c0, '0'), Bool.and(is_some.chomp(ch0), Char.is_eq(c1, '0'))),
969 u => HeadBad{"found an indentation indicator equal to 0 while scanning a block scalar"},
970 u => when(Head, Bool.and(is_some.chomp(ch0), d1), u => head.end(chomp.or(ch0, Clip{}), digit.value(c1), adv.n(2n, cur)),
971 u => when(Head, is_some.chomp(ch0), u => head.end(chomp.or(ch0, Clip{}), 0n, adv(cur)),
972 u => when(Head, Bool.and(d0, is_some.chomp(ch1)), u => head.end(chomp.or(ch1, Clip{}), digit.value(c0), adv.n(2n, cur)),
973 u => when(Head, d0, u => head.end(Clip{}, digit.value(c0), adv(cur)), u => head.end(Clip{}, 0n, cur))))))
974
975# Blank lines and the indentation before content. The first content line
976# sets the indentation when the header didn't; no blank line before it may
977# be deeper. A tab may start content but not indentation. A last line of
978# spaces with no break after it still counts as a line.
979def bs.breaks(+cur: Cur, +acc: String, +ind: Nat, +max: Nat, +lead: Bool, +lblank: Bool, +brks: Nat, +pind: Nat, +start: T.Mark) -> Bs:
980 +c = peek(cur.s(cur), 0n)
981 +col = cur.col(cur)
982 +inside = Bool.or(Nat.is_eq(ind, 0n), Nat.is_lt(1n+col, ind))
983 +first = Bool.and(Nat.is_eq(ind, 0n), Nat.is_ge(col, pind))
984 +partial = Bool.and(Nat.is_gt(col, 0n), Nat.is_gt(T.mark.line(cur.at(cur)), T.mark.line(start)))
985 Bool.pick(Bs, Bool.and(Char.is_eq(c, ' '), inside), Bs{adv(cur), acc, ind, max, lead, lblank, brks, Breaks{}},
986 Bool.pick(Bs, is_break(c), Bs{adv(cur), acc, ind, Nat.max(col, max), lead, lblank, 1n+brks, Breaks{}},
987 Bool.pick(Bs, is_end(c), Bs{cur, acc, ind, max, lead, lblank, Bool.pick(Nat, partial, 1n+brks, brks), LineStart{}},
988 Bool.pick(Bs, Bool.and(Bool.and(Char.is_eq(c, '\t'), inside), Bool.not(first)),
989 BsBad{T.Error{"found a tab character where an indentation space is expected while scanning a block scalar", start, T.Syntax{}}},
990 Bool.pick(Bs, Bool.and(first, Nat.is_gt(max, col)),
991 BsBad{T.Error{"found a leading blank line with more spaces than the first content line while scanning a block scalar", start, T.Syntax{}}},
992 Bs{cur, acc, Bool.pick(Nat, Nat.is_eq(ind, 0n), 1n+Nat.max(col, pind), ind), max, lead, lblank, brks, LineStart{}})))))
993
994# A content line starts at the indentation: what waits joins the text.
995def bs.line(+cur: Cur, +acc: String, +ind: Nat, +max: Nat, +lead: Bool, +lblank: Bool, +brks: Nat, +folded: Bool) -> Bs:
996 +s = cur.s(cur)
997 +c = peek(s, 0n)
998 +fold = Bool.and(Bool.and(folded, lead), Bool.and(Bool.not(lblank), Bool.not(is_blank(c))))
999 +joined = String.repeat("\n", brks) ++ Bool.pick(String, fold, Bool.pick(String, Nat.is_eq(brks, 0n), " ", ""), Bool.pick(String, lead, "\n", "")) ++ acc
1000 Bool.pick(Bs, Bool.or(Bool.not(Nat.is_eq(1n+cur.col(cur), ind)), Bool.or(is_end(c), Bool.and(Nat.is_eq(cur.col(cur), 0n), is_marker(s)))),
1001 BsDone{acc, lead, brks, cur},
1002 Bs{cur, joined, ind, max, False{}, is_blank(c), 0n, Content{}})
1003
1004def bs.content(+cur: Cur, +acc: String, +ind: Nat, +max: Nat, +lblank: Bool) -> Bs:
1005 +c = peek(cur.s(cur), 0n)
1006 Bool.pick(Bs, is_end(c), BsDone{acc, True{}, 0n, cur},
1007 Bool.pick(Bs, is_break(c), Bs{adv(cur), acc, ind, max, True{}, lblank, 0n, Breaks{}},
1008 Bs{adv(cur), SCon{c, acc}, ind, max, False{}, lblank, 0n, Content{}}))
1009
1010def bs.step(b: Bs, +folded: Bool, +pind: Nat, +start: T.Mark) -> Bs:
1011 match b:
1012 case Bs{cur, acc, ind, max, lead, lblank, brks, phase}:
1013 match phase:
1014 case Breaks{}:
1015 bs.breaks(cur, acc, ind, max, lead, lblank, brks, pind, start)
1016 case LineStart{}:
1017 bs.line(cur, acc, ind, max, lead, lblank, brks, folded)
1018 case Content{}:
1019 bs.content(cur, acc, ind, max, lblank)
1020 case _:
1021 b
1022
1023def bs.run(fuel: Nat, +folded: Bool, +pind: Nat, +start: T.Mark, b: Bs) -> Bs:
1024 match fuel:
1025 case 0n:
1026 b
1027 case 1n+k:
1028 match b:
1029 case Bs{cur, acc, ind, max, lead, lblank, brks, phase}:
1030 bs.run(k, folded, pind, start, bs.step(Bs{cur, acc, ind, max, lead, lblank, brks, phase}, folded, pind, start))
1031 case _:
1032 b
1033
1034# The final line break and the blank lines after the text.
1035def chomped(chomp: Chomp, +acc: String, +lead: Bool, +brks: Nat) -> String:
1036 match chomp:
1037 case Clip{}:
1038 Bool.pick(String, lead, SCon{'\n', acc}, acc)
1039 case Strip{}:
1040 acc
1041 case Keep{}:
1042 String.repeat("\n", brks) ++ Bool.pick(String, lead, SCon{'\n', acc}, acc)
1043
1044def block.done(b: Bs, chomp: Chomp, +st: Sc, +folded: Bool, start: T.Mark) -> Step:
1045 match b:
1046 case BsDone{acc, lead, brks, cur}:
1047 Go{emit(sc.with.cur(st, cur), T.ScalarTok{String.reverse(chomped(chomp, acc, lead, brks)), Bool.pick(T.Style, folded, T.Folded{}, T.Literal{})}, start)}
1048 case BsBad{e}:
1049 fail.with(st, e)
1050 case _:
1051 fail(st, "found unexpected end of stream while scanning a block scalar", start)
1052
1053def block.body(h: Head, +st: Sc, +folded: Bool, +start: T.Mark) -> Step:
1054 match h:
1055 case Head{chomp, +inc, +cur}:
1056 +pind = sc.ind(st)
1057 +given = Nat.add(pind, inc)
1058 block.done(bs.run(Nat.add(Nat.mul(cur.left(cur), 3n), 4n), folded, pind, start,
1059 Bs{cur, "", Bool.pick(Nat, Nat.is_eq(inc, 0n), 0n, given), 0n, False{}, False{}, 0n, Breaks{}}), chomp, st, folded, start)
1060 case HeadBad{why}:
1061 fail(st, why, start)
1062
1063def block.scan(+st: Sc, +folded: Bool) -> Step:
1064 +cur = sc.cur(st)
1065 block.body(header(adv(cur)), st, folded, cur.at(cur))
1066
1067def block.scalar(+st: Sc, +folded: Bool) -> Step:
1068 then(remove.key(st), st => block.scan(sc.allow(st, True{}), folded))
1069
1070def taken.cons(c: Char, +k: Nat, t: Taken) -> Taken:
1071 match t:
1072 case Taken{text, n}:
1073 Taken{SCon{c, text}, Nat.add(k, n)}
1074
1075def took.or(stop: Bool, more: Unit -> Taken) -> Taken:
1076 match stop:
1077 case True{}:
1078 Taken{"", 0n}
1079 case False{}:
1080 more(Unit{})
1081
1082# An anchor's name: anything up to a blank, a break or a flow indicator.
1083def anchor.name(s: String) -> Taken:
1084 match s:
1085 case SNil{}:
1086 Taken{"", 0n}
1087 case SCon{+c, t}:
1088 took.or(Bool.or(is_blankz(c), is_flow.ind(c)), u => taken.cons(c, 1n, anchor.name(t)))
1089
1090def anchor.took(t: Taken, +st: Sc, +is_anchor: Bool, +at: T.Mark) -> Step:
1091 match t:
1092 case Taken{+name, +n}:
1093 when(Step, String.is_empty(name),
1094 u => fail(st, Bool.pick(String, is_anchor, "did not find expected alphabetic or numeric character while scanning an anchor",
1095 "did not find expected alphabetic or numeric character while scanning an alias"), at),
1096 u => Go{emit(skip(st, 1n+n), Bool.pick(T.Tok, is_anchor, T.AnchorTok{name}, T.AliasTok{name}), at)})
1097
1098def anchor.scan(+st: Sc, +is_anchor: Bool) -> Step:
1099 +at = sc.at(st)
1100 anchor.took(anchor.name(String.drop(sc.s(st), 1n)), st, is_anchor, at)
1101
1102def anchor(+st: Sc, +is_anchor: Bool) -> Step:
1103 then(save.key(st), st => anchor.scan(sc.allow(st, False{}), is_anchor))
1104
1105# Tag characters, with %XX escapes decoded; +uri+ allows all URI characters
1106# (in a verbatim tag or a %TAG prefix).
1107def taken.skip(t: Taken) -> Taken:
1108 match t:
1109 case Taken{text, n}:
1110 Taken{text, 1n+n}
1111
1112def tag.chars(s: String, +uri: Bool, +pending: Nat) -> Taken:
1113 match s:
1114 case SNil{}:
1115 Taken{"", 0n}
1116 case SCon{+c, +t}:
1117 match pending:
1118 case 1n+k:
1119 taken.skip(tag.chars(t, uri, k))
1120 case 0n:
1121 +esc = Bool.and(Char.is_eq(c, '%'), Bool.and(is_hex(peek(t, 0n)), is_hex(peek(t, 1n))))
1122 +ok = Bool.pick(Bool, uri, is_uri(c), is_tag(c))
1123 took.or(Bool.not(Bool.or(esc, ok)),
1124 u => taken.cons(Bool.pick(Char, esc, Chr{hex.value(String.take(t, 2n), 0)}, c), 1n,
1125 tag.chars(t, uri, Bool.pick(Nat, esc, 2n, 0n))))
1126
1127# What may follow a tag: a blank, a break, the end, or in flow a flow
1128# indicator.
1129def tag.ends(+c: Char, +flow: Bool) -> Bool:
1130 Bool.or(is_blankz(c), Bool.and(flow, is_flow.ind(c)))
1131
1132def tag.emit(+st: Sc, handle: String, suffix: String, +n: Nat, +at: T.Mark) -> Step:
1133 +st2 = skip(st, n)
1134 +next = peek(sc.s(st2), 0n)
1135 when(Step, tag.ends(next, sc.in_flow(st2)),
1136 u => Go{emit(st2, T.TagTok{handle, suffix}, at)},
1137 u => fail(st2, Bool.pick(String, Char.is_eq(next, '%'), "did not find URI escaped octet while parsing a tag",
1138 "did not find expected whitespace or line break while scanning a tag"), at))
1139
1140# "!<uri>".
1141def tag.verbatim(t: Taken, +st: Sc, +at: T.Mark) -> Step:
1142 match t:
1143 case Taken{+uri, +n}:
1144 when(Step, Bool.or(String.is_empty(uri), Bool.not(Char.is_eq(peek(String.drop(sc.s(st), Nat.add(n, 2n)), 0n), '>'))),
1145 u => fail(st, "did not find the expected '>' while scanning a tag", at),
1146 u => tag.emit(st, "", uri, Nat.add(n, 3n), at))
1147
1148def tag.suffix(t: Taken, +named: Bool, +word: String, +k: Nat, +st: Sc, +at: T.Mark) -> Step:
1149 match t:
1150 case Taken{+suffix, +n}:
1151 +handle = Bool.pick(String, named, "!" ++ word ++ "!", "!")
1152 +used = Nat.add(1n, Bool.pick(Nat, named, Nat.add(1n+k, n), n))
1153 when(Step, Bool.and(named, String.is_empty(suffix)),
1154 u => fail(st, "did not find expected tag URI while parsing a tag", at),
1155 u => when(Step, String.is_empty(suffix), u => tag.emit(st, "", "!", used, at), u => tag.emit(st, handle, suffix, used, at)))
1156
1157# "!suffix", "!!suffix", "!name!suffix", or "!" alone.
1158def tag.short(w: Taken, +st: Sc, +at: T.Mark) -> Step:
1159 match w:
1160 case Taken{+word, +k}:
1161 +s = String.drop(sc.s(st), 1n)
1162 +named = Char.is_eq(peek(s, k), '!')
1163 tag.suffix(tag.chars(String.drop(s, Bool.pick(Nat, named, 1n+k, 0n)), False{}, 0n), named, word, k, st, at)
1164
1165def word.chars(s: String) -> Taken:
1166 match s:
1167 case SNil{}:
1168 Taken{"", 0n}
1169 case SCon{+c, t}:
1170 took.or(Bool.not(is_word(c)), u => taken.cons(c, 1n, word.chars(t)))
1171
1172def tag.scan(+st: Sc) -> Step:
1173 +s = sc.s(st)
1174 +at = sc.at(st)
1175 when(Step, Char.is_eq(peek(s, 1n), '<'),
1176 u => tag.verbatim(tag.chars(String.drop(s, 2n), True{}, 0n), st, at),
1177 u => tag.short(word.chars(String.drop(s, 1n)), st, at))
1178
1179def tag(+st: Sc) -> Step:
1180 then(save.key(st), st => tag.scan(sc.allow(st, False{})))
1181
1182# Directives
1183# ----------
1184
1185def count.or(go: Bool, more: Unit -> Nat) -> Nat:
1186 match go:
1187 case True{}:
1188 more(Unit{})
1189 case False{}:
1190 0n
1191
1192def blanks(s: String) -> Nat:
1193 match s:
1194 case SNil{}:
1195 0n
1196 case SCon{+c, t}:
1197 count.or(is_blank(c), u => 1n+blanks(t))
1198
1199def digits(s: String) -> Taken:
1200 match s:
1201 case SNil{}:
1202 Taken{"", 0n}
1203 case SCon{+c, t}:
1204 took.or(Bool.not(Char.is_digit(c)), u => taken.cons(c, 1n, digits(t)))
1205
1206def nat.of(s: String, acc: Nat) -> Nat:
1207 match s:
1208 case SNil{}:
1209 acc
1210 case SCon{c, t}:
1211 nat.of(t, Nat.add(Nat.mul(acc, 10n), digit.value(c)))
1212
1213def directive.put(+st: Sc, tok: Maybe<&2, T.Tok>, at: T.Mark) -> Sc:
1214 match tok:
1215 case None{}:
1216 st
1217 case Some{t}:
1218 emit(st, t, at)
1219
1220def directive.tail(+cur: Cur, +st: Sc, tok: Maybe<&2, T.Tok>, +at: T.Mark) -> Step:
1221 when(Step, is_breakz(peek(cur.s(cur), 0n)),
1222 u => Go{directive.put(sc.with.cur(st, cur), tok, at)},
1223 u => fail(st, "did not find expected comment or line break while scanning a directive", at))
1224
1225# After a directive: blanks, a comment, then a line break or the end.
1226def directive.end(+st: Sc, tok: Maybe<&2, T.Tok>, at: T.Mark) -> Step:
1227 directive.tail(head.tail(sc.s(st), sc.at(st), cur.left(sc.cur(st)), False{}, False{}), st, tok, at)
1228
1229def version.done(m: Taken, +st: Sc, major: String, +used: Nat, +at: T.Mark) -> Step:
1230 match m:
1231 case Taken{+minor, +k}:
1232 when(Step, Nat.is_eq(k, 0n), u => fail(st, "did not find expected digit or '.' character while scanning a %YAML directive", at),
1233 u => directive.end(skip(st, Nat.add(used, k)), Some{T.VersionDir{nat.of(major, 0n), nat.of(minor, 0n)}}, at))
1234
1235def version.minor(m: Taken, +st: Sc, +b: Nat, +at: T.Mark) -> Step:
1236 match m:
1237 case Taken{+major, +k}:
1238 +rest = String.drop(sc.s(st), Nat.add(b, k))
1239 +minor = digits(String.drop(rest, 1n))
1240 when(Step, Nat.is_eq(b, 0n), u => fail(st, "did not find expected whitespace while scanning a %YAML directive", at),
1241 u => when(Step, Bool.or(Nat.is_eq(k, 0n), Bool.not(Char.is_eq(peek(rest, 0n), '.'))),
1242 u => fail(st, "did not find expected digit or '.' character while scanning a %YAML directive", at),
1243 u => version.done(minor, st, major, Nat.add(b, Nat.add(k, 1n)), at)))
1244
1245# "%YAML 1.2".
1246def version.scan(+st: Sc, +at: T.Mark) -> Step:
1247 +s = sc.s(st)
1248 +b = blanks(s)
1249 +major = digits(String.drop(s, b))
1250 version.minor(major, st, b, at)
1251
1252def tagdir.prefix(p: Taken, +st: Sc, handle: String, +used: Nat, +at: T.Mark) -> Step:
1253 match p:
1254 case Taken{+prefix, +k}:
1255 when(Step, Nat.is_eq(k, 0n), u => fail(st, "did not find expected tag URI while scanning a %TAG directive", at),
1256 u => directive.end(skip(st, Nat.add(used, k)), Some{T.TagDir{handle, prefix}}, at))
1257
1258def tagdir.handle(w: Taken, +st: Sc, +b: Nat, +at: T.Mark) -> Step:
1259 match w:
1260 case Taken{+word, +k}:
1261 +rest = String.drop(sc.s(st), b)
1262 +closed = Char.is_eq(peek(rest, 1n+k), '!')
1263 +hlen = Nat.add(1n+k, Bool.pick(Nat, closed, 1n, 0n))
1264 +handle = String.take(rest, hlen)
1265 +b2 = blanks(String.drop(rest, hlen))
1266 +ok = Bool.and(Char.is_eq(peek(rest, 0n), '!'), Bool.or(closed, Nat.is_eq(k, 0n)))
1267 when(Step, Bool.or(Nat.is_eq(b, 0n), Bool.not(ok)), u => fail(st, "did not find expected '!' while scanning a %TAG directive", at),
1268 u => when(Step, Nat.is_eq(b2, 0n), u => fail(st, "did not find expected whitespace while scanning a %TAG directive", at),
1269 u => tagdir.prefix(tag.chars(String.drop(rest, Nat.add(hlen, b2)), True{}, 0n), st, handle, Nat.add(b, Nat.add(hlen, b2)), at)))
1270
1271# "%TAG !e! prefix".
1272def tagdir.scan(+st: Sc, +at: T.Mark) -> Step:
1273 +s = sc.s(st)
1274 +b = blanks(s)
1275 +rest = String.drop(s, b)
1276 tagdir.handle(word.chars(String.drop(rest, 1n)), st, b, at)
1277
1278# Past the rest of a line.
1279def line.rest(s: String) -> Nat:
1280 match s:
1281 case SNil{}:
1282 0n
1283 case SCon{+c, t}:
1284 count.or(Bool.not(is_break(c)), u => 1n+line.rest(t))
1285
1286# %YAML, %TAG, or a reserved directive, which is skipped.
1287def directive.named(+name: String, +st: Sc, +at: T.Mark) -> Step:
1288 when(Step, String.eq(name, "YAML"), u => version.scan(st, at),
1289 u => when(Step, String.eq(name, "TAG"), u => tagdir.scan(st, at),
1290 u => Go{skip(st, line.rest(sc.s(st)))}))
1291
1292def directive.name(w: Taken, +st: Sc, +at: T.Mark) -> Step:
1293 match w:
1294 case Taken{+name, +k}:
1295 when(Step, Nat.is_eq(k, 0n), u => fail(st, "could not find expected directive name while scanning a directive", at),
1296 u => directive.named(name, skip(st, 1n+k), at))
1297
1298def directive.scan(+st: Sc) -> Step:
1299 +at = sc.at(st)
1300 directive.name(word.chars(String.drop(sc.s(st), 1n)), st, at)
1301
1302def directive(+st: Sc) -> Step:
1303 then(remove.key(unroll(st, 0n)), st => directive.scan(sc.allow(st, False{})))
1304
1305# Tokens
1306# ------
1307
1308def flow.start(+st: Sc, t: T.Tok) -> Step:
1309 then(save.key(st), st => Go{single(sc.allow(flow.push(st), True{}), t)})
1310
1311def flow.end(+st: Sc, t: T.Tok) -> Step:
1312 then(remove.key(st), st => Go{single(sc.allow(flow.pop(st), False{}), t)})
1313
1314def flow.entry(+st: Sc) -> Step:
1315 then(remove.key(st), st => Go{single(sc.allow(st, True{}), T.FEntry{})})
1316
1317def block.entry.rest(+st: Sc) -> Step:
1318 then(remove.key(st), st => Go{single(sc.allow(st, True{}), T.BEntry{})})
1319
1320# "- ": a block sequence's entry, which may open the sequence.
1321def block.entry(+st: Sc) -> Step:
1322 when(Step, sc.in_flow(st), u => block.entry.rest(st),
1323 u => when(Step, sc.tab(st), u => fail(st, "found a tab character that violates indentation", sc.at(st)),
1324 u => when(Step, sc.allowed(st), u => block.entry.rest(roll(st, sc.col(st), None{}, T.BSeqStart{}, sc.at(st))),
1325 u => fail(st, "block sequence entries are not allowed in this context", sc.at(st)))))
1326
1327def key.tok(+st: Sc) -> Sc:
1328 single(sc.allow(st, Bool.not(sc.in_flow(st))), T.KeyTok{})
1329
1330def key.rest(+st: Sc) -> Step:
1331 then(remove.key(st), st => Go{key.tok(st)})
1332
1333# "? ": an explicit key.
1334def key(+st: Sc) -> Step:
1335 when(Step, sc.in_flow(st), u => key.rest(st),
1336 u => when(Step, sc.tab(st), u => fail(st, "found a tab character that violates indentation", sc.at(st)),
1337 u => when(Step, sc.allowed(st), u => key.rest(roll(st, sc.col(st), None{}, T.BMapStart{}, sc.at(st))),
1338 u => fail(st, "mapping keys are not allowed in this context", sc.at(st)))))
1339
1340def value.rest(+st: Sc) -> Sc:
1341 single(sc.allow(st, Bool.not(sc.in_flow(st))), T.ValueTok{})
1342
1343def value.alone(+st: Sc) -> Step:
1344 when(Step, sc.in_flow(st), u => Go{value.rest(st)},
1345 u => when(Step, sc.allowed(st), u => Go{value.rest(roll(st, sc.col(st), None{}, T.BMapStart{}, sc.at(st)))},
1346 u => fail(st, "mapping values are not allowed in this context", sc.at(st))))
1347
1348def insert.key(st: Sc, number: Nat, +at: T.Mark) -> Sc:
1349 match st:
1350 case Sc{cur, ind, inds, flow, allowed, tab, keys, out, +n}:
1351 Sc{cur, ind, inds, flow, allowed, tab, key.cleared(keys), insert(out, n, Some{number}, T.Token{T.KeyTok{}, at, at}), 1n+n}
1352
1353# ": ": a value. When a simple key waits, it becomes a key: a KeyTok goes
1354# before it, and a block mapping may start at its column.
1355def value.with(k: SKey, +st: Sc) -> Step:
1356 match k:
1357 case SKey{possible, required, +number, +at, tab}:
1358 +rolls = Bool.and(Nat.is_eq(sc.flow(st), 0n), Nat.is_lt(sc.ind(st), 1n+T.mark.col(at)))
1359 when(Step, possible,
1360 u => when(Step, Bool.and(tab, rolls), u => fail(st, "found a tab character that violates indentation", at),
1361 u => Go{single(sc.allow(roll(insert.key(st, number, at), T.mark.col(at), Some{number}, T.BMapStart{}, at), False{}), T.ValueTok{})}),
1362 u => value.alone(st))
1363
1364def value(+st: Sc) -> Step:
1365 value.with(key.top(sc.keys(st)), st)
1366
1367def marker.tok(+st: Sc, t: T.Tok) -> Sc:
1368 emit(skip(sc.allow(st, False{}), 3n), t, sc.at(st))
1369
1370# The stream ends on a line of its own.
1371def end.mark(m: T.Mark) -> T.Mark:
1372 match m:
1373 case T.Mark{+line, +col, +idx}:
1374 Bool.pick(T.Mark, Nat.is_eq(col, 0n), T.Mark{line, col, idx}, T.Mark{1n+line, 0n, idx})
1375
1376def end.tok(+st: Sc) -> Sc:
1377 +at = end.mark(sc.at(st))
1378 emit.span(sc.allow(st, False{}), T.StreamEnd{}, at, at)
1379
1380def doc.end.check(+st: Sc, t: T.Tok) -> Step:
1381 match t:
1382 case T.DocEnd{}:
1383 when(Step, is_breakz(peek(cur.s(head.tail(sc.s(st), sc.at(st), cur.left(sc.cur(st)), False{}, False{})), 0n)), u => Go{st},
1384 u => fail(st, "did not find expected comment or line break after a document end marker", sc.at(st)))
1385 case _:
1386 Go{st}
1387
1388def doc.indicator(+st: Sc, +t: T.Tok) -> Step:
1389 then(remove.key(unroll(st, 0n)), st => doc.end.check(marker.tok(st, t), t))
1390
1391def stream.end(+st: Sc) -> Step:
1392 then(remove.key(unroll(st, 0n)), st => Stop{end.tok(st)})
1393
1394def is_plain.start(+c: Char) -> Bool:
1395 Bool.and(is_printable(c), Bool.not(Bool.or(is_blankz(c), is_in(c, "-?:,[]{}#&*!|>'\"%@`"))))
1396
1397# The token a character starts.
1398def fetch.char(c: Char, +c1: Char, +flow: Bool, +st: Sc) -> Step:
1399 match c:
1400 case '[':
1401 flow.start(st, T.FSeqStart{})
1402 case '{':
1403 flow.start(st, T.FMapStart{})
1404 case ']':
1405 flow.end(st, T.FSeqEnd{})
1406 case '}':
1407 flow.end(st, T.FMapEnd{})
1408 case ',':
1409 flow.entry(st)
1410 case '-':
1411 when(Step, is_blankz(c1), u => block.entry(st), u => when(Step, Bool.and(flow, is_flow.ind(c1)), u => cannot(st), u => plain(st)))
1412 case '?':
1413 when(Step, Bool.or(is_blankz(c1), Bool.and(flow, is_flow.ind(c1))), u => key(st), u => plain(st))
1414 case ':':
1415 when(Step, Bool.or(is_blankz(c1), Bool.and(flow, Bool.or(is_flow.ind(c1), json.before(sc.out(st))))), u => value(st), u => plain(st))
1416 case '*':
1417 anchor(st, False{})
1418 case '&':
1419 anchor(st, True{})
1420 case '!':
1421 tag(st)
1422 case '|':
1423 when(Step, flow, u => cannot(st), u => block.scalar(st, False{}))
1424 case '>':
1425 when(Step, flow, u => cannot(st), u => block.scalar(st, True{}))
1426 case '\'':
1427 quoted(st, False{})
1428 case '"':
1429 quoted(st, True{})
1430 case _:
1431 when(Step, is_plain.start(c), u => plain(st), u => cannot(st))
1432
1433def fetch.next(+st: Sc) -> Step:
1434 +s = sc.s(st)
1435 +c = peek(s, 0n)
1436 +col0 = Nat.is_eq(sc.col(st), 0n)
1437 when(Step, String.is_empty(s), u => stream.end(st),
1438 u => when(Step, Bool.and(col0, Char.is_eq(c, '%')), u => directive(st),
1439 u => when(Step, Bool.and(col0, marker(s, '-')), u => doc.indicator(st, T.DocStart{}),
1440 u => when(Step, Bool.and(col0, marker(s, '.')), u => doc.indicator(st, T.DocEnd{}),
1441 u => fetch.char(c, peek(s, 1n), sc.in_flow(st), st)))))
1442
1443def fetch.after(+st: Sc) -> Step:
1444 fetch.next(unroll(st, 1n+sc.col(st)))
1445
1446# One token (and the block tokens before it).
1447def fetch(+st: Sc) -> Step:
1448 then(skip.ws(st), st => then(stale(st), st => fetch.after(st)))
1449
1450def scan.run(fuel: Nat, s: Step) -> List<&2, T.Token>:
1451 match fuel:
1452 case 0n:
1453 []
1454 case 1n+k:
1455 match s:
1456 case Go{st}:
1457 scan.run(k, fetch(st))
1458 case Stop{st}:
1459 List.reverse(&2, T.Token, sc.out(st))
1460
1461# Where the first character YAML doesn't allow is (a control character, a
1462# surrogate), if there is one.
1463def unprintable(s: String, +at: T.Mark) -> Maybe<&2, T.Mark>:
1464 match s:
1465 case SNil{}:
1466 None{}
1467 case SCon{+c, t}:
1468 when(Maybe<&2, T.Mark>, is_printable(c), u => unprintable(t, past(at, c)), u => Some{at})
1469
1470def stream.start() -> T.Token:
1471 T.Token{T.StreamStart{}, T.Mark{0n, 0n, 0n}, T.Mark{0n, 0n, 0n}}
1472
1473def scan.start(m: Maybe<&2, T.Mark>, +s: String) -> List<&2, T.Token>:
1474 match m:
1475 case Some{+at}:
1476 [stream.start(), T.Token{T.Broken{T.Error{"control characters are not allowed", at, T.Syntax{}}}, at, at}]
1477 case None{}:
1478 scan.run(Nat.add(String.length(s), 4n), Go{Sc{Cur{s, T.Mark{0n, 0n, 0n}, String.length(s)}, 0n, [], 0n, True{}, False{}, [SKey{False{}, False{}, 0n, T.Mark{0n, 0n, 0n}, False{}}], [stream.start()], 1n}})
1479
1480# The tokens of +text+, from StreamStart to StreamEnd, or to a Broken token
1481# at the first error.
1482def scan(text: String) -> List<&2, T.Token>:
1483 +s = strip.bom(lf(text))
1484 scan.start(unprintable(s, T.Mark{0n, 0n, 0n}), s)