1	# The scanner: YAML text to tokens.
2	#
3	# It follows the design of libyaml's scanner. One step fetches the next
4	# token. Block collections open and close by indentation, so the scanner
5	# keeps a stack of indentations and adds BSeqStart, BMapStart and BEnd
6	# tokens as they change. A "simple key" (a key without "?") is only known to
7	# be a key when its ":" arrives, so the scanner remembers where each
8	# possible key started and, at the ":", inserts a KeyTok (and BMapStart)
9	# before the key's first token.
10	#
11	# It reads YAML 1.2 as the YAML test suite has it: tabs separate tokens
12	# but never indent, a flow collection's lines and a quoted scalar's indent
13	# past the block around them, a comment follows a blank, anchors take any
14	# character but blanks and flow indicators, ":" is a value indicator right
15	# after a JSON-like node in flow, and reserved directives are skipped.
16	#
17	# An error ends the tokens with a Broken token, so the parser still sees
18	# the tokens before it.
19	
20	import Base
21	
22	import ./token.bend as T
23	
24	# The text left to read, where it starts, and how many characters are left.
25	type Cur is Data:
26	  Cur{s: String, at: T.Mark, left: Nat}
27	
28	# A place a simple key may start: whether one still may, whether it must (a
29	# block key at the indentation), the number its first token has, and where.
30	type SKey is Data:
31	  SKey{possible: Bool, required: Bool, number: Nat, at: T.Mark, tab: Bool}
32	
33	# The scanner: the cursor, the block indentation plus one (0 outside any
34	# block collection) and the ones it returns to, how deep in flow collections
35	# it is, whether a simple key may start here, one possible key per flow
36	# level (the block level last), the tokens so far (newest first) and their
37	# count.
38	type Sc is Data:
39	  Sc{cur: Cur, ind: Nat, inds: List<&2, Nat>, flow: Nat, allowed: Bool, tab: Bool, keys: List<&2, SKey>, out: List<&2, T.Token>, n: Nat}
40	
41	# A step goes on, or stops: at the stream's end or at an error, as the last
42	# token says.
43	type Step is Data:
44	  Go{st: Sc}
45	  Stop{st: Sc}
46	
47	type Lay is Data:
48	  Lay{ind: Nat, inds: List<&2, Nat>, out: List<&2, T.Token>, n: Nat}
49	
50	# Between tokens
51	# --------------
52	
53	type Skipped is Data:
54	  Skipped{cur: Cur, allowed: Bool, tab: Bool, nl: Bool, sp: Nat, bad: Bool}
55	
56	# Plain scalars
57	# -------------
58	
59	# Reading a plain scalar: the cursor, the text so far (reversed), where the
60	# last word ended, the blanks after it (reversed), whether a line break came
61	# since, how many more breaks did, and whether a word is being read.
62	type Pl is Data:
63	  Pl{cur: Cur, acc: String, end: T.Mark, ws: String, lead: Bool, brks: Nat, word: Bool, bad: Bool}
64	  PlDone{cur: Cur, acc: String, end: T.Mark, lead: Bool}
65	  PlBad{error: T.Error}
66	
67	# Quoted scalars
68	# --------------
69	
70	# Reading a quoted scalar: the cursor, the text so far (reversed), the
71	# blanks waiting (reversed), whether a line break came since the last text
72	# and how many more did, whether that break was escaped, and whether text
73	# (not blanks) is being read.
74	type Qs is Data:
75	  Qs{cur: Cur, acc: String, ws: String, lead: Bool, brks: Nat, esc: Bool, text: Bool, bad: Bool}
76	  QsDone{cur: Cur, acc: String}
77	  QsBad{error: T.Error}
78	
79	# Block scalars
80	# -------------
81	
82	type Chomp is Data:
83	  Clip{}
84	  Strip{}
85	  Keep{}
86	
87	# A block scalar's header: how to chomp, the indentation indicator (0 when
88	# there's none), and the cursor after it; or an error.
89	type Head is Data:
90	  Head{chomp: Chomp, inc: Nat, cur: Cur}
91	  HeadBad{why: String}
92	
93	type BPhase is Data:
94	  Breaks{}
95	  LineStart{}
96	  Content{}
97	
98	# Reading a block scalar: the cursor, the text so far (reversed), the
99	# content's indentation plus one (0 until known), the deepest blank line, whether a
100	# line break waits, whether the last line started with a blank, how many
101	# more breaks wait, and the phase: blank lines, a line's start, or content.
102	type Bs is Data:
103	  Bs{cur: Cur, acc: String, ind: Nat, max: Nat, lead: Bool, lblank: Bool, brks: Nat, phase: BPhase}
104	  BsDone{acc: String, lead: Bool, brks: Nat, cur: Cur}
105	  BsBad{error: T.Error}
106	
107	# Anchors, aliases and tags
108	# -------------------------
109	
110	# What a scan took: the text it read and how many characters that was.
111	type Taken is Data:
112	  Taken{text: String, n: Nat}
113	
114	# The step after +v+, when it goes on.
115	def then(v: Step, f: Sc -> Step) -> Step:
116	  match v:
117	    case Go{st}:
118	      f(st)
119	    case Stop{st}:
120	      Stop{st}
121	
122	# +yes+ when +hit+, else +no+; only the one chosen runs.
123	def when(-A: Type, hit: Bool, yes: Unit -> A, no: Unit -> A) -> A:
124	  match hit:
125	    case True{}:
126	      yes(Unit{})
127	    case False{}:
128	      no(Unit{})
129	
130	# Characters
131	# ----------
132	
133	def code(c: Char) -> U32:
134	  match c:
135	    case Chr{x}:
136	      x
137	
138	# The character +n+ places on, or NUL past the end.
139	def peek(s: String, n: Nat) -> Char:
140	  match s:
141	    case SNil{}:
142	      Chr{0}
143	    case SCon{c, t}:
144	      match n:
145	        case 0n:
146	          c
147	        case 1n+k:
148	          peek(t, k)
149	
150	def is_blank(c: Char) -> Bool:
151	  match c:
152	    case ' ':
153	      True{}
154	    case '\t':
155	      True{}
156	    case _:
157	      False{}
158	
159	def is_break(c: Char) -> Bool:
160	  Char.is_eq(c, '\n')
161	
162	def is_end(c: Char) -> Bool:
163	  U32.is_eq(code(c), 0)
164	
165	def is_blankz(+c: Char) -> Bool:
166	  Bool.or(is_blank(c), Bool.or(is_break(c), is_end(c)))
167	
168	def is_breakz(+c: Char) -> Bool:
169	  Bool.or(is_break(c), is_end(c))
170	
171	def is_flow.ind(c: Char) -> Bool:
172	  match c:
173	    case ',':
174	      True{}
175	    case '[':
176	      True{}
177	    case ']':
178	      True{}
179	    case '{':
180	      True{}
181	    case '}':
182	      True{}
183	    case _:
184	      False{}
185	
186	def between(+x: U32, lo: U32, hi: U32) -> Bool:
187	  Bool.and(U32.is_ge(x, lo), U32.is_le(x, hi))
188	
189	def is_digit(c: Char) -> Bool:
190	  Char.is_digit(c)
191	
192	def is_hex(+c: Char) -> Bool:
193	  Bool.or(Char.is_digit(c), Bool.or(between(code(c), 65, 70), between(code(c), 97, 102)))
194	
195	# A letter, digit or "-": what a tag handle's name is made of.
196	def is_word(+c: Char) -> Bool:
197	  Bool.or(Char.is_alpha(c), Bool.or(Char.is_digit(c), Char.is_eq(c, '-')))
198	
199	def is_in(c: Char, s: String) -> Bool:
200	  String.contains(s, SCon{c, SNil{}})
201	
202	# A URI character other than "%" (which starts an escape).
203	def is_uri(+c: Char) -> Bool:
204	  Bool.or(is_word(c), is_in(c, "#;/?:@&=+$,_.!~*'()[]"))
205	
206	# A tag character: a URI character but "!" and the flow indicators.
207	def is_tag(+c: Char) -> Bool:
208	  Bool.or(is_word(c), is_in(c, "#;/?:@&=+$_.~*'()"))
209	
210	# What YAML allows in a document: tab, line feed, and printable characters.
211	def is_printable(c: Char) -> Bool:
212	  match c:
213	    case Chr{+x}:
214	      Bool.or(Bool.or(U32.is_eq(x, 9), U32.is_eq(x, 10)),
215	        Bool.or(between(x, 32, 126), Bool.or(U32.is_eq(x, 133), Bool.or(between(x, 160, 55295),
216	          Bool.or(between(x, 57344, 65533), between(x, 65536, 1114111))))))
217	
218	def hex.digit(c: Char) -> U32:
219	  +x = code(c)
220	  U32.sub(x, Bool.pick(U32, U32.is_ge(x, 97), 87, Bool.pick(U32, U32.is_ge(x, 65), 55, 48)))
221	
222	def hex.value(s: String, acc: U32) -> U32:
223	  match s:
224	    case SNil{}:
225	      acc
226	    case SCon{c, t}:
227	      hex.value(t, U32.add(U32.mul(acc, 16), hex.digit(c)))
228	
229	def all.hex(s: String) -> Bool:
230	  match s:
231	    case SNil{}:
232	      True{}
233	    case SCon{+c, t}:
234	      Bool.and(is_hex(c), all.hex(t))
235	
236	# "---" or "..." (+c+ thrice) and then a blank, a break or the end.
237	def marker(+s: String, +c: Char) -> Bool:
238	  Bool.and(Bool.and(Char.is_eq(peek(s, 0n), c), Char.is_eq(peek(s, 1n), c)), Bool.and(Char.is_eq(peek(s, 2n), c), is_blankz(peek(s, 3n))))
239	
240	def is_marker(+s: String) -> Bool:
241	  Bool.or(marker(s, '-'), marker(s, '.'))
242	
243	# Line breaks as "\n": "\r\n" and a lone "\r" both become one, as YAML reads
244	# them.
245	def lf(s: String) -> String:
246	  match s:
247	    case SNil{}:
248	      SNil{}
249	    case SCon{c, t}:
250	      match c:
251	        case '\r':
252	          match t:
253	            case SCon{'\n', u}:
254	              SCon{'\n', lf(u)}
255	            case _:
256	              SCon{'\n', lf(t)}
257	        case _:
258	          SCon{c, lf(t)}
259	
260	def strip.bom(s: String) -> String:
261	  match s:
262	    case SCon{c, +t}:
263	      match c:
264	        case Chr{+x}:
265	          Bool.pick(String, U32.is_eq(x, 65279), t, SCon{Chr{x}, t})
266	    case SNil{}:
267	      SNil{}
268	
269	# The cursor
270	# ----------
271	
272	def past(at: T.Mark, c: Char) -> T.Mark:
273	  match at:
274	    case T.Mark{line, col, idx}:
275	      match c:
276	        case '\n':
277	          T.Mark{1n+line, 0n, 1n+idx}
278	        case _:
279	          T.Mark{line, 1n+col, 1n+idx}
280	
281	def adv(c: Cur) -> Cur:
282	  match c:
283	    case Cur{s, at, left}:
284	      match s:
285	        case SNil{}:
286	          Cur{SNil{}, at, left}
287	        case SCon{h, t}:
288	          Cur{t, past(at, h), Nat.sub(left, 1n)}
289	
290	def adv.n(n: Nat, c: Cur) -> Cur:
291	  match n:
292	    case 0n:
293	      c
294	    case 1n+k:
295	      adv.n(k, adv(c))
296	
297	def cur.s(c: Cur) -> String:
298	  match c:
299	    case Cur{s, at, left}:
300	      s
301	
302	def cur.at(c: Cur) -> T.Mark:
303	  match c:
304	    case Cur{s, at, left}:
305	      at
306	
307	def cur.left(c: Cur) -> Nat:
308	  match c:
309	    case Cur{s, at, left}:
310	      left
311	
312	def cur.col(c: Cur) -> Nat:
313	  T.mark.col(cur.at(c))
314	
315	# The scanner's state
316	# -------------------
317	
318	def sc.cur(st: Sc) -> Cur:
319	  match st:
320	    case Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}:
321	      cur
322	
323	def sc.s(st: Sc) -> String:
324	  cur.s(sc.cur(st))
325	
326	def sc.at(st: Sc) -> T.Mark:
327	  cur.at(sc.cur(st))
328	
329	def sc.col(st: Sc) -> Nat:
330	  cur.col(sc.cur(st))
331	
332	def sc.ind(st: Sc) -> Nat:
333	  match st:
334	    case Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}:
335	      ind
336	
337	def sc.flow(st: Sc) -> Nat:
338	  match st:
339	    case Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}:
340	      flow
341	
342	def sc.in_flow(st: Sc) -> Bool:
343	  Nat.is_gt(sc.flow(st), 0n)
344	
345	def sc.tab(st: Sc) -> Bool:
346	  match st:
347	    case Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}:
348	      tab
349	
350	def sc.allowed(st: Sc) -> Bool:
351	  match st:
352	    case Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}:
353	      allowed
354	
355	def sc.keys(st: Sc) -> List<&2, SKey>:
356	  match st:
357	    case Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}:
358	      keys
359	
360	def sc.out(st: Sc) -> List<&2, T.Token>:
361	  match st:
362	    case Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}:
363	      out
364	
365	def sc.with.cur(st: Sc, c: Cur) -> Sc:
366	  match st:
367	    case Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}:
368	      Sc{c, ind, inds, flow, allowed, tab, keys, out, n}
369	
370	def sc.allow(st: Sc, b: Bool) -> Sc:
371	  match st:
372	    case Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}:
373	      Sc{cur, ind, inds, flow, b, tab, keys, out, n}
374	
375	def sc.with.keys(st: Sc, ks: List<&2, SKey>) -> Sc:
376	  match st:
377	    case Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}:
378	      Sc{cur, ind, inds, flow, allowed, tab, ks, out, n}
379	
380	def skip(+st: Sc, k: Nat) -> Sc:
381	  sc.with.cur(st, adv.n(k, sc.cur(st)))
382	
383	# A token from +start+ to +end+.
384	def emit.span(st: Sc, tok: T.Tok, start: T.Mark, end: T.Mark) -> Sc:
385	  match st:
386	    case Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}:
387	      Sc{cur, ind, inds, flow, allowed, tab, keys, T.Token{tok, start, end} <> out, 1n+n}
388	
389	# A token from +start+ to here.
390	def emit(+st: Sc, tok: T.Tok, start: T.Mark) -> Sc:
391	  emit.span(st, tok, start, sc.at(st))
392	
393	# A one-character token here.
394	def single(+st: Sc, tok: T.Tok) -> Sc:
395	  emit(skip(st, 1n), tok, sc.at(st))
396	
397	def fail(+st: Sc, why: String, +at: T.Mark) -> Step:
398	  Stop{emit(st, T.Broken{T.Error{why, at, T.Syntax{}}}, at)}
399	
400	def fail.with(+st: Sc, e: T.Error) -> Step:
401	  match e:
402	    case T.Error{why, at, cause}:
403	      fail(st, why, at)
404	
405	# "found character that cannot start any token"
406	def cannot(+st: Sc) -> Step:
407	  fail(st, "found character that cannot start any token while scanning for the next token", sc.at(st))
408	
409	# The token before this one: whether it was a JSON-like node (a quoted
410	# scalar or a flow collection), after which ":" is a value indicator even
411	# without a space.
412	def json.tok(t: T.Tok) -> Bool:
413	  match t:
414	    case T.ScalarTok{text, style}:
415	      match style:
416	        case T.Single{}:
417	          True{}
418	        case T.Double{}:
419	          True{}
420	        case _:
421	          False{}
422	    case T.FSeqEnd{}:
423	      True{}
424	    case T.FMapEnd{}:
425	      True{}
426	    case _:
427	      False{}
428	
429	def json.before(out: List<&2, T.Token>) -> Bool:
430	  match out:
431	    case Nil{}:
432	      False{}
433	    case Con{t, rest}:
434	      match t:
435	        case T.Token{tok, start, end}:
436	          json.tok(tok)
437	
438	# Simple keys
439	# -----------
440	
441	# The possible key on top, which a ":" would make a key.
442	def key.top(keys: List<&2, SKey>) -> SKey:
443	  match keys:
444	    case Nil{}:
445	      SKey{False{}, False{}, 0n, T.Mark{0n, 0n, 0n}, False{}}
446	    case Con{k, rest}:
447	      k
448	
449	def key.or(r: Result<&2, &2, T.Error, List<&2, SKey>>, +st: Sc) -> Step:
450	  match r:
451	    case Fail{e}:
452	      fail.with(st, e)
453	    case Done{ks}:
454	      Go{sc.with.keys(st, ks)}
455	
456	def key.removed(keys: List<&2, SKey>) -> Result<&2, &2, T.Error, List<&2, SKey>>:
457	  match keys:
458	    case Nil{}:
459	      Done{[]}
460	    case Con{k, rest}:
461	      match k:
462	        case SKey{+possible, +required, number, +at, tab}:
463	          when(Result<&2, &2, T.Error, List<&2, SKey>>, Bool.and(possible, required),
464	            u => Fail{T.Error{"could not find expected ':' while scanning a simple key", at, T.Syntax{}}},
465	            u => Done{SKey{False{}, required, number, at, tab} <> rest})
466	
467	# The key on top can no longer be; an error if it had to be.
468	def remove.key(+st: Sc) -> Step:
469	  key.or(key.removed(sc.keys(st)), st)
470	
471	def key.put(st: Sc, required: Bool) -> Sc:
472	  match st:
473	    case Sc{+cur, ind, inds, flow, allowed, +tab, keys, out, +n}:
474	      Sc{cur, ind, inds, flow, allowed, tab, SKey{True{}, required, n, cur.at(cur), tab} <> List.drop(&2, SKey, keys, 1n), out, n}
475	
476	def key.saved(r: Step, +required: Bool) -> Step:
477	  match r:
478	    case Go{st}:
479	      Go{key.put(st, required)}
480	    case Stop{st}:
481	      Stop{st}
482	
483	# A simple key may start here. It must be one (no "?" can come later) when
484	# it starts a block line at the indentation.
485	def save.key(+st: Sc) -> Step:
486	  +required = Bool.and(Nat.is_eq(sc.flow(st), 0n), Nat.is_eq(sc.ind(st), 1n+sc.col(st)))
487	  when(Step, sc.allowed(st), u => key.saved(remove.key(st), required), u => Go{st})
488	
489	def stale.cons(k: SKey, r: Result<&2, &2, T.Error, List<&2, SKey>>) -> Result<&2, &2, T.Error, List<&2, SKey>>:
490	  match r:
491	    case Fail{e}:
492	      Fail{e}
493	    case Done{ks}:
494	      Done{k <> ks}
495	
496	# Keys left behind: a simple key stays on one line and within 1024
497	# characters.
498	def stale.go(keys: List<&2, SKey>, +at: T.Mark) -> Result<&2, &2, T.Error, List<&2, SKey>>:
499	  match keys:
500	    case Nil{}:
501	      Done{[]}
502	    case Con{k, rest}:
503	      match k:
504	        case SKey{+possible, +required, number, +kat, tab}:
505	          +old = Bool.and(possible, Bool.or(Nat.is_lt(T.mark.line(kat), T.mark.line(at)), Nat.is_lt(Nat.add(T.mark.idx(kat), 1024n), T.mark.idx(at))))
506	          when(Result<&2, &2, T.Error, List<&2, SKey>>, Bool.and(old, required),
507	            u => Fail{T.Error{"could not find expected ':' while scanning a simple key", kat, T.Syntax{}}},
508	            u => stale.cons(SKey{Bool.and(possible, Bool.not(old)), required, number, kat, tab}, stale.go(rest, at)))
509	
510	def stale(+st: Sc) -> Step:
511	  key.or(stale.go(sc.keys(st), sc.at(st)), st)
512	
513	def key.cleared(keys: List<&2, SKey>) -> List<&2, SKey>:
514	  match keys:
515	    case Nil{}:
516	      []
517	    case Con{k, rest}:
518	      match k:
519	        case SKey{possible, required, number, at, tab}:
520	          SKey{False{}, required, number, at, tab} <> rest
521	
522	def flow.push(st: Sc) -> Sc:
523	  match st:
524	    case Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}:
525	      Sc{cur, ind, inds, 1n+flow, allowed, tab, SKey{False{}, False{}, 0n, T.Mark{0n, 0n, 0n}, False{}} <> keys, out, n}
526	
527	def flow.pop(st: Sc) -> Sc:
528	  match st:
529	    case Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}:
530	      match flow:
531	        case 0n:
532	          Sc{cur, ind, inds, 0n, allowed, tab, keys, out, n}
533	        case 1n+f:
534	          Sc{cur, ind, inds, f, allowed, tab, List.drop(&2, SKey, keys, 1n), out, n}
535	
536	# Indentation
537	# -----------
538	
539	def insert.at(d: Nat, out: List<&2, T.Token>, t: T.Token) -> List<&2, T.Token>:
540	  match d:
541	    case 0n:
542	      t <> out
543	    case 1n+k:
544	      match out:
545	        case Nil{}:
546	          [t]
547	        case Con{h, rest}:
548	          h <> insert.at(k, rest, t)
549	
550	# A token at the end, or (+number+) before the token with that number.
551	def insert(out: List<&2, T.Token>, n: Nat, number: Maybe<&2, Nat>, t: T.Token) -> List<&2, T.Token>:
552	  match number:
553	    case None{}:
554	      t <> out
555	    case Some{k}:
556	      insert.at(Nat.sub(n, k), out, t)
557	
558	def roll.do(st: Sc, col: Nat, number: Maybe<&2, Nat>, tok: T.Tok, +at: T.Mark) -> Sc:
559	  match st:
560	    case Sc{cur, ind, inds, flow, allowed, tab, keys, out, +n}:
561	      Sc{cur, 1n+col, ind <> inds, flow, allowed, tab, keys, insert(out, n, number, T.Token{tok, at, at}), 1n+n}
562	
563	# A block collection starts at +col+ when that's deeper than the indentation.
564	def roll(+st: Sc, +col: Nat, number: Maybe<&2, Nat>, tok: T.Tok, at: T.Mark) -> Sc:
565	  when(Sc, Bool.and(Nat.is_eq(sc.flow(st), 0n), Nat.is_lt(sc.ind(st), 1n+col)), u => roll.do(st, col, number, tok, at), u => st)
566	
567	def unroll.go(inds: List<&2, Nat>, +ind: Nat, +lim: Nat, +at: T.Mark, +out: List<&2, T.Token>, +n: Nat) -> Lay:
568	  match inds:
569	    case Nil{}:
570	      Lay{ind, [], out, n}
571	    case Con{+h, +t}:
572	      when(Lay, Nat.is_gt(ind, lim), u => unroll.go(t, h, lim, at, T.Token{T.BEnd{}, at, at} <> out, 1n+n), u => Lay{ind, h <> t, out, n})
573	
574	def unrolled(l: Lay, cur: Cur, flow: Nat, allowed: Bool, tab: Bool, keys: List<&2, SKey>) -> Sc:
575	  match l:
576	    case Lay{ind, inds, out, n}:
577	      Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}
578	
579	def unroll.do(st: Sc, +lim: Nat) -> Sc:
580	  match st:
581	    case Sc{+cur, ind, inds, flow, allowed, tab, keys, out, n}:
582	      unrolled(unroll.go(inds, ind, lim, cur.at(cur), out, n), cur, flow, allowed, tab, keys)
583	
584	# Block collections deeper than +lim+ (a column plus one) end.
585	def unroll(+st: Sc, +lim: Nat) -> Sc:
586	  when(Sc, sc.in_flow(st), u => st, u => unroll.do(st, lim))
587	
588	def skip.or(eat: Bool, more: Unit -> Skipped, stop: Skipped) -> Skipped:
589	  match eat:
590	    case True{}:
591	      more(Unit{})
592	    case False{}:
593	      stop
594	
595	# Blanks, comments and line breaks before a token. A "#" starts a comment
596	# after a blank or at a line's start (+gap+). A line break in block context
597	# lets a simple key start. Along the way: whether a tab came on the token's
598	# line (+tab+), whether a line break came (+nl+), the spaces that start the
599	# token's line (+sp+, while +lead+), and whether a tab stood where block
600	# indentation belongs (+bad+).
601	def skip.go(s: String, +at: T.Mark, +left: Nat, +flow: Bool, +ind: Nat, +allowed: Bool, +comment: Bool, +gap: Bool,
602	            +tab: Bool, +nl: Bool, +lead: Bool, +sp: Nat, +bad: Bool) -> Skipped:
603	  match s:
604	    case SNil{}:
605	      Skipped{Cur{SNil{}, at, left}, allowed, tab, nl, sp, False{}}
606	    case SCon{+c, +t}:
607	      +brk = is_break(c)
608	      +space = Char.is_eq(c, ' ')
609	      +htab = Char.is_eq(c, '\t')
610	      +eat = Bool.or(Bool.or(brk, comment), Bool.or(Bool.or(space, htab), Bool.and(Char.is_eq(c, '#'), gap)))
611	      skip.or(eat,
612	        u => skip.go(t, past(at, c), Nat.sub(left, 1n), flow, ind, Bool.or(allowed, Bool.and(brk, Bool.not(flow))),
613	          Bool.and(Bool.not(brk), Bool.or(comment, Char.is_eq(c, '#'))), True{},
614	          Bool.and(Bool.not(brk), Bool.or(tab, htab)), Bool.or(nl, brk), Bool.or(brk, Bool.and(lead, Bool.or(space, htab))),
615	          Bool.pick(Nat, brk, 0n, Bool.pick(Nat, Bool.and(Bool.and(lead, space), Bool.not(tab)), 1n+sp, sp)),
616	          Bool.and(Bool.not(brk), Bool.or(bad, Bool.and(Bool.and(htab, lead), Bool.and(Bool.not(flow), Nat.is_lt(1n+sp, ind)))))),
617	        Skipped{Cur{SCon{c, t}, at, left}, allowed, tab, nl, sp, bad})
618	
619	def skipped(k: Skipped, st: Sc) -> Sc:
620	  match k:
621	    case Skipped{cur, allowed, tab, nl, sp, bad}:
622	      match st:
623	        case Sc{c, ind, inds, flow, a, b, keys, out, n}:
624	          Sc{cur, ind, inds, flow, allowed, tab, keys, out, n}
625	
626	# Whether the last token was a plain or block scalar, which end past the
627	# blanks after them, so a "#" here follows a blank.
628	def last.plain(out: List<&2, T.Token>) -> Bool:
629	  match out:
630	    case Nil{}:
631	      False{}
632	    case Con{t, rest}:
633	      match t:
634	        case T.Token{tok, start, end}:
635	          match tok:
636	            case T.ScalarTok{text, style}:
637	              match style:
638	                case T.Plain{}:
639	                  True{}
640	                case T.Literal{}:
641	                  True{}
642	                case T.Folded{}:
643	                  True{}
644	                case _:
645	                  False{}
646	            case _:
647	              False{}
648	
649	# What the skipped whitespace allows: no tab where block indentation
650	# belongs, and in flow, lines indented past the block around them.
651	def skip.check(k: Skipped, +st: Sc) -> Step:
652	  match k:
653	    case Skipped{+cur, +allowed, +tab, +nl, +sp, +bad}:
654	      +content = Bool.not(is_end(peek(cur.s(cur), 0n)))
655	      +at = cur.at(cur)
656	      when(Step, Bool.and(bad, content), u => fail(st, "found a tab character where an indentation space is expected", at),
657	        u => when(Step, Bool.and(Bool.and(sc.in_flow(st), nl), Bool.and(content, Nat.is_lt(sp, sc.ind(st)))),
658	          u => fail(st, "did not find expected indentation in a flow collection", at),
659	          u => Go{skipped(Skipped{cur, allowed, tab, nl, sp, bad}, st)}))
660	
661	def skip.ws.at(c: Cur, +st: Sc) -> Step:
662	  match c:
663	    case Cur{s, +at, left}:
664	      +lead = Nat.is_eq(T.mark.col(at), 0n)
665	      skip.check(skip.go(s, at, left, sc.in_flow(st), sc.ind(st), sc.allowed(st), False{}, Bool.or(lead, last.plain(sc.out(st))),
666	        False{}, False{}, lead, 0n, False{}), st)
667	
668	def skip.ws(+st: Sc) -> Step:
669	  skip.ws.at(sc.cur(st), st)
670	
671	# Folded line breaks: one break is a space, more are newlines (less one).
672	def folded.breaks(+brks: Nat) -> String:
673	  Bool.pick(String, Nat.is_eq(brks, 0n), " ", String.repeat("\n", brks))
674	
675	# The blanks or breaks before a word join the text.
676	def flush(acc: String, ws: String, lead: Bool, +brks: Nat) -> String:
677	  match lead:
678	    case True{}:
679	      folded.breaks(brks) ++ acc
680	    case False{}:
681	      ws ++ acc
682	
683	# In a word: a blank ends it; ": " or a flow indicator in flow ends the scalar.
684	def pl.word(cur: Cur, +acc: String, +end: T.Mark, +ws: String, +lead: Bool, +brks: Nat, +flow: Bool) -> Pl:
685	  match cur:
686	    case Cur{+s, +at, +left}:
687	      +c = peek(s, 0n)
688	      +c1 = peek(s, 1n)
689	      +stop = Bool.or(Bool.and(Char.is_eq(c, ':'), Bool.or(is_blankz(c1), Bool.and(flow, is_flow.ind(c1)))), Bool.and(flow, is_flow.ind(c)))
690	      Bool.pick(Pl, is_blankz(c), Pl{Cur{s, at, left}, acc, end, ws, lead, brks, False{}, False{}},
691	        Bool.pick(Pl, stop, PlDone{Cur{s, at, left}, acc, end, lead},
692	          Pl{Cur{String.drop(s, 1n), past(at, c), Nat.sub(left, 1n)}, SCon{c, flush(acc, ws, lead, brks)}, past(at, c), "", False{}, 0n, True{}, False{}}))
693	
694	# Between words: blanks and breaks wait to join the text; then the next word
695	# starts, unless the scalar ended (less indented, a document marker, a
696	# comment, or the end). A tab may not stand where indentation belongs on a
697	# line that goes on.
698	def pl.blank(cur: Cur, +acc: String, +end: T.Mark, +ws: String, +lead: Bool, +brks: Nat, +bad: Bool, +ind: Nat, +flow: Bool, +start: T.Mark) -> Pl:
699	  match cur:
700	    case Cur{+s, +at, +left}:
701	      +c = peek(s, 0n)
702	      +next = {Cur{String.drop(s, 1n), past(at, c), Nat.sub(left, 1n)} : Cur}
703	      +here = {Cur{s, at, left} : Cur}
704	      +col = T.mark.col(at)
705	      +done = Bool.or(Bool.or(is_end(c), Bool.and(Bool.not(flow), Nat.is_lt(col, ind))),
706	        Bool.or(Bool.and(Nat.is_eq(col, 0n), is_marker(s)), Char.is_eq(c, '#')))
707	      +tabbed = Bool.and(Bool.and(lead, Char.is_eq(c, '\t')), Nat.is_lt(col, ind))
708	      Bool.pick(Pl, is_blank(c), Pl{next, acc, end, Bool.pick(String, lead, ws, SCon{c, ws}), lead, brks, False{}, Bool.or(bad, tabbed)},
709	        Bool.pick(Pl, is_break(c), Pl{next, acc, end, Bool.pick(String, lead, ws, ""), True{}, Bool.pick(Nat, lead, 1n+brks, brks), False{}, False{}},
710	          Bool.pick(Pl, done, PlDone{here, acc, end, lead},
711	            Bool.pick(Pl, bad, PlBad{T.Error{"found a tab character that violates indentation while scanning a plain scalar", start, T.Syntax{}}},
712	              Pl{here, acc, end, ws, lead, brks, True{}, False{}}))))
713	
714	def pl.step(p: Pl, +ind: Nat, +flow: Bool, +start: T.Mark) -> Pl:
715	  match p:
716	    case Pl{cur, acc, end, ws, lead, brks, word, bad}:
717	      match word:
718	        case True{}:
719	          pl.word(cur, acc, end, ws, lead, brks, flow)
720	        case False{}:
721	          pl.blank(cur, acc, end, ws, lead, brks, bad, ind, flow, start)
722	    case _:
723	      p
724	
725	def pl.run(fuel: Nat, +ind: Nat, +flow: Bool, +start: T.Mark, p: Pl) -> Pl:
726	  match fuel:
727	    case 0n:
728	      p
729	    case 1n+k:
730	      match p:
731	        case Pl{cur, acc, end, ws, lead, brks, word, bad}:
732	          pl.run(k, ind, flow, start, pl.step(Pl{cur, acc, end, ws, lead, brks, word, bad}, ind, flow, start))
733	        case _:
734	          p
735	
736	def plain.done(p: Pl, +st: Sc, start: T.Mark) -> Step:
737	  match p:
738	    case PlDone{cur, acc, end, lead}:
739	      Go{sc.allow(emit.span(sc.with.cur(st, cur), T.ScalarTok{String.reverse(acc), T.Plain{}}, start, end), lead)}
740	    case PlBad{e}:
741	      fail.with(st, e)
742	    case Pl{cur, acc, end, ws, lead, brks, word, bad}:
743	      Go{sc.allow(emit.span(sc.with.cur(st, cur), T.ScalarTok{String.reverse(acc), T.Plain{}}, start, end), lead)}
744	
745	def plain.scan(+st: Sc) -> Step:
746	  +cur = sc.cur(st)
747	  +at = cur.at(cur)
748	  plain.done(pl.run(Nat.add(Nat.mul(cur.left(cur), 2n), 4n), sc.ind(st), sc.in_flow(st), at, Pl{cur, "", at, "", False{}, 0n, True{}, False{}}), st, at)
749	
750	def plain(+st: Sc) -> Step:
751	  then(save.key(st), st => plain.scan(sc.allow(st, False{})))
752	
753	def q.text(cur: Cur, acc: String) -> Qs:
754	  Qs{cur, acc, "", False{}, 0n, False{}, True{}, False{}}
755	
756	def simple.escape(e: Char) -> Maybe<&2, Char>:
757	  match e:
758	    case '0':
759	      Some{Chr{0}}
760	    case 'a':
761	      Some{Chr{7}}
762	    case 'b':
763	      Some{Chr{8}}
764	    case 't':
765	      Some{Chr{9}}
766	    case '\t':
767	      Some{Chr{9}}
768	    case 'n':
769	      Some{Chr{10}}
770	    case 'v':
771	      Some{Chr{11}}
772	    case 'f':
773	      Some{Chr{12}}
774	    case 'r':
775	      Some{Chr{13}}
776	    case 'e':
777	      Some{Chr{27}}
778	    case ' ':
779	      Some{' '}
780	    case '"':
781	      Some{'"'}
782	    case '/':
783	      Some{'/'}
784	    case '\\':
785	      Some{'\\'}
786	    case 'N':
787	      Some{Chr{133}}
788	    case '_':
789	      Some{Chr{160}}
790	    case 'L':
791	      Some{Chr{8232}}
792	    case 'P':
793	      Some{Chr{8233}}
794	    case _:
795	      None{}
796	
797	def q.simple(m: Maybe<&2, Char>, cur: Cur, acc: String, start: T.Mark) -> Qs:
798	  match m:
799	    case None{}:
800	      QsBad{T.Error{"found unknown escape character while parsing a quoted scalar", start, T.Syntax{}}}
801	    case Some{c}:
802	      q.text(adv.n(2n, cur), SCon{c, acc})
803	
804	# A code point that is a character: not a surrogate, and at most 10FFFF.
805	def is_scalar.value(+x: U32) -> Bool:
806	  Bool.and(Bool.not(between(x, 55296, 57343)), U32.is_le(x, 1114111))
807	
808	# \x, \u or \U and +n+ hex digits.
809	def q.hex(+n: Nat, +cur: Cur, acc: String, +start: T.Mark) -> Qs:
810	  +digits = String.take(String.drop(cur.s(cur), 2n), n)
811	  +x = hex.value(digits, 0)
812	  Bool.pick(Qs, Bool.not(Bool.and(Nat.is_eq(String.length(digits), n), all.hex(digits))),
813	    QsBad{T.Error{"did not find expected hexdecimal number while parsing a quoted scalar", start, T.Syntax{}}},
814	    Bool.pick(Qs, Bool.not(is_scalar.value(x)),
815	      QsBad{T.Error{"found invalid Unicode character escape code while parsing a quoted scalar", start, T.Syntax{}}},
816	      q.text(adv.n(Nat.add(n, 2n), cur), SCon{Chr{x}, acc})))
817	
818	def q.escape(e: Char, cur: Cur, acc: String, start: T.Mark) -> Qs:
819	  match e:
820	    case '\n':
821	      Qs{adv.n(2n, cur), acc, "", True{}, 0n, True{}, False{}, False{}}
822	    case 'x':
823	      q.hex(2n, cur, acc, start)
824	    case 'u':
825	      q.hex(4n, cur, acc, start)
826	    case 'U':
827	      q.hex(8n, cur, acc, start)
828	    case _:
829	      q.simple(simple.escape(e), cur, acc, start)
830	
831	def q.char(double: Bool, c: Char, +c1: Char, +cur: Cur, +acc: String, start: T.Mark) -> Qs:
832	  match double:
833	    case False{}:
834	      match c:
835	        case '\'':
836	          Bool.pick(Qs, Char.is_eq(c1, '\''), q.text(adv.n(2n, cur), SCon{'\'', acc}), QsDone{adv(cur), acc})
837	        case _:
838	          q.text(adv(cur), SCon{c, acc})
839	    case True{}:
840	      match c:
841	        case '"':
842	          QsDone{adv(cur), acc}
843	        case '\\':
844	          q.escape(c1, cur, acc, start)
845	        case _:
846	          q.text(adv(cur), SCon{c, acc})
847	
848	def q.in.text(+double: Bool, +cur: Cur, +acc: String, +start: T.Mark) -> Qs:
849	  +s = cur.s(cur)
850	  +c = peek(s, 0n)
851	  Bool.pick(Qs, Bool.and(Nat.is_eq(cur.col(cur), 0n), is_marker(s)),
852	    QsBad{T.Error{"found unexpected document indicator while scanning a quoted scalar", start, T.Syntax{}}},
853	    Bool.pick(Qs, is_end(c), QsBad{T.Error{"found unexpected end of stream while scanning a quoted scalar", start, T.Syntax{}}},
854	      Bool.pick(Qs, Bool.or(is_blank(c), is_break(c)), Qs{cur, acc, "", False{}, 0n, False{}, False{}, False{}},
855	        q.char(double, c, peek(s, 1n), cur, acc, start))))
856	
857	# The blanks and breaks between text: kept, or folded, once text resumes.
858	def q.joined(+acc: String, ws: String, lead: Bool, +brks: Nat, esc: Bool) -> String:
859	  match lead:
860	    case False{}:
861	      ws ++ acc
862	    case True{}:
863	      Bool.pick(String, esc, String.repeat("\n", brks), folded.breaks(brks)) ++ acc
864	
865	# Between text: blanks and breaks wait to join it. A line after a break
866	# must be indented past the block around it (+ind+), with spaces.
867	def q.in.blanks(+cur: Cur, +acc: String, +ws: String, +lead: Bool, +brks: Nat, +esc: Bool, +bad: Bool, +ind: Nat, +start: T.Mark) -> Qs:
868	  +c = peek(cur.s(cur), 0n)
869	  +shallow = Nat.is_lt(cur.col(cur), ind)
870	  Bool.pick(Qs, is_blank(c), Qs{adv(cur), acc, Bool.pick(String, lead, ws, SCon{c, ws}), lead, brks, esc, False{}, Bool.or(bad, Bool.and(lead, Bool.and(shallow, Char.is_eq(c, '\t'))))},
871	    Bool.pick(Qs, is_break(c), Qs{adv(cur), acc, "", True{}, Bool.pick(Nat, lead, 1n+brks, 0n), esc, False{}, False{}},
872	      Bool.pick(Qs, Bool.and(lead, Bool.and(Bool.not(is_end(c)), Bool.or(bad, shallow))),
873	        QsBad{T.Error{"found a line indented too little in a quoted scalar while scanning a quoted scalar", start, T.Syntax{}}},
874	        q.text(cur, q.joined(acc, ws, lead, brks, esc)))))
875	
876	def q.step(q: Qs, +double: Bool, +ind: Nat, +start: T.Mark) -> Qs:
877	  match q:
878	    case Qs{cur, acc, ws, lead, brks, esc, text, bad}:
879	      match text:
880	        case True{}:
881	          q.in.text(double, cur, acc, start)
882	        case False{}:
883	          q.in.blanks(cur, acc, ws, lead, brks, esc, bad, ind, start)
884	    case _:
885	      q
886	
887	def q.run(fuel: Nat, +double: Bool, +ind: Nat, +start: T.Mark, q: Qs) -> Qs:
888	  match fuel:
889	    case 0n:
890	      q
891	    case 1n+k:
892	      match q:
893	        case Qs{cur, acc, ws, lead, brks, esc, text, bad}:
894	          q.run(k, double, ind, start, q.step(Qs{cur, acc, ws, lead, brks, esc, text, bad}, double, ind, start))
895	        case _:
896	          q
897	
898	def quoted.done(q: Qs, +st: Sc, +double: Bool, start: T.Mark) -> Step:
899	  match q:
900	    case QsDone{cur, acc}:
901	      Go{emit(sc.with.cur(st, cur), T.ScalarTok{String.reverse(acc), Bool.pick(T.Style, double, T.Double{}, T.Single{})}, start)}
902	    case QsBad{e}:
903	      fail.with(st, e)
904	    case _:
905	      fail(st, "found unexpected end of stream while scanning a quoted scalar", start)
906	
907	def quoted.scan(+st: Sc, +double: Bool) -> Step:
908	  +cur = sc.cur(st)
909	  +at = cur.at(cur)
910	  quoted.done(q.run(Nat.add(Nat.mul(cur.left(cur), 2n), 4n), double, sc.ind(st), at, q.text(adv(cur), "")), st, double, at)
911	
912	def quoted(+st: Sc, +double: Bool) -> Step:
913	  then(save.key(st), st => quoted.scan(sc.allow(st, False{}), double))
914	
915	def digit.value(c: Char) -> Nat:
916	  U32.to_nat(U32.sub(code(c), 48))
917	
918	def chomp.of(c: Char) -> Maybe<&2, Chomp>:
919	  match c:
920	    case '+':
921	      Some{Keep{}}
922	    case '-':
923	      Some{Strip{}}
924	    case _:
925	      None{}
926	
927	def chomp.or(m: Maybe<&2, Chomp>, d: Chomp) -> Chomp:
928	  match m:
929	    case None{}:
930	      d
931	    case Some{c}:
932	      c
933	
934	def is_some.chomp(m: Maybe<&2, Chomp>) -> Bool:
935	  match m:
936	    case None{}:
937	      False{}
938	    case Some{c}:
939	      True{}
940	
941	# Past blanks and a comment (after a blank), to a line break or the end.
942	def head.tail(s: String, +at: T.Mark, +left: Nat, +comment: Bool, +gap: Bool) -> Cur:
943	  match s:
944	    case SNil{}:
945	      Cur{SNil{}, at, left}
946	    case SCon{+c, +t}:
947	      +hash = Bool.and(Char.is_eq(c, '#'), gap)
948	      when(Cur, Bool.or(is_break(c), Bool.not(Bool.or(comment, Bool.or(is_blank(c), hash)))), u => Cur{SCon{c, t}, at, left},
949	        u => head.tail(t, past(at, c), Nat.sub(left, 1n), Bool.or(comment, hash), is_blank(c)))
950	
951	def head.check(chomp: Chomp, +inc: Nat, +cur: Cur) -> Head:
952	  Bool.pick(Head, is_breakz(peek(cur.s(cur), 0n)), Head{chomp, inc, adv(cur)}, HeadBad{"did not find expected comment or line break while scanning a block scalar"})
953	
954	def head.end(chomp: Chomp, +inc: Nat, cur: Cur) -> Head:
955	  match cur:
956	    case Cur{s, at, left}:
957	      head.check(chomp, inc, head.tail(s, at, left, False{}, False{}))
958	
959	# The indicators after "|" or ">", in either order.
960	def header(+cur: Cur) -> Head:
961	  +s = cur.s(cur)
962	  +c0 = peek(s, 0n)
963	  +c1 = peek(s, 1n)
964	  +ch0 = chomp.of(c0)
965	  +ch1 = chomp.of(c1)
966	  +d0 = Bool.and(Char.is_digit(c0), Bool.not(Char.is_eq(c0, '0')))
967	  +d1 = Bool.and(Char.is_digit(c1), Bool.not(Char.is_eq(c1, '0')))
968	  when(Head, Bool.or(Char.is_eq(c0, '0'), Bool.and(is_some.chomp(ch0), Char.is_eq(c1, '0'))),
969	    u => HeadBad{"found an indentation indicator equal to 0 while scanning a block scalar"},
970	    u => when(Head, Bool.and(is_some.chomp(ch0), d1), u => head.end(chomp.or(ch0, Clip{}), digit.value(c1), adv.n(2n, cur)),
971	      u => when(Head, is_some.chomp(ch0), u => head.end(chomp.or(ch0, Clip{}), 0n, adv(cur)),
972	        u => when(Head, Bool.and(d0, is_some.chomp(ch1)), u => head.end(chomp.or(ch1, Clip{}), digit.value(c0), adv.n(2n, cur)),
973	          u => when(Head, d0, u => head.end(Clip{}, digit.value(c0), adv(cur)), u => head.end(Clip{}, 0n, cur))))))
974	
975	# Blank lines and the indentation before content. The first content line
976	# sets the indentation when the header didn't; no blank line before it may
977	# be deeper. A tab may start content but not indentation. A last line of
978	# spaces with no break after it still counts as a line.
979	def bs.breaks(+cur: Cur, +acc: String, +ind: Nat, +max: Nat, +lead: Bool, +lblank: Bool, +brks: Nat, +pind: Nat, +start: T.Mark) -> Bs:
980	  +c = peek(cur.s(cur), 0n)
981	  +col = cur.col(cur)
982	  +inside = Bool.or(Nat.is_eq(ind, 0n), Nat.is_lt(1n+col, ind))
983	  +first = Bool.and(Nat.is_eq(ind, 0n), Nat.is_ge(col, pind))
984	  +partial = Bool.and(Nat.is_gt(col, 0n), Nat.is_gt(T.mark.line(cur.at(cur)), T.mark.line(start)))
985	  Bool.pick(Bs, Bool.and(Char.is_eq(c, ' '), inside), Bs{adv(cur), acc, ind, max, lead, lblank, brks, Breaks{}},
986	    Bool.pick(Bs, is_break(c), Bs{adv(cur), acc, ind, Nat.max(col, max), lead, lblank, 1n+brks, Breaks{}},
987	      Bool.pick(Bs, is_end(c), Bs{cur, acc, ind, max, lead, lblank, Bool.pick(Nat, partial, 1n+brks, brks), LineStart{}},
988	        Bool.pick(Bs, Bool.and(Bool.and(Char.is_eq(c, '\t'), inside), Bool.not(first)),
989	          BsBad{T.Error{"found a tab character where an indentation space is expected while scanning a block scalar", start, T.Syntax{}}},
990	          Bool.pick(Bs, Bool.and(first, Nat.is_gt(max, col)),
991	            BsBad{T.Error{"found a leading blank line with more spaces than the first content line while scanning a block scalar", start, T.Syntax{}}},
992	            Bs{cur, acc, Bool.pick(Nat, Nat.is_eq(ind, 0n), 1n+Nat.max(col, pind), ind), max, lead, lblank, brks, LineStart{}})))))
993	
994	# A content line starts at the indentation: what waits joins the text.
995	def bs.line(+cur: Cur, +acc: String, +ind: Nat, +max: Nat, +lead: Bool, +lblank: Bool, +brks: Nat, +folded: Bool) -> Bs:
996	  +s = cur.s(cur)
997	  +c = peek(s, 0n)
998	  +fold = Bool.and(Bool.and(folded, lead), Bool.and(Bool.not(lblank), Bool.not(is_blank(c))))
999	  +joined = String.repeat("\n", brks) ++ Bool.pick(String, fold, Bool.pick(String, Nat.is_eq(brks, 0n), " ", ""), Bool.pick(String, lead, "\n", "")) ++ acc
1000	  Bool.pick(Bs, Bool.or(Bool.not(Nat.is_eq(1n+cur.col(cur), ind)), Bool.or(is_end(c), Bool.and(Nat.is_eq(cur.col(cur), 0n), is_marker(s)))),
1001	    BsDone{acc, lead, brks, cur},
1002	    Bs{cur, joined, ind, max, False{}, is_blank(c), 0n, Content{}})
1003	
1004	def bs.content(+cur: Cur, +acc: String, +ind: Nat, +max: Nat, +lblank: Bool) -> Bs:
1005	  +c = peek(cur.s(cur), 0n)
1006	  Bool.pick(Bs, is_end(c), BsDone{acc, True{}, 0n, cur},
1007	    Bool.pick(Bs, is_break(c), Bs{adv(cur), acc, ind, max, True{}, lblank, 0n, Breaks{}},
1008	      Bs{adv(cur), SCon{c, acc}, ind, max, False{}, lblank, 0n, Content{}}))
1009	
1010	def bs.step(b: Bs, +folded: Bool, +pind: Nat, +start: T.Mark) -> Bs:
1011	  match b:
1012	    case Bs{cur, acc, ind, max, lead, lblank, brks, phase}:
1013	      match phase:
1014	        case Breaks{}:
1015	          bs.breaks(cur, acc, ind, max, lead, lblank, brks, pind, start)
1016	        case LineStart{}:
1017	          bs.line(cur, acc, ind, max, lead, lblank, brks, folded)
1018	        case Content{}:
1019	          bs.content(cur, acc, ind, max, lblank)
1020	    case _:
1021	      b
1022	
1023	def bs.run(fuel: Nat, +folded: Bool, +pind: Nat, +start: T.Mark, b: Bs) -> Bs:
1024	  match fuel:
1025	    case 0n:
1026	      b
1027	    case 1n+k:
1028	      match b:
1029	        case Bs{cur, acc, ind, max, lead, lblank, brks, phase}:
1030	          bs.run(k, folded, pind, start, bs.step(Bs{cur, acc, ind, max, lead, lblank, brks, phase}, folded, pind, start))
1031	        case _:
1032	          b
1033	
1034	# The final line break and the blank lines after the text.
1035	def chomped(chomp: Chomp, +acc: String, +lead: Bool, +brks: Nat) -> String:
1036	  match chomp:
1037	    case Clip{}:
1038	      Bool.pick(String, lead, SCon{'\n', acc}, acc)
1039	    case Strip{}:
1040	      acc
1041	    case Keep{}:
1042	      String.repeat("\n", brks) ++ Bool.pick(String, lead, SCon{'\n', acc}, acc)
1043	
1044	def block.done(b: Bs, chomp: Chomp, +st: Sc, +folded: Bool, start: T.Mark) -> Step:
1045	  match b:
1046	    case BsDone{acc, lead, brks, cur}:
1047	      Go{emit(sc.with.cur(st, cur), T.ScalarTok{String.reverse(chomped(chomp, acc, lead, brks)), Bool.pick(T.Style, folded, T.Folded{}, T.Literal{})}, start)}
1048	    case BsBad{e}:
1049	      fail.with(st, e)
1050	    case _:
1051	      fail(st, "found unexpected end of stream while scanning a block scalar", start)
1052	
1053	def block.body(h: Head, +st: Sc, +folded: Bool, +start: T.Mark) -> Step:
1054	  match h:
1055	    case Head{chomp, +inc, +cur}:
1056	      +pind = sc.ind(st)
1057	      +given = Nat.add(pind, inc)
1058	      block.done(bs.run(Nat.add(Nat.mul(cur.left(cur), 3n), 4n), folded, pind, start,
1059	        Bs{cur, "", Bool.pick(Nat, Nat.is_eq(inc, 0n), 0n, given), 0n, False{}, False{}, 0n, Breaks{}}), chomp, st, folded, start)
1060	    case HeadBad{why}:
1061	      fail(st, why, start)
1062	
1063	def block.scan(+st: Sc, +folded: Bool) -> Step:
1064	  +cur = sc.cur(st)
1065	  block.body(header(adv(cur)), st, folded, cur.at(cur))
1066	
1067	def block.scalar(+st: Sc, +folded: Bool) -> Step:
1068	  then(remove.key(st), st => block.scan(sc.allow(st, True{}), folded))
1069	
1070	def taken.cons(c: Char, +k: Nat, t: Taken) -> Taken:
1071	  match t:
1072	    case Taken{text, n}:
1073	      Taken{SCon{c, text}, Nat.add(k, n)}
1074	
1075	def took.or(stop: Bool, more: Unit -> Taken) -> Taken:
1076	  match stop:
1077	    case True{}:
1078	      Taken{"", 0n}
1079	    case False{}:
1080	      more(Unit{})
1081	
1082	# An anchor's name: anything up to a blank, a break or a flow indicator.
1083	def anchor.name(s: String) -> Taken:
1084	  match s:
1085	    case SNil{}:
1086	      Taken{"", 0n}
1087	    case SCon{+c, t}:
1088	      took.or(Bool.or(is_blankz(c), is_flow.ind(c)), u => taken.cons(c, 1n, anchor.name(t)))
1089	
1090	def anchor.took(t: Taken, +st: Sc, +is_anchor: Bool, +at: T.Mark) -> Step:
1091	  match t:
1092	    case Taken{+name, +n}:
1093	      when(Step, String.is_empty(name),
1094	        u => fail(st, Bool.pick(String, is_anchor, "did not find expected alphabetic or numeric character while scanning an anchor",
1095	          "did not find expected alphabetic or numeric character while scanning an alias"), at),
1096	        u => Go{emit(skip(st, 1n+n), Bool.pick(T.Tok, is_anchor, T.AnchorTok{name}, T.AliasTok{name}), at)})
1097	
1098	def anchor.scan(+st: Sc, +is_anchor: Bool) -> Step:
1099	  +at = sc.at(st)
1100	  anchor.took(anchor.name(String.drop(sc.s(st), 1n)), st, is_anchor, at)
1101	
1102	def anchor(+st: Sc, +is_anchor: Bool) -> Step:
1103	  then(save.key(st), st => anchor.scan(sc.allow(st, False{}), is_anchor))
1104	
1105	# Tag characters, with %XX escapes decoded; +uri+ allows all URI characters
1106	# (in a verbatim tag or a %TAG prefix).
1107	def taken.skip(t: Taken) -> Taken:
1108	  match t:
1109	    case Taken{text, n}:
1110	      Taken{text, 1n+n}
1111	
1112	def tag.chars(s: String, +uri: Bool, +pending: Nat) -> Taken:
1113	  match s:
1114	    case SNil{}:
1115	      Taken{"", 0n}
1116	    case SCon{+c, +t}:
1117	      match pending:
1118	        case 1n+k:
1119	          taken.skip(tag.chars(t, uri, k))
1120	        case 0n:
1121	          +esc = Bool.and(Char.is_eq(c, '%'), Bool.and(is_hex(peek(t, 0n)), is_hex(peek(t, 1n))))
1122	          +ok = Bool.pick(Bool, uri, is_uri(c), is_tag(c))
1123	          took.or(Bool.not(Bool.or(esc, ok)),
1124	            u => taken.cons(Bool.pick(Char, esc, Chr{hex.value(String.take(t, 2n), 0)}, c), 1n,
1125	              tag.chars(t, uri, Bool.pick(Nat, esc, 2n, 0n))))
1126	
1127	# What may follow a tag: a blank, a break, the end, or in flow a flow
1128	# indicator.
1129	def tag.ends(+c: Char, +flow: Bool) -> Bool:
1130	  Bool.or(is_blankz(c), Bool.and(flow, is_flow.ind(c)))
1131	
1132	def tag.emit(+st: Sc, handle: String, suffix: String, +n: Nat, +at: T.Mark) -> Step:
1133	  +st2 = skip(st, n)
1134	  +next = peek(sc.s(st2), 0n)
1135	  when(Step, tag.ends(next, sc.in_flow(st2)),
1136	    u => Go{emit(st2, T.TagTok{handle, suffix}, at)},
1137	    u => fail(st2, Bool.pick(String, Char.is_eq(next, '%'), "did not find URI escaped octet while parsing a tag",
1138	      "did not find expected whitespace or line break while scanning a tag"), at))
1139	
1140	# "!<uri>".
1141	def tag.verbatim(t: Taken, +st: Sc, +at: T.Mark) -> Step:
1142	  match t:
1143	    case Taken{+uri, +n}:
1144	      when(Step, Bool.or(String.is_empty(uri), Bool.not(Char.is_eq(peek(String.drop(sc.s(st), Nat.add(n, 2n)), 0n), '>'))),
1145	        u => fail(st, "did not find the expected '>' while scanning a tag", at),
1146	        u => tag.emit(st, "", uri, Nat.add(n, 3n), at))
1147	
1148	def tag.suffix(t: Taken, +named: Bool, +word: String, +k: Nat, +st: Sc, +at: T.Mark) -> Step:
1149	  match t:
1150	    case Taken{+suffix, +n}:
1151	      +handle = Bool.pick(String, named, "!" ++ word ++ "!", "!")
1152	      +used = Nat.add(1n, Bool.pick(Nat, named, Nat.add(1n+k, n), n))
1153	      when(Step, Bool.and(named, String.is_empty(suffix)),
1154	        u => fail(st, "did not find expected tag URI while parsing a tag", at),
1155	        u => when(Step, String.is_empty(suffix), u => tag.emit(st, "", "!", used, at), u => tag.emit(st, handle, suffix, used, at)))
1156	
1157	# "!suffix", "!!suffix", "!name!suffix", or "!" alone.
1158	def tag.short(w: Taken, +st: Sc, +at: T.Mark) -> Step:
1159	  match w:
1160	    case Taken{+word, +k}:
1161	      +s = String.drop(sc.s(st), 1n)
1162	      +named = Char.is_eq(peek(s, k), '!')
1163	      tag.suffix(tag.chars(String.drop(s, Bool.pick(Nat, named, 1n+k, 0n)), False{}, 0n), named, word, k, st, at)
1164	
1165	def word.chars(s: String) -> Taken:
1166	  match s:
1167	    case SNil{}:
1168	      Taken{"", 0n}
1169	    case SCon{+c, t}:
1170	      took.or(Bool.not(is_word(c)), u => taken.cons(c, 1n, word.chars(t)))
1171	
1172	def tag.scan(+st: Sc) -> Step:
1173	  +s = sc.s(st)
1174	  +at = sc.at(st)
1175	  when(Step, Char.is_eq(peek(s, 1n), '<'),
1176	    u => tag.verbatim(tag.chars(String.drop(s, 2n), True{}, 0n), st, at),
1177	    u => tag.short(word.chars(String.drop(s, 1n)), st, at))
1178	
1179	def tag(+st: Sc) -> Step:
1180	  then(save.key(st), st => tag.scan(sc.allow(st, False{})))
1181	
1182	# Directives
1183	# ----------
1184	
1185	def count.or(go: Bool, more: Unit -> Nat) -> Nat:
1186	  match go:
1187	    case True{}:
1188	      more(Unit{})
1189	    case False{}:
1190	      0n
1191	
1192	def blanks(s: String) -> Nat:
1193	  match s:
1194	    case SNil{}:
1195	      0n
1196	    case SCon{+c, t}:
1197	      count.or(is_blank(c), u => 1n+blanks(t))
1198	
1199	def digits(s: String) -> Taken:
1200	  match s:
1201	    case SNil{}:
1202	      Taken{"", 0n}
1203	    case SCon{+c, t}:
1204	      took.or(Bool.not(Char.is_digit(c)), u => taken.cons(c, 1n, digits(t)))
1205	
1206	def nat.of(s: String, acc: Nat) -> Nat:
1207	  match s:
1208	    case SNil{}:
1209	      acc
1210	    case SCon{c, t}:
1211	      nat.of(t, Nat.add(Nat.mul(acc, 10n), digit.value(c)))
1212	
1213	def directive.put(+st: Sc, tok: Maybe<&2, T.Tok>, at: T.Mark) -> Sc:
1214	  match tok:
1215	    case None{}:
1216	      st
1217	    case Some{t}:
1218	      emit(st, t, at)
1219	
1220	def directive.tail(+cur: Cur, +st: Sc, tok: Maybe<&2, T.Tok>, +at: T.Mark) -> Step:
1221	  when(Step, is_breakz(peek(cur.s(cur), 0n)),
1222	    u => Go{directive.put(sc.with.cur(st, cur), tok, at)},
1223	    u => fail(st, "did not find expected comment or line break while scanning a directive", at))
1224	
1225	# After a directive: blanks, a comment, then a line break or the end.
1226	def directive.end(+st: Sc, tok: Maybe<&2, T.Tok>, at: T.Mark) -> Step:
1227	  directive.tail(head.tail(sc.s(st), sc.at(st), cur.left(sc.cur(st)), False{}, False{}), st, tok, at)
1228	
1229	def version.done(m: Taken, +st: Sc, major: String, +used: Nat, +at: T.Mark) -> Step:
1230	  match m:
1231	    case Taken{+minor, +k}:
1232	      when(Step, Nat.is_eq(k, 0n), u => fail(st, "did not find expected digit or '.' character while scanning a %YAML directive", at),
1233	        u => directive.end(skip(st, Nat.add(used, k)), Some{T.VersionDir{nat.of(major, 0n), nat.of(minor, 0n)}}, at))
1234	
1235	def version.minor(m: Taken, +st: Sc, +b: Nat, +at: T.Mark) -> Step:
1236	  match m:
1237	    case Taken{+major, +k}:
1238	      +rest = String.drop(sc.s(st), Nat.add(b, k))
1239	      +minor = digits(String.drop(rest, 1n))
1240	      when(Step, Nat.is_eq(b, 0n), u => fail(st, "did not find expected whitespace while scanning a %YAML directive", at),
1241	        u => when(Step, Bool.or(Nat.is_eq(k, 0n), Bool.not(Char.is_eq(peek(rest, 0n), '.'))),
1242	          u => fail(st, "did not find expected digit or '.' character while scanning a %YAML directive", at),
1243	          u => version.done(minor, st, major, Nat.add(b, Nat.add(k, 1n)), at)))
1244	
1245	# "%YAML 1.2".
1246	def version.scan(+st: Sc, +at: T.Mark) -> Step:
1247	  +s = sc.s(st)
1248	  +b = blanks(s)
1249	  +major = digits(String.drop(s, b))
1250	  version.minor(major, st, b, at)
1251	
1252	def tagdir.prefix(p: Taken, +st: Sc, handle: String, +used: Nat, +at: T.Mark) -> Step:
1253	  match p:
1254	    case Taken{+prefix, +k}:
1255	      when(Step, Nat.is_eq(k, 0n), u => fail(st, "did not find expected tag URI while scanning a %TAG directive", at),
1256	        u => directive.end(skip(st, Nat.add(used, k)), Some{T.TagDir{handle, prefix}}, at))
1257	
1258	def tagdir.handle(w: Taken, +st: Sc, +b: Nat, +at: T.Mark) -> Step:
1259	  match w:
1260	    case Taken{+word, +k}:
1261	      +rest = String.drop(sc.s(st), b)
1262	      +closed = Char.is_eq(peek(rest, 1n+k), '!')
1263	      +hlen = Nat.add(1n+k, Bool.pick(Nat, closed, 1n, 0n))
1264	      +handle = String.take(rest, hlen)
1265	      +b2 = blanks(String.drop(rest, hlen))
1266	      +ok = Bool.and(Char.is_eq(peek(rest, 0n), '!'), Bool.or(closed, Nat.is_eq(k, 0n)))
1267	      when(Step, Bool.or(Nat.is_eq(b, 0n), Bool.not(ok)), u => fail(st, "did not find expected '!' while scanning a %TAG directive", at),
1268	        u => when(Step, Nat.is_eq(b2, 0n), u => fail(st, "did not find expected whitespace while scanning a %TAG directive", at),
1269	          u => tagdir.prefix(tag.chars(String.drop(rest, Nat.add(hlen, b2)), True{}, 0n), st, handle, Nat.add(b, Nat.add(hlen, b2)), at)))
1270	
1271	# "%TAG !e! prefix".
1272	def tagdir.scan(+st: Sc, +at: T.Mark) -> Step:
1273	  +s = sc.s(st)
1274	  +b = blanks(s)
1275	  +rest = String.drop(s, b)
1276	  tagdir.handle(word.chars(String.drop(rest, 1n)), st, b, at)
1277	
1278	# Past the rest of a line.
1279	def line.rest(s: String) -> Nat:
1280	  match s:
1281	    case SNil{}:
1282	      0n
1283	    case SCon{+c, t}:
1284	      count.or(Bool.not(is_break(c)), u => 1n+line.rest(t))
1285	
1286	# %YAML, %TAG, or a reserved directive, which is skipped.
1287	def directive.named(+name: String, +st: Sc, +at: T.Mark) -> Step:
1288	  when(Step, String.eq(name, "YAML"), u => version.scan(st, at),
1289	    u => when(Step, String.eq(name, "TAG"), u => tagdir.scan(st, at),
1290	      u => Go{skip(st, line.rest(sc.s(st)))}))
1291	
1292	def directive.name(w: Taken, +st: Sc, +at: T.Mark) -> Step:
1293	  match w:
1294	    case Taken{+name, +k}:
1295	      when(Step, Nat.is_eq(k, 0n), u => fail(st, "could not find expected directive name while scanning a directive", at),
1296	        u => directive.named(name, skip(st, 1n+k), at))
1297	
1298	def directive.scan(+st: Sc) -> Step:
1299	  +at = sc.at(st)
1300	  directive.name(word.chars(String.drop(sc.s(st), 1n)), st, at)
1301	
1302	def directive(+st: Sc) -> Step:
1303	  then(remove.key(unroll(st, 0n)), st => directive.scan(sc.allow(st, False{})))
1304	
1305	# Tokens
1306	# ------
1307	
1308	def flow.start(+st: Sc, t: T.Tok) -> Step:
1309	  then(save.key(st), st => Go{single(sc.allow(flow.push(st), True{}), t)})
1310	
1311	def flow.end(+st: Sc, t: T.Tok) -> Step:
1312	  then(remove.key(st), st => Go{single(sc.allow(flow.pop(st), False{}), t)})
1313	
1314	def flow.entry(+st: Sc) -> Step:
1315	  then(remove.key(st), st => Go{single(sc.allow(st, True{}), T.FEntry{})})
1316	
1317	def block.entry.rest(+st: Sc) -> Step:
1318	  then(remove.key(st), st => Go{single(sc.allow(st, True{}), T.BEntry{})})
1319	
1320	# "- ": a block sequence's entry, which may open the sequence.
1321	def block.entry(+st: Sc) -> Step:
1322	  when(Step, sc.in_flow(st), u => block.entry.rest(st),
1323	    u => when(Step, sc.tab(st), u => fail(st, "found a tab character that violates indentation", sc.at(st)),
1324	    u => when(Step, sc.allowed(st), u => block.entry.rest(roll(st, sc.col(st), None{}, T.BSeqStart{}, sc.at(st))),
1325	      u => fail(st, "block sequence entries are not allowed in this context", sc.at(st)))))
1326	
1327	def key.tok(+st: Sc) -> Sc:
1328	  single(sc.allow(st, Bool.not(sc.in_flow(st))), T.KeyTok{})
1329	
1330	def key.rest(+st: Sc) -> Step:
1331	  then(remove.key(st), st => Go{key.tok(st)})
1332	
1333	# "? ": an explicit key.
1334	def key(+st: Sc) -> Step:
1335	  when(Step, sc.in_flow(st), u => key.rest(st),
1336	    u => when(Step, sc.tab(st), u => fail(st, "found a tab character that violates indentation", sc.at(st)),
1337	    u => when(Step, sc.allowed(st), u => key.rest(roll(st, sc.col(st), None{}, T.BMapStart{}, sc.at(st))),
1338	      u => fail(st, "mapping keys are not allowed in this context", sc.at(st)))))
1339	
1340	def value.rest(+st: Sc) -> Sc:
1341	  single(sc.allow(st, Bool.not(sc.in_flow(st))), T.ValueTok{})
1342	
1343	def value.alone(+st: Sc) -> Step:
1344	  when(Step, sc.in_flow(st), u => Go{value.rest(st)},
1345	    u => when(Step, sc.allowed(st), u => Go{value.rest(roll(st, sc.col(st), None{}, T.BMapStart{}, sc.at(st)))},
1346	      u => fail(st, "mapping values are not allowed in this context", sc.at(st))))
1347	
1348	def insert.key(st: Sc, number: Nat, +at: T.Mark) -> Sc:
1349	  match st:
1350	    case Sc{cur, ind, inds, flow, allowed, tab, keys, out, +n}:
1351	      Sc{cur, ind, inds, flow, allowed, tab, key.cleared(keys), insert(out, n, Some{number}, T.Token{T.KeyTok{}, at, at}), 1n+n}
1352	
1353	# ": ": a value. When a simple key waits, it becomes a key: a KeyTok goes
1354	# before it, and a block mapping may start at its column.
1355	def value.with(k: SKey, +st: Sc) -> Step:
1356	  match k:
1357	    case SKey{possible, required, +number, +at, tab}:
1358	      +rolls = Bool.and(Nat.is_eq(sc.flow(st), 0n), Nat.is_lt(sc.ind(st), 1n+T.mark.col(at)))
1359	      when(Step, possible,
1360	        u => when(Step, Bool.and(tab, rolls), u => fail(st, "found a tab character that violates indentation", at),
1361	          u => Go{single(sc.allow(roll(insert.key(st, number, at), T.mark.col(at), Some{number}, T.BMapStart{}, at), False{}), T.ValueTok{})}),
1362	        u => value.alone(st))
1363	
1364	def value(+st: Sc) -> Step:
1365	  value.with(key.top(sc.keys(st)), st)
1366	
1367	def marker.tok(+st: Sc, t: T.Tok) -> Sc:
1368	  emit(skip(sc.allow(st, False{}), 3n), t, sc.at(st))
1369	
1370	# The stream ends on a line of its own.
1371	def end.mark(m: T.Mark) -> T.Mark:
1372	  match m:
1373	    case T.Mark{+line, +col, +idx}:
1374	      Bool.pick(T.Mark, Nat.is_eq(col, 0n), T.Mark{line, col, idx}, T.Mark{1n+line, 0n, idx})
1375	
1376	def end.tok(+st: Sc) -> Sc:
1377	  +at = end.mark(sc.at(st))
1378	  emit.span(sc.allow(st, False{}), T.StreamEnd{}, at, at)
1379	
1380	def doc.end.check(+st: Sc, t: T.Tok) -> Step:
1381	  match t:
1382	    case T.DocEnd{}:
1383	      when(Step, is_breakz(peek(cur.s(head.tail(sc.s(st), sc.at(st), cur.left(sc.cur(st)), False{}, False{})), 0n)), u => Go{st},
1384	        u => fail(st, "did not find expected comment or line break after a document end marker", sc.at(st)))
1385	    case _:
1386	      Go{st}
1387	
1388	def doc.indicator(+st: Sc, +t: T.Tok) -> Step:
1389	  then(remove.key(unroll(st, 0n)), st => doc.end.check(marker.tok(st, t), t))
1390	
1391	def stream.end(+st: Sc) -> Step:
1392	  then(remove.key(unroll(st, 0n)), st => Stop{end.tok(st)})
1393	
1394	def is_plain.start(+c: Char) -> Bool:
1395	  Bool.and(is_printable(c), Bool.not(Bool.or(is_blankz(c), is_in(c, "-?:,[]{}#&*!|>'\"%@`"))))
1396	
1397	# The token a character starts.
1398	def fetch.char(c: Char, +c1: Char, +flow: Bool, +st: Sc) -> Step:
1399	  match c:
1400	    case '[':
1401	      flow.start(st, T.FSeqStart{})
1402	    case '{':
1403	      flow.start(st, T.FMapStart{})
1404	    case ']':
1405	      flow.end(st, T.FSeqEnd{})
1406	    case '}':
1407	      flow.end(st, T.FMapEnd{})
1408	    case ',':
1409	      flow.entry(st)
1410	    case '-':
1411	      when(Step, is_blankz(c1), u => block.entry(st), u => when(Step, Bool.and(flow, is_flow.ind(c1)), u => cannot(st), u => plain(st)))
1412	    case '?':
1413	      when(Step, Bool.or(is_blankz(c1), Bool.and(flow, is_flow.ind(c1))), u => key(st), u => plain(st))
1414	    case ':':
1415	      when(Step, Bool.or(is_blankz(c1), Bool.and(flow, Bool.or(is_flow.ind(c1), json.before(sc.out(st))))), u => value(st), u => plain(st))
1416	    case '*':
1417	      anchor(st, False{})
1418	    case '&':
1419	      anchor(st, True{})
1420	    case '!':
1421	      tag(st)
1422	    case '|':
1423	      when(Step, flow, u => cannot(st), u => block.scalar(st, False{}))
1424	    case '>':
1425	      when(Step, flow, u => cannot(st), u => block.scalar(st, True{}))
1426	    case '\'':
1427	      quoted(st, False{})
1428	    case '"':
1429	      quoted(st, True{})
1430	    case _:
1431	      when(Step, is_plain.start(c), u => plain(st), u => cannot(st))
1432	
1433	def fetch.next(+st: Sc) -> Step:
1434	  +s = sc.s(st)
1435	  +c = peek(s, 0n)
1436	  +col0 = Nat.is_eq(sc.col(st), 0n)
1437	  when(Step, String.is_empty(s), u => stream.end(st),
1438	    u => when(Step, Bool.and(col0, Char.is_eq(c, '%')), u => directive(st),
1439	      u => when(Step, Bool.and(col0, marker(s, '-')), u => doc.indicator(st, T.DocStart{}),
1440	        u => when(Step, Bool.and(col0, marker(s, '.')), u => doc.indicator(st, T.DocEnd{}),
1441	          u => fetch.char(c, peek(s, 1n), sc.in_flow(st), st)))))
1442	
1443	def fetch.after(+st: Sc) -> Step:
1444	  fetch.next(unroll(st, 1n+sc.col(st)))
1445	
1446	# One token (and the block tokens before it).
1447	def fetch(+st: Sc) -> Step:
1448	  then(skip.ws(st), st => then(stale(st), st => fetch.after(st)))
1449	
1450	def scan.run(fuel: Nat, s: Step) -> List<&2, T.Token>:
1451	  match fuel:
1452	    case 0n:
1453	      []
1454	    case 1n+k:
1455	      match s:
1456	        case Go{st}:
1457	          scan.run(k, fetch(st))
1458	        case Stop{st}:
1459	          List.reverse(&2, T.Token, sc.out(st))
1460	
1461	# Where the first character YAML doesn't allow is (a control character, a
1462	# surrogate), if there is one.
1463	def unprintable(s: String, +at: T.Mark) -> Maybe<&2, T.Mark>:
1464	  match s:
1465	    case SNil{}:
1466	      None{}
1467	    case SCon{+c, t}:
1468	      when(Maybe<&2, T.Mark>, is_printable(c), u => unprintable(t, past(at, c)), u => Some{at})
1469	
1470	def stream.start() -> T.Token:
1471	  T.Token{T.StreamStart{}, T.Mark{0n, 0n, 0n}, T.Mark{0n, 0n, 0n}}
1472	
1473	def scan.start(m: Maybe<&2, T.Mark>, +s: String) -> List<&2, T.Token>:
1474	  match m:
1475	    case Some{+at}:
1476	      [stream.start(), T.Token{T.Broken{T.Error{"control characters are not allowed", at, T.Syntax{}}}, at, at}]
1477	    case None{}:
1478	      scan.run(Nat.add(String.length(s), 4n), Go{Sc{Cur{s, T.Mark{0n, 0n, 0n}, String.length(s)}, 0n, [], 0n, True{}, False{}, [SKey{False{}, False{}, 0n, T.Mark{0n, 0n, 0n}, False{}}], [stream.start()], 1n}})
1479	
1480	# The tokens of +text+, from StreamStart to StreamEnd, or to a Broken token
1481	# at the first error.
1482	def scan(text: String) -> List<&2, T.Token>:
1483	  +s = strip.bom(lf(text))
1484	  scan.start(unprintable(s, T.Mark{0n, 0n, 0n}), s)