src/text.bend

bend-toon-codec@0.1.0.0 · 260 lines · 42 definitions

1import Base
2import bend-toon-codec@0.1.0.0/src/num.bend as N
3import bend-toon-codec@0.1.0.0/src/head.bend as Hd
4
5# Strings and keys (spec section 7): escaping, quoting decisions, unescaping.
6
7# ---- characters --------------------------------------------------------------
8
9# A decoded value with the text that follows it.
10type Cut is Data:
11  Cut{val: String, rest: String}
12
13# Unescape state: normal, after a backslash, or reading the hex digits of a
14# \uXXXX escape (`k` digits to go, `acc` so far).
15type Us is Data:
16  UN{}
17  UB{}
18  UH{k: U32, acc: U32}
19
20# What one character does to the output.
21type Ek is Data:
22  EkChar{u: U32}
23  EkNone{}
24  EkEnd{}
25  EkErr{m: String}
26
27# The whitespace JavaScript's String.trim removes: what section 7.2 means by
28# leading or trailing whitespace.
29def is_ws(+c: U32) -> Bool:
30  U32.is_eq(c, 32) || (U32.is_ge(c, 9) && U32.is_le(c, 13)) || U32.is_eq(c, 160)
31    || U32.is_eq(c, 5760) || (U32.is_ge(c, 8192) && U32.is_le(c, 8202))
32    || U32.is_eq(c, 8232) || U32.is_eq(c, 8233) || U32.is_eq(c, 8239)
33    || U32.is_eq(c, 8287) || U32.is_eq(c, 12288) || U32.is_eq(c, 65279)
34
35def head_ws(+s: String) -> Bool:
36  match s:
37    case SNil{}:
38      False{}
39    case SCon{Chr{c}, t}:
40      is_ws(c)
41
42def last_ws(+s: String) -> Bool:
43  head_ws(String.reverse(s))
44
45def sel.s(+b: Bool, +x: String, +y: String) -> String:
46  match b:
47    case True{}:
48      x
49    case False{}:
50      y
51
52# ---- encoding ------------------------------------------------------------------
53
54def sel.u(+b: Bool, +x: U32, +y: U32) -> U32:
55  match b:
56    case True{}:
57      x
58    case False{}:
59      y
60
61def hexd(+n: U32) -> Char:
62  Chr{sel.u(U32.is_lt(n, 10), (n + 48 : U32), (n + 87 : U32))}
63
64# \u00xx for a control character, lowercase hex.
65def esc.ctl(+c: U32) -> String:
66  "\\u00" ++ SCon{hexd(((c >> 4n) : U32)), SCon{hexd((c .&. 15 : U32)), SNil{}}}
67
68def esc.other(+c: U32) -> String:
69  sel.s(U32.is_lt(c, 32), esc.ctl(c), SCon{Chr{c}, SNil{}})
70
71# One character inside quotes (section 7.1 encoder column).
72def esc.char(+c: U32) -> String:
73  sel.s(U32.is_eq(c, 34), "\\\"",
74  sel.s(U32.is_eq(c, 92), "\\\\",
75  sel.s(U32.is_eq(c, 10), "\\n",
76  sel.s(U32.is_eq(c, 13), "\\r",
77  sel.s(U32.is_eq(c, 9), "\\t", esc.other(c))))))
78
79def esc.body(+s: String) -> String:
80  match s:
81    case SNil{}:
82      SNil{}
83    case SCon{Chr{c}, t}:
84      esc.char(c) ++ esc.body(t)
85
86# `"text"` with escapes.
87def quote(+s: String) -> String:
88  "\"" ++ esc.body(s) ++ "\""
89
90# A character that forces quotes anywhere in a string value: colon, double
91# quote, backslash, brackets, braces, control characters, or the delimiter.
92def special(+c: U32, +d: U32) -> Bool:
93  U32.is_eq(c, 58) || U32.is_eq(c, 34) || U32.is_eq(c, 92) || U32.is_eq(c, 91)
94    || U32.is_eq(c, 93) || U32.is_eq(c, 123) || U32.is_eq(c, 125)
95    || U32.is_lt(c, 32) || U32.is_eq(c, d)
96
97def has_special(+s: String, +d: U32) -> Bool:
98  match s:
99    case SNil{}:
100      False{}
101    case SCon{Chr{c}, t}:
102      special(c, d) || has_special(t, d)
103
104def is_some(+m: Maybe<&2, N.Num>) -> Bool:
105  match m:
106    case Some{n}:
107      True{}
108    case None{}:
109      False{}
110
111def numeric_like(+s: String) -> Bool:
112  is_some(N.split(s))
113
114def starts_hyphen(+s: String) -> Bool:
115  Hd.lead(s, 45)
116
117def reserved(+s: String) -> Bool:
118  String.eq(s, "true") || String.eq(s, "false") || String.eq(s, "null")
119
120# Section 7.2: must this string value be quoted? `d` is the delimiter that
121# applies (the document delimiter for field values, the active delimiter for
122# array cells), as a code point.
123def needs_quote(+s: String, +d: U32) -> Bool:
124  String.is_empty(s) || head_ws(s) || last_ws(s) || reserved(s)
125    || numeric_like(s) || starts_hyphen(s) || has_special(s, d)
126
127# A string value, quoted when it must be.
128def enc.str(+s: String, +d: U32) -> String:
129  sel.s(needs_quote(s, d), quote(s), s)
130
131def key_start(+c: Char) -> Bool:
132  Char.is_alpha(c) || Char.is_eq(c, '_')
133
134def key_rest(+c: Char) -> Bool:
135  Char.is_alpha(c) || Char.is_digit(c) || Char.is_eq(c, '_') || Char.is_eq(c, '.')
136
137def key_tail(+s: String) -> Bool:
138  match s:
139    case SNil{}:
140      True{}
141    case SCon{c, t}:
142      key_rest(c) && key_tail(t)
143
144# `^[A-Za-z_][A-Za-z0-9_.]*$`.
145def key_plain(+s: String) -> Bool:
146  match s:
147    case SNil{}:
148      False{}
149    case SCon{c, t}:
150      key_start(c) && key_tail(t)
151
152def enc.key(+k: String) -> String:
153  sel.s(key_plain(k), k, quote(k))
154
155# ---- decoding ------------------------------------------------------------------
156
157def hex.pick(+dg: Bool, +lo: Bool, +up: Bool, +c: U32) -> Maybe<&2, U32>:
158  match dg lo up:
159    case True{} _ _:
160      Some{(c - 48 : U32)}
161    case False{} True{} _:
162      Some{(c - 87 : U32)}
163    case False{} False{} True{}:
164      Some{(c - 55 : U32)}
165    case _ _ _:
166      None{}
167
168def hex.val(+c: U32) -> Maybe<&2, U32>:
169  hex.pick(U32.is_ge(c, 48) && U32.is_le(c, 57), U32.is_ge(c, 97) && U32.is_le(c, 102),
170    U32.is_ge(c, 65) && U32.is_le(c, 70), c)
171
172def push(+c: U32, +r: Result<&2, &2, String, Cut>) -> Result<&2, &2, String, Cut>:
173  match r:
174    case Fail{e}:
175      Fail{e}
176    case Done{Cut{v, rest}}:
177      Done{Cut{SCon{Chr{c}, v}, rest}}
178
179# The code point a simple escape letter stands for, or the sentinel 4294967295.
180def esc.code(+e: U32) -> U32:
181  sel.u(U32.is_eq(e, 110), 10, sel.u(U32.is_eq(e, 114), 13, sel.u(U32.is_eq(e, 116), 9,
182    sel.u(U32.is_eq(e, 34), 34, sel.u(U32.is_eq(e, 92), 92, 4294967295)))))
183
184def us.pick(+b: Bool, +x: Us, +y: Us) -> Us:
185  match b:
186    case True{}:
187      x
188    case False{}:
189      y
190
191def hex.next(+m: Maybe<&2, U32>, +k: U32, +acc: U32) -> Us:
192  match m:
193    case None{}:
194      UN{}
195    case Some{h}:
196      us.pick(U32.is_eq(k, 1), UN{}, UH{(k - 1 : U32), (((acc << 4n) : U32) + h : U32)})
197
198# The state after reading character `c`.
199def next(+st: Us, +c: U32) -> Us:
200  match st:
201    case UN{}:
202      us.pick(U32.is_eq(c, 92), UB{}, UN{})
203    case UB{}:
204      us.pick(U32.is_eq(c, 117), UH{4, 0}, UN{})
205    case UH{k, acc}:
206      hex.next(hex.val(c), k, acc)
207
208def ek.pick(+b: Bool, +x: Ek, +y: Ek) -> Ek:
209  match b:
210    case True{}:
211      x
212    case False{}:
213      y
214
215def simple.emit(+code: U32) -> Ek:
216  ek.pick(U32.is_eq(code, 4294967295), EkErr{"invalid escape sequence"}, EkChar{code})
217
218# A finished \uXXXX value: a surrogate is refused (section 7.1).
219def unit.emit(+n: U32) -> Ek:
220  ek.pick(U32.is_ge(n, 55296) && U32.is_le(n, 57343), EkErr{"surrogate code point in \\u escape"}, EkChar{n})
221
222def hex.emit(+m: Maybe<&2, U32>, +k: U32, +acc: U32) -> Ek:
223  match m:
224    case None{}:
225      EkErr{"invalid \\u escape"}
226    case Some{h}:
227      ek.pick(U32.is_eq(k, 1), unit.emit((((acc << 4n) : U32) + h : U32)), EkNone{})
228
229# What character `c` emits in state `st`.
230def emit(+st: Us, +c: U32) -> Ek:
231  match st:
232    case UN{}:
233      ek.pick(U32.is_eq(c, 34), EkEnd{}, ek.pick(U32.is_eq(c, 92), EkNone{}, EkChar{c}))
234    case UB{}:
235      ek.pick(U32.is_eq(c, 117), EkNone{}, simple.emit(esc.code(c)))
236    case UH{k, acc}:
237      hex.emit(hex.val(c), k, acc)
238
239def unq.k(+ek: Ek, +t: String, +r: Result<&2, &2, String, Cut>) -> Result<&2, &2, String, Cut>:
240  match ek:
241    case EkChar{u}:
242      push(u, r)
243    case EkNone{}:
244      r
245    case EkEnd{}:
246      Done{Cut{SNil{}, t}}
247    case EkErr{m}:
248      Fail{m}
249
250def unq.go(+s: String, +st: Us) -> Result<&2, &2, String, Cut>:
251  match s:
252    case SNil{}:
253      Fail{"unterminated string"}
254    case SCon{Chr{c}, t}:
255      unq.k(emit(st, c), t, unq.go(t, next(st, c)))
256
257# Unescape a quoted string whose opening quote is already consumed; the
258# result carries the text after the closing quote.
259def unq(+s: String) -> Result<&2, &2, String, Cut>:
260  unq.go(s, UN{})