src/unescape.bend

emerging-ezhttp@0.21.0.0 · 344 lines · 42 definitions

1# ezhttp/src/unescape: HTML character references back to the characters they
2# stand for, the way Python's html.unescape does it (the HTML Standard's
3# §13.2.5.72 to §13.2.5.80 character reference states, with Python's choices
4# where the parser would report an error). Numeric references are decimal
5# (`&`) or hexadecimal (`&`); named ones come from the HTML
6# Standard's table (src/entities.bend), the legacy names also without their
7# `;`. Pure. SPEC.md HTTP-REF lists what is guaranteed.
8import Base
9import emerging-ezhttp@0.21.0.0/src/entities.bend as Ents
10
11# where the scan stands: in text, after `&`, after `&#`, after `&#x` (the `x`
12# or `X` kept), inside a number (hexadecimal or not, its value so far), or
13# inside a name (its text so far and how many characters it has)
14type Scan is Data:
15  Txt{}
16  Amp{}
17  Hash{}
18  HashX{x: Char}
19  Num{hex: Bool, n: U32}
20  Name{text: String, len: Nat}
21
22# U+FFFD REPLACEMENT CHARACTER, alone
23def fffd() -> String:
24  SCon{Chr{65533}, ""}
25
26# --- numeric references (HTTP-REF-2) ---
27
28# the value a number is held at once it is past U+10FFFF: any larger one
29# reads the same (U+FFFD)
30def cap() -> U32:
31  1114112
32
33# a digit's value in a base (16 or 10), or 16 when it is not one
34def digit.of(hex: Bool, +cp: U32) -> U32:
35  match hex:
36    case True{}:
37      Bool.pick(U32, Bool.and(U32.is_le(48, cp), U32.is_le(cp, 57)),
38        U32.sub(cp, 48),
39        Bool.pick(U32, Bool.and(U32.is_le(65, cp), U32.is_le(cp, 70)),
40          U32.sub(cp, 55),
41          Bool.pick(U32, Bool.and(U32.is_le(97, cp), U32.is_le(cp, 102)),
42            U32.sub(cp, 87), 16)))
43    case False{}:
44      Bool.pick(U32, Bool.and(U32.is_le(48, cp), U32.is_le(cp, 57)),
45        U32.sub(cp, 48), 16)
46
47# a character's value as a digit in a base, or 16 when it is not one
48def digit(hex: Bool, c: Char) -> U32:
49  digit.of(hex, Char.to_u32(c))
50
51# a number with one more digit, held at cap() once past it
52def grow(hex: Bool, +n: U32, +d: U32) -> U32:
53  U32.min(U32.add(U32.mul(n, Bool.pick(U32, hex, 16, 10)), d), cap())
54
55# what 0x80 to 0x9F stand for (HTML §13.2.5.80's table: Windows-1252, the
56# five it leaves undefined as themselves)
57def w1252() -> List<&2, U32>:
58  [8364, 129, 8218, 402, 8222, 8230, 8224, 8225, 710, 8240, 352, 8249, 338,
59    141, 381, 143, 144, 8216, 8217, 8220, 8221, 8226, 8211, 8212, 732, 8482,
60    353, 8250, 339, 157, 382, 376]
61
62# the code point at a place in a list, 65533 past its end
63def nth(xs: List<&2, U32>, i: Nat) -> U32:
64  match xs i:
65    case [] _:
66      65533
67    case x <> t 0n:
68      x
69    case x <> t 1n+p:
70      nth(t, p)
71
72# a code point Python drops: C0 controls but TAB, LF, FF and CR, DEL and the
73# C1 controls, and the noncharacters U+FDD0 to U+FDEF and U+xFFFE, U+xFFFF
74def dropped(+n: U32) -> Bool:
75  Bool.or(Bool.and(U32.is_le(1, n), U32.is_le(n, 8)),
76    Bool.or(U32.is_eq(n, 11),
77      Bool.or(Bool.and(U32.is_le(14, n), U32.is_le(n, 31)),
78        Bool.or(Bool.and(U32.is_le(127, n), U32.is_le(n, 159)),
79          Bool.or(Bool.and(U32.is_le(64976, n), U32.is_le(n, 65007)),
80            U32.is_eq(U32.and(n, 65534), 65534))))))
81
82# a code point in range: nothing when Python drops it, else the character
83def code.kept(+n: U32, drop: Bool) -> String:
84  match drop:
85    case True{}:
86      ""
87    case False{}:
88      SCon{Chr{n}, ""}
89
90# a surrogate or a number past U+10FFFF is U+FFFD
91def code.range(+n: U32, bad: Bool) -> String:
92  match bad:
93    case True{}:
94      fffd()
95    case False{}:
96      code.kept(n, dropped(n))
97
98# 0x80 to 0x9F as the table says
99def code.c1(+n: U32, c1: Bool) -> String:
100  match c1:
101    case True{}:
102      SCon{Chr{nth(w1252(), U32.to_nat(U32.sub(n, 128)))}, ""}
103    case False{}:
104      code.range(n, Bool.or(Bool.and(U32.is_le(55296, n), U32.is_le(n, 57343)),
105        U32.is_lt(1114111, n)))
106
107# 0x0D is a carriage return
108def code.cr(+n: U32, cr: Bool) -> String:
109  match cr:
110    case True{}:
111      "\r"
112    case False{}:
113      code.c1(n, Bool.and(U32.is_le(128, n), U32.is_le(n, 159)))
114
115# 0 is U+FFFD, else on to 0x0D
116def code.zero(+n: U32, zero: Bool) -> String:
117  match zero:
118    case True{}:
119      fffd()
120    case False{}:
121      code.cr(n, U32.is_eq(n, 13))
122
123# what a numeric reference stands for: 0 is U+FFFD
124def code(+n: U32) -> String:
125  code.zero(n, U32.is_eq(n, 0))
126
127# --- named references (HTTP-REF-3) ---
128
129# a name's length, or 0 for none yet
130def best.len(m: Maybe<&2, Ents.Ref>) -> Nat:
131  match m:
132    case None{}:
133      0n
134    case Some{Ents.Ref{name, first, second}}:
135      String.length(name)
136
137# the better of the best so far and a name the text starts with: the longer
138def best.take(best: Maybe<&2, Ents.Ref>, +r: Ents.Ref, longer: Bool)
139  -> Maybe<&2, Ents.Ref>:
140  match longer:
141    case True{}:
142      Some{r}
143    case False{}:
144      best
145
146# a name the text starts with or not
147def best.hit(+best: Maybe<&2, Ents.Ref>, +r: Ents.Ref, +name: String,
148  hit: Bool) -> Maybe<&2, Ents.Ref>:
149  match hit:
150    case True{}:
151      best.take(best, r, Nat.is_lt(best.len(best), String.length(name)))
152    case False{}:
153      best
154
155# one table row against the best so far
156def best.step(+best: Maybe<&2, Ents.Ref>, +r: Ents.Ref, +s: String)
157  -> Maybe<&2, Ents.Ref>:
158  match r:
159    case Ents.Ref{+name, first, second}:
160      best.hit(best, r, name, String.starts_with(s, name))
161
162# the longest name in the rows that the text starts with
163def best(rs: List<&2, Ents.Ref>, +s: String, +sofar: Maybe<&2, Ents.Ref>)
164  -> Maybe<&2, Ents.Ref>:
165  match rs:
166    case []:
167      sofar
168    case r <> t:
169      best(t, s, best.step(sofar, r, s))
170
171# the one or two characters a row stands for
172def chars(first: U32, +second: U32) -> String:
173  SCon{Chr{first}, Bool.pick(String, U32.is_eq(second, 0), "",
174    SCon{Chr{second}, ""})}
175
176# a reference's text once its longest name is known: the characters, then
177# the text after the name as written; with no name, `&` and the text
178def named.of(m: Maybe<&2, Ents.Ref>, s: String) -> String:
179  match m:
180    case None{}:
181      SCon{'&', s}
182    case Some{Ents.Ref{+name, first, second}}:
183      chars(first, second) ++ String.drop(s, String.length(name))
184
185# a named reference's text (after the `&`, with its `;` if it has one)
186def named(+s: String) -> String:
187  named.of(best(Ents.table(), s, None{}), s)
188
189# --- the scan (HTTP-REF-1) ---
190
191# a character that may be in a name: anything but TAB, LF, FF, space, `<`,
192# `&`, `#` and `;`
193def name.char(+c: Char) -> Bool:
194  Bool.not(Bool.or(Char.is_eq(c, '\t'), Bool.or(Char.is_eq(c, '\n'),
195    Bool.or(Char.is_eq(c, '\u{C}'), Bool.or(Char.is_eq(c, ' '),
196      Bool.or(Char.is_eq(c, '<'), Bool.or(Char.is_eq(c, '&'),
197        Bool.or(Char.is_eq(c, '#'), Char.is_eq(c, ';')))))))))
198
199# a character in text once it is known whether it is `&`
200def text.of(+c: Char, k: Scan -> String, amp: Bool) -> String:
201  match amp:
202    case True{}:
203      k(Amp{})
204    case False{}:
205      SCon{c, k(Txt{})}
206
207# a character in text: `&` may open a reference
208def text(+c: Char, k: Scan -> String) -> String:
209  text.of(c, k, Char.is_eq(c, '&'))
210
211# after `&`: a name character opens a name, else the `&` is text
212def amp.name(+c: Char, k: Scan -> String, name: Bool) -> String:
213  match name:
214    case True{}:
215      k(Name{SCon{c, ""}, 1n})
216    case False{}:
217      SCon{'&', text(c, k)}
218
219# after `&`: `#` opens a number
220def amp(+c: Char, k: Scan -> String, hash: Bool) -> String:
221  match hash:
222    case True{}:
223      k(Hash{})
224    case False{}:
225      amp.name(c, k, name.char(c))
226
227# after `&#`: `x` or `X` opens a hexadecimal number
228def hash.x(+c: Char, k: Scan -> String, x: Bool) -> String:
229  match x:
230    case True{}:
231      k(HashX{c})
232    case False{}:
233      "&#" ++ text(c, k)
234
235# after `&#`, once it is known whether the character is a decimal digit
236def hash.dec(+c: Char, k: Scan -> String, +d: U32, digit: Bool) -> String:
237  match digit:
238    case True{}:
239      k(Num{False{}, d})
240    case False{}:
241      hash.x(c, k, Bool.or(Char.is_eq(c, 'x'), Char.is_eq(c, 'X')))
242
243# after `&#`: a decimal digit opens a decimal number
244def hash(+c: Char, k: Scan -> String, +d: U32) -> String:
245  hash.dec(c, k, d, U32.is_lt(d, 16))
246
247# after `&#x`, once it is known whether the character is a hex digit
248def hash.hex.of(+x: Char, +c: Char, k: Scan -> String, +d: U32, digit: Bool)
249  -> String:
250  match digit:
251    case True{}:
252      k(Num{True{}, d})
253    case False{}:
254      SCon{'&', SCon{'#', SCon{x, text(c, k)}}}
255
256# after `&#x`: a hex digit opens the number, else `&#x` is text
257def hash.hex(+x: Char, +c: Char, k: Scan -> String, +d: U32) -> String:
258  hash.hex.of(x, c, k, d, U32.is_lt(d, 16))
259
260# inside a number, a character that is not a digit: `;` ends the reference
261# with it, anything else ends it and is text
262def num.end(+n: U32, +c: Char, k: Scan -> String, semi: Bool) -> String:
263  match semi:
264    case True{}:
265      code(n) ++ k(Txt{})
266    case False{}:
267      code(n) ++ text(c, k)
268
269# inside a number: a digit grows it
270def num(+hex: Bool, +n: U32, +c: Char, k: Scan -> String, +d: U32,
271  digit: Bool) -> String:
272  match digit:
273    case True{}:
274      k(Num{hex, grow(hex, n, d)})
275    case False{}:
276      num.end(n, c, k, Char.is_eq(c, ';'))
277
278# inside a name, a character that does not join it: the name ends there
279def name.end(+s: String, +c: Char, k: Scan -> String) -> String:
280  named(s) ++ text(c, k)
281
282# inside a name, a character that is not `;`: a name character joins it
283# while it has fewer than 32
284def name.more(+s: String, len: Nat, +c: Char, k: Scan -> String, join: Bool)
285  -> String:
286  match join:
287    case True{}:
288      k(Name{s ++ SCon{c, ""}, 1n+len})
289    case False{}:
290      name.end(s, c, k)
291
292# inside a name: `;` ends the reference with it
293def name(+s: String, +len: Nat, +c: Char, k: Scan -> String, semi: Bool)
294  -> String:
295  match semi:
296    case True{}:
297      named(s ++ ";") ++ k(Txt{})
298    case False{}:
299      name.more(s, len, c, k, Bool.and(name.char(c), Nat.is_lt(len, 32n)))
300
301# one character where the scan stands
302def step(st: Scan, +c: Char, k: Scan -> String) -> String:
303  match st:
304    case Txt{}:
305      text(c, k)
306    case Amp{}:
307      amp(c, k, Char.is_eq(c, '#'))
308    case Hash{}:
309      hash(c, k, digit(False{}, c))
310    case HashX{x}:
311      hash.hex(x, c, k, digit(True{}, c))
312    case Num{+hex, n}:
313      num(hex, n, c, k, digit(hex, c), U32.is_lt(digit(hex, c), 16))
314    case Name{s, len}:
315      name(s, len, c, k, Char.is_eq(c, ';'))
316
317# the end of the text: a reference left open ends there, an `&`, `&#` or
318# `&#x` with nothing after it is text
319def end(st: Scan) -> String:
320  match st:
321    case Txt{}:
322      ""
323    case Amp{}:
324      "&"
325    case Hash{}:
326      "&#"
327    case HashX{x}:
328      SCon{'&', SCon{'#', SCon{x, ""}}}
329    case Num{hex, n}:
330      code(n)
331    case Name{s, len}:
332      named(s)
333
334# a text scanned from where it stands, character by character
335def scan(s: String, st: Scan) -> String:
336  match s:
337    case SNil{}:
338      end(st)
339    case SCon{+c, t}:
340      step(st, c, st2 => scan(t, st2))
341
342# a text with its character references replaced, as html.unescape
343def unescape(s: String) -> String:
344  scan(s, Txt{})