1 # ezhttp/src/unescape: HTML character references back to the characters they
2 # stand for, the way Python's html.unescape does it (the HTML Standard's
3 # §13.2.5.72 to §13.2.5.80 character reference states, with Python's choices
4 # where the parser would report an error). Numeric references are decimal
5 # (`&`) or hexadecimal (`&`); named ones come from the HTML
6 # Standard's table (src/entities.bend), the legacy names also without their
7 # `;`. Pure. SPEC.md HTTP-REF lists what is guaranteed.
8 import Base
9 import ./entities.bend as Ents
10
11 # where the scan stands: in text, after `&`, after `&#`, after `&#x` (the `x`
12 # or `X` kept), inside a number (hexadecimal or not, its value so far), or
13 # inside a name (its text so far and how many characters it has)
14 type Scan is Data:
15 Txt{}
16 Amp{}
17 Hash{}
18 HashX{x: Char}
19 Num{hex: Bool, n: U32}
20 Name{text: String, len: Nat}
21
22 # U+FFFD REPLACEMENT CHARACTER, alone
23 def fffd() -> String:
24 SCon{Chr{65533}, ""}
25
26 # --- numeric references (HTTP-REF-2) ---
27
28 # the value a number is held at once it is past U+10FFFF: any larger one
29 # reads the same (U+FFFD)
30 def cap() -> U32:
31 1114112
32
33 # a digit's value in a base (16 or 10), or 16 when it is not one
34 def digit.of(hex: Bool, +cp: U32) -> U32:
35 match hex:
36 case True{}:
37 Bool.pick(U32, Bool.and(U32.is_le(48, cp), U32.is_le(cp, 57)),
38 U32.sub(cp, 48),
39 Bool.pick(U32, Bool.and(U32.is_le(65, cp), U32.is_le(cp, 70)),
40 U32.sub(cp, 55),
41 Bool.pick(U32, Bool.and(U32.is_le(97, cp), U32.is_le(cp, 102)),
42 U32.sub(cp, 87), 16)))
43 case False{}:
44 Bool.pick(U32, Bool.and(U32.is_le(48, cp), U32.is_le(cp, 57)),
45 U32.sub(cp, 48), 16)
46
47 # a character's value as a digit in a base, or 16 when it is not one
48 def digit(hex: Bool, c: Char) -> U32:
49 digit.of(hex, Char.to_u32(c))
50
51 # a number with one more digit, held at cap() once past it
52 def grow(hex: Bool, +n: U32, +d: U32) -> U32:
53 U32.min(U32.add(U32.mul(n, Bool.pick(U32, hex, 16, 10)), d), cap())
54
55 # what 0x80 to 0x9F stand for (HTML §13.2.5.80's table: Windows-1252, the
56 # five it leaves undefined as themselves)
57 def w1252() -> List<&2, U32>:
58 [8364, 129, 8218, 402, 8222, 8230, 8224, 8225, 710, 8240, 352, 8249, 338,
59 141, 381, 143, 144, 8216, 8217, 8220, 8221, 8226, 8211, 8212, 732, 8482,
60 353, 8250, 339, 157, 382, 376]
61
62 # the code point at a place in a list, 65533 past its end
63 def nth(xs: List<&2, U32>, i: Nat) -> U32:
64 match xs i:
65 case [] _:
66 65533
67 case x <> t 0n:
68 x
69 case x <> t 1n+p:
70 nth(t, p)
71
72 # a code point Python drops: C0 controls but TAB, LF, FF and CR, DEL and the
73 # C1 controls, and the noncharacters U+FDD0 to U+FDEF and U+xFFFE, U+xFFFF
74 def dropped(+n: U32) -> Bool:
75 Bool.or(Bool.and(U32.is_le(1, n), U32.is_le(n, 8)),
76 Bool.or(U32.is_eq(n, 11),
77 Bool.or(Bool.and(U32.is_le(14, n), U32.is_le(n, 31)),
78 Bool.or(Bool.and(U32.is_le(127, n), U32.is_le(n, 159)),
79 Bool.or(Bool.and(U32.is_le(64976, n), U32.is_le(n, 65007)),
80 U32.is_eq(U32.and(n, 65534), 65534))))))
81
82 # a code point in range: nothing when Python drops it, else the character
83 def code.kept(+n: U32, drop: Bool) -> String:
84 match drop:
85 case True{}:
86 ""
87 case False{}:
88 SCon{Chr{n}, ""}
89
90 # a surrogate or a number past U+10FFFF is U+FFFD
91 def code.range(+n: U32, bad: Bool) -> String:
92 match bad:
93 case True{}:
94 fffd()
95 case False{}:
96 code.kept(n, dropped(n))
97
98 # 0x80 to 0x9F as the table says
99 def code.c1(+n: U32, c1: Bool) -> String:
100 match c1:
101 case True{}:
102 SCon{Chr{nth(w1252(), U32.to_nat(U32.sub(n, 128)))}, ""}
103 case False{}:
104 code.range(n, Bool.or(Bool.and(U32.is_le(55296, n), U32.is_le(n, 57343)),
105 U32.is_lt(1114111, n)))
106
107 # 0x0D is a carriage return
108 def code.cr(+n: U32, cr: Bool) -> String:
109 match cr:
110 case True{}:
111 "\r"
112 case False{}:
113 code.c1(n, Bool.and(U32.is_le(128, n), U32.is_le(n, 159)))
114
115 # 0 is U+FFFD, else on to 0x0D
116 def code.zero(+n: U32, zero: Bool) -> String:
117 match zero:
118 case True{}:
119 fffd()
120 case False{}:
121 code.cr(n, U32.is_eq(n, 13))
122
123 # what a numeric reference stands for: 0 is U+FFFD
124 def code(+n: U32) -> String:
125 code.zero(n, U32.is_eq(n, 0))
126
127 # --- named references (HTTP-REF-3) ---
128
129 # a name's length, or 0 for none yet
130 def best.len(m: Maybe<&2, Ents.Ref>) -> Nat:
131 match m:
132 case None{}:
133 0n
134 case Some{Ents.Ref{name, first, second}}:
135 String.length(name)
136
137 # the better of the best so far and a name the text starts with: the longer
138 def best.take(best: Maybe<&2, Ents.Ref>, +r: Ents.Ref, longer: Bool)
139 -> Maybe<&2, Ents.Ref>:
140 match longer:
141 case True{}:
142 Some{r}
143 case False{}:
144 best
145
146 # a name the text starts with or not
147 def best.hit(+best: Maybe<&2, Ents.Ref>, +r: Ents.Ref, +name: String,
148 hit: Bool) -> Maybe<&2, Ents.Ref>:
149 match hit:
150 case True{}:
151 best.take(best, r, Nat.is_lt(best.len(best), String.length(name)))
152 case False{}:
153 best
154
155 # one table row against the best so far
156 def best.step(+best: Maybe<&2, Ents.Ref>, +r: Ents.Ref, +s: String)
157 -> Maybe<&2, Ents.Ref>:
158 match r:
159 case Ents.Ref{+name, first, second}:
160 best.hit(best, r, name, String.starts_with(s, name))
161
162 # the longest name in the rows that the text starts with
163 def best(rs: List<&2, Ents.Ref>, +s: String, +sofar: Maybe<&2, Ents.Ref>)
164 -> Maybe<&2, Ents.Ref>:
165 match rs:
166 case []:
167 sofar
168 case r <> t:
169 best(t, s, best.step(sofar, r, s))
170
171 # the one or two characters a row stands for
172 def chars(first: U32, +second: U32) -> String:
173 SCon{Chr{first}, Bool.pick(String, U32.is_eq(second, 0), "",
174 SCon{Chr{second}, ""})}
175
176 # a reference's text once its longest name is known: the characters, then
177 # the text after the name as written; with no name, `&` and the text
178 def named.of(m: Maybe<&2, Ents.Ref>, s: String) -> String:
179 match m:
180 case None{}:
181 SCon{'&', s}
182 case Some{Ents.Ref{+name, first, second}}:
183 chars(first, second) ++ String.drop(s, String.length(name))
184
185 # a named reference's text (after the `&`, with its `;` if it has one)
186 def named(+s: String) -> String:
187 named.of(best(Ents.table(), s, None{}), s)
188
189 # --- the scan (HTTP-REF-1) ---
190
191 # a character that may be in a name: anything but TAB, LF, FF, space, `<`,
192 # `&`, `#` and `;`
193 def name.char(+c: Char) -> Bool:
194 Bool.not(Bool.or(Char.is_eq(c, '\t'), Bool.or(Char.is_eq(c, '\n'),
195 Bool.or(Char.is_eq(c, '\u{C}'), Bool.or(Char.is_eq(c, ' '),
196 Bool.or(Char.is_eq(c, '<'), Bool.or(Char.is_eq(c, '&'),
197 Bool.or(Char.is_eq(c, '#'), Char.is_eq(c, ';')))))))))
198
199 # a character in text once it is known whether it is `&`
200 def text.of(+c: Char, k: Scan -> String, amp: Bool) -> String:
201 match amp:
202 case True{}:
203 k(Amp{})
204 case False{}:
205 SCon{c, k(Txt{})}
206
207 # a character in text: `&` may open a reference
208 def text(+c: Char, k: Scan -> String) -> String:
209 text.of(c, k, Char.is_eq(c, '&'))
210
211 # after `&`: a name character opens a name, else the `&` is text
212 def amp.name(+c: Char, k: Scan -> String, name: Bool) -> String:
213 match name:
214 case True{}:
215 k(Name{SCon{c, ""}, 1n})
216 case False{}:
217 SCon{'&', text(c, k)}
218
219 # after `&`: `#` opens a number
220 def amp(+c: Char, k: Scan -> String, hash: Bool) -> String:
221 match hash:
222 case True{}:
223 k(Hash{})
224 case False{}:
225 amp.name(c, k, name.char(c))
226
227 # after `&#`: `x` or `X` opens a hexadecimal number
228 def hash.x(+c: Char, k: Scan -> String, x: Bool) -> String:
229 match x:
230 case True{}:
231 k(HashX{c})
232 case False{}:
233 "&#" ++ text(c, k)
234
235 # after `&#`, once it is known whether the character is a decimal digit
236 def hash.dec(+c: Char, k: Scan -> String, +d: U32, digit: Bool) -> String:
237 match digit:
238 case True{}:
239 k(Num{False{}, d})
240 case False{}:
241 hash.x(c, k, Bool.or(Char.is_eq(c, 'x'), Char.is_eq(c, 'X')))
242
243 # after `&#`: a decimal digit opens a decimal number
244 def hash(+c: Char, k: Scan -> String, +d: U32) -> String:
245 hash.dec(c, k, d, U32.is_lt(d, 16))
246
247 # after `&#x`, once it is known whether the character is a hex digit
248 def hash.hex.of(+x: Char, +c: Char, k: Scan -> String, +d: U32, digit: Bool)
249 -> String:
250 match digit:
251 case True{}:
252 k(Num{True{}, d})
253 case False{}:
254 SCon{'&', SCon{'#', SCon{x, text(c, k)}}}
255
256 # after `&#x`: a hex digit opens the number, else `&#x` is text
257 def hash.hex(+x: Char, +c: Char, k: Scan -> String, +d: U32) -> String:
258 hash.hex.of(x, c, k, d, U32.is_lt(d, 16))
259
260 # inside a number, a character that is not a digit: `;` ends the reference
261 # with it, anything else ends it and is text
262 def num.end(+n: U32, +c: Char, k: Scan -> String, semi: Bool) -> String:
263 match semi:
264 case True{}:
265 code(n) ++ k(Txt{})
266 case False{}:
267 code(n) ++ text(c, k)
268
269 # inside a number: a digit grows it
270 def num(+hex: Bool, +n: U32, +c: Char, k: Scan -> String, +d: U32,
271 digit: Bool) -> String:
272 match digit:
273 case True{}:
274 k(Num{hex, grow(hex, n, d)})
275 case False{}:
276 num.end(n, c, k, Char.is_eq(c, ';'))
277
278 # inside a name, a character that does not join it: the name ends there
279 def name.end(+s: String, +c: Char, k: Scan -> String) -> String:
280 named(s) ++ text(c, k)
281
282 # inside a name, a character that is not `;`: a name character joins it
283 # while it has fewer than 32
284 def name.more(+s: String, len: Nat, +c: Char, k: Scan -> String, join: Bool)
285 -> String:
286 match join:
287 case True{}:
288 k(Name{s ++ SCon{c, ""}, 1n+len})
289 case False{}:
290 name.end(s, c, k)
291
292 # inside a name: `;` ends the reference with it
293 def name(+s: String, +len: Nat, +c: Char, k: Scan -> String, semi: Bool)
294 -> String:
295 match semi:
296 case True{}:
297 named(s ++ ";") ++ k(Txt{})
298 case False{}:
299 name.more(s, len, c, k, Bool.and(name.char(c), Nat.is_lt(len, 32n)))
300
301 # one character where the scan stands
302 def step(st: Scan, +c: Char, k: Scan -> String) -> String:
303 match st:
304 case Txt{}:
305 text(c, k)
306 case Amp{}:
307 amp(c, k, Char.is_eq(c, '#'))
308 case Hash{}:
309 hash(c, k, digit(False{}, c))
310 case HashX{x}:
311 hash.hex(x, c, k, digit(True{}, c))
312 case Num{+hex, n}:
313 num(hex, n, c, k, digit(hex, c), U32.is_lt(digit(hex, c), 16))
314 case Name{s, len}:
315 name(s, len, c, k, Char.is_eq(c, ';'))
316
317 # the end of the text: a reference left open ends there, an `&`, `&#` or
318 # `&#x` with nothing after it is text
319 def end(st: Scan) -> String:
320 match st:
321 case Txt{}:
322 ""
323 case Amp{}:
324 "&"
325 case Hash{}:
326 "&#"
327 case HashX{x}:
328 SCon{'&', SCon{'#', SCon{x, ""}}}
329 case Num{hex, n}:
330 code(n)
331 case Name{s, len}:
332 named(s)
333
334 # a text scanned from where it stands, character by character
335 def scan(s: String, st: Scan) -> String:
336 match s:
337 case SNil{}:
338 end(st)
339 case SCon{+c, t}:
340 step(st, c, st2 => scan(t, st2))
341
342 # a text with its character references replaced, as html.unescape
343 def unescape(s: String) -> String:
344 scan(s, Txt{})