src/utf8.bend

bend-kube-runtime@0.2.0.0 · 182 lines · 27 definitions

1import Base
2
3# UTF-8
4# -----
5#
6# Bytes arrive in arbitrary cuts, so the decoder carries an unfinished
7# sequence across calls: (code so far, continuation bytes still needed).
8# A malformed sequence decodes to U+FFFD, as io_str does.
9
10type Utf8 is Data:
11  Utf8{code: U32, need: U32}
12
13def Utf8.init() -> Utf8:
14  Utf8{0, 0}
15
16def Utf8.bad() -> Char:
17  Chr{65533}
18
19def Utf8.lead.go(+b: U32, a: Bool, c: Bool, two: Bool, three: Bool, four: Bool) -> U32:
20  match a c two three four:
21    case True{} _ _ _ _:
22      0
23    case False{} True{} _ _ _:
24      4
25    case False{} False{} True{} _ _:
26      1
27    case False{} False{} False{} True{} _:
28      2
29    case False{} False{} False{} False{} True{}:
30      3
31    case _ _ _ _ _:
32      4
33
34# The lead byte's class: 0 ASCII, 1..3 continuation count, 4 invalid.
35def Utf8.lead(+b: U32) -> U32:
36  Utf8.lead.go(b, U32.is_lt(b, 128), U32.is_lt(b, 194), U32.is_lt(b, 224),
37    U32.is_lt(b, 240), U32.is_lt(b, 245))
38
39def Utf8.is_cont(+b: U32) -> Bool:
40  U32.is_eq((b .&. 192 : U32), 128)
41
42# The fresh-byte case: the state and the chars a lead byte makes.
43def Utf8.fresh.st(+b: U32, +k: U32, mid: Bool) -> Utf8:
44  match mid:
45    case True{}:
46      Utf8{U32.and(b, U32.shrn(63, U32.to_nat(k))), k}
47    case False{}:
48      Utf8{0, 0}
49
50def Utf8.fresh.out(+b: U32, +k: U32, acc: String, ascii: Bool, bad: Bool) -> String:
51  match ascii bad:
52    case True{} _:
53      SCon{Chr{b}, acc}
54    case False{} True{}:
55      SCon{Utf8.bad(), acc}
56    case False{} False{}:
57      acc
58
59def Utf8.fresh.mid(+k: U32) -> Bool:
60  Bool.and(Bool.not(U32.is_eq(k, 0)), Bool.not(U32.is_eq(k, 4)))
61
62def Utf8.next.cont.if(+code: U32, +need: U32, done: Bool) -> Utf8:
63  match done:
64    case True{}:
65      Utf8{0, 0}
66    case False{}:
67      Utf8{code, need}
68
69def Utf8.next.cont(+code: U32, +need: U32) -> Utf8:
70  Utf8.next.cont.if(code, need, U32.is_eq(need, 0))
71
72# The state after byte b.
73def Utf8.next.go(+code: U32, +need: U32, +b: U32, cont: Bool, idle: Bool) -> Utf8:
74  match cont idle:
75    case True{} False{}:
76      Utf8.next.cont(((code << 6n) .|. (b .&. 63) : U32), (need - 1 : U32))
77    case _ _:
78      Utf8.fresh.st(b, Utf8.lead(b), Utf8.fresh.mid(Utf8.lead(b)))
79
80def Utf8.next(st: Utf8, +b: U32) -> Utf8:
81  match st:
82    case Utf8{+code, +need}:
83      Utf8.next.go(code, need, b, Utf8.is_cont(b), U32.is_eq(need, 0))
84
85def Utf8.emit.cont(+code: U32, acc: String, done: Bool) -> String:
86  match done:
87    case True{}:
88      SCon{Chr{code}, acc}
89    case False{}:
90      acc
91
92# The chars byte b completes, pushed onto acc (reversed).
93def Utf8.emit.go(+code: U32, +need: U32, +b: U32, acc: String, cont: Bool, idle: Bool) -> String:
94  match cont idle:
95    case True{} False{}:
96      Utf8.emit.cont(((code << 6n) .|. (b .&. 63) : U32), acc, U32.is_eq(need, 1))
97    case False{} False{}:
98      Utf8.fresh.out(b, Utf8.lead(b), SCon{Utf8.bad(), acc},
99        U32.is_eq(Utf8.lead(b), 0), U32.is_eq(Utf8.lead(b), 4))
100    case _ _:
101      Utf8.fresh.out(b, Utf8.lead(b), acc, U32.is_eq(Utf8.lead(b), 0), U32.is_eq(Utf8.lead(b), 4))
102
103def Utf8.emit(st: Utf8, +b: U32, acc: String) -> String:
104  match st:
105    case Utf8{+code, +need}:
106      Utf8.emit.go(code, need, b, acc, Utf8.is_cont(b), U32.is_eq(need, 0))
107
108# Decodes bytes onto the front of acc (reversed), carrying the state.
109def Utf8.go(bs: List<&2, U32>, +st: Utf8, acc: String) -> Utf8 & String:
110  match bs:
111    case Nil{}:
112      (st, acc)
113    case Con{+b, t}:
114      Utf8.go(t, Utf8.next(st, b), Utf8.emit(st, b, acc))
115
116def Utf8.decode.fin(r: Utf8 & String) -> Utf8 & String:
117  match r:
118    case (st, acc):
119      (st, String.reverse(acc))
120
121# Decodes a cut of bytes; returns the carried state and the text.
122def Utf8.decode(st: Utf8, bs: List<&2, U32>) -> Utf8 & String:
123  Utf8.decode.fin(Utf8.go(bs, st, SNil{}))
124
125def Utf8.decode_all.tail(s: String, whole: Bool) -> String:
126  match whole:
127    case True{}:
128      s
129    case False{}:
130      s ++ SCon{Utf8.bad(), SNil{}}
131
132def Utf8.decode_all.fin(r: Utf8 & String) -> String:
133  match r:
134    case (Utf8{code, need}, s):
135      Utf8.decode_all.tail(s, U32.is_eq(need, 0))
136
137# Decodes a whole buffer.
138def Utf8.decode_all(bs: List<&2, U32>) -> String:
139  Utf8.decode_all.fin(Utf8.decode(Utf8.init(), bs))
140
141def Utf8.put.go(+c: U32, rest: List<&2, U32>, a: Bool, b: Bool, d: Bool) -> List<&2, U32>:
142  match a b d:
143    case True{} _ _:
144      c <> rest
145    case False{} True{} _:
146      (192 .|. (c >> 6n) : U32) <> (128 .|. (c .&. 63) : U32) <> rest
147    case False{} False{} True{}:
148      (224 .|. (c >> 12n) : U32) <> (128 .|. ((c >> 6n) .&. 63) : U32)
149        <> (128 .|. (c .&. 63) : U32) <> rest
150    case _ _ _:
151      (240 .|. (c >> 18n) : U32) <> (128 .|. ((c >> 12n) .&. 63) : U32)
152        <> (128 .|. ((c >> 6n) .&. 63) : U32) <> (128 .|. (c .&. 63) : U32) <> rest
153
154def Utf8.put(+c: U32, rest: List<&2, U32>) -> List<&2, U32>:
155  Utf8.put.go(c, rest, U32.is_lt(c, 128), U32.is_lt(c, 2048), U32.is_lt(c, 65536))
156
157def Utf8.encode(s: String) -> List<&2, U32>:
158  match s:
159    case SNil{}:
160      Nil{}
161    case SCon{Chr{c}, t}:
162      Utf8.put(c, Utf8.encode(t))
163
164def Utf8.size.of(+c: U32, a: Bool, b: Bool, d: Bool) -> U32:
165  match a b d:
166    case True{} _ _:
167      1
168    case False{} True{} _:
169      2
170    case False{} False{} True{}:
171      3
172    case _ _ _:
173      4
174
175# The UTF-8 length of a string, in bytes.
176def Utf8.size(s: String) -> U32:
177  match s:
178    case SNil{}:
179      0
180    case SCon{Chr{+c}, t}:
181      (Utf8.size.of(c, U32.is_lt(c, 128), U32.is_lt(c, 2048), U32.is_lt(c, 65536))
182        + Utf8.size(t) : U32)