unicode.bend

bend-kit-unicode@0.1.0.0 · 235 lines · 36 definitions

1# Unicode 17.0 general category, NFC/NFD, full case folding, and grapheme clusters. Source: https://github.com/paymog/bend-kit/tree/main/unicode
2import Base
3import bend-kit-unicode@0.1.0.0/tables.bend as T
4
5# Text is a String of code points. Decode octets with the encoding package first.
6
7# General category as its two-letter UCD alias: "Lu", "Nd", "Zs", ... Unassigned is "Cn".
8def category(c: Char) -> String:
9  Chr{+u} = c
10  T.gc(u)
11
12# Canonical combining class: 0 for a starter.
13def ccc(c: Char) -> U32:
14  Chr{+u} = c
15  T.ccc(u)
16
17# Hangul syllables decompose and compose by formula (Unicode §3.12).
18def hangul.dec(+s: U32) -> String:
19  +t = (s % 28 : U32)
20  tail = Bool.pick(String, U32.is_eq(t, 0), SNil{}, SCon{Chr{(4519 + t : U32)}, SNil{}})
21  SCon{Chr{(4352 + (s / 588) : U32)}, SCon{Chr{(4449 + ((s % 588) / 28) : U32)}, tail}}
22
23def decomp.if(hangul: Bool, +s: U32, +c: U32) -> String:
24  match hangul:
25    case True{}:
26      hangul.dec(s)
27    case False{}:
28      T.dec(c)
29
30# The full canonical decomposition of c; c itself when it has none.
31def decomp(+c: U32) -> String:
32  +s = (c - 44032 : U32)
33  decomp.if(U32.is_lt(s, 11172), s, c)
34
35# Does the reversed output start with a mark of class above k?
36def order.gt(acc: String, +k: U32) -> Bool:
37  match acc:
38    case SNil{}:
39      False{}
40    case SCon{Chr{+h}, t}:
41      U32.is_lt(k, T.ccc(h))
42
43# Canonical ordering: c, of class k > 0, goes past each mark of a higher class. go says whether acc's head is one.
44def order.put(acc: String, go: Bool, +c: U32, +k: U32) -> String:
45  match acc:
46    case SNil{}:
47      SCon{Chr{c}, SNil{}}
48    case SCon{Chr{+h}, +t}:
49      match go:
50        case True{}:
51          SCon{Chr{h}, order.put(t, order.gt(t, k), c, k)}
52        case False{}:
53          SCon{Chr{c}, SCon{Chr{h}, t}}
54
55def nfd.put(starter: Bool, +acc: String, +c: U32, +k: U32) -> String:
56  match starter:
57    case True{}:
58      SCon{Chr{c}, acc}
59    case False{}:
60      order.put(acc, order.gt(acc, k), c, k)
61
62def nfd.push(d: String, acc: String) -> String:
63  match d:
64    case SNil{}:
65      acc
66    case SCon{Chr{+c}, t}:
67      +k = T.ccc(c)
68      nfd.push(t, nfd.put(U32.is_eq(k, 0), acc, c, k))
69
70def nfd.go(s: String, acc: String) -> String:
71  match s:
72    case SNil{}:
73      String.reverse(acc)
74    case SCon{Chr{+c}, t}:
75      nfd.go(t, nfd.push(decomp(c), acc))
76
77# Normalization Form D (UAX #15): full canonical decomposition, then canonical order.
78def nfd(s: String) -> String:
79  nfd.go(s, SNil{})
80
81def hangul.lvt(ok: Bool, +a: U32, +t: U32) -> Maybe<&2, U32>:
82  match ok:
83    case True{}:
84      Some{(a + t : U32)}
85    case False{}:
86      T.comp((4519 + t : U32), a)
87
88def hangul.lv(ok: Bool, +a: U32, +b: U32) -> Maybe<&2, U32>:
89  match ok:
90    case True{}:
91      Some{(44032 + ((((a - 4352) * 21) + (b - 4449)) * 28) : U32)}
92    case False{}:
93      +t = (b - 4519 : U32)
94      +s = (a - 44032 : U32)
95      hangul.lvt(Bool.and(Bool.and(U32.is_lt(s, 11172), U32.is_eq((s % 28 : U32), 0)), Bool.and(U32.is_lt(0, t), U32.is_lt(t, 28))), a, t)
96
97# The primary composite of a and b, if there is one.
98def compose(+a: U32, +b: U32) -> Maybe<&2, U32>:
99  hangul.lv(Bool.and(U32.is_lt((a - 4352 : U32), 19), U32.is_lt((b - 4449 : U32), 21)), a, b)
100
101# Composition state: is there a starter, the starter, the class of the last pending mark
102# (0 when none), the marks after the starter reversed, and the output reversed.
103type Nfc is Data:
104  Nfc{has: Bool, st: U32, last: U32, pend: String, out: String}
105
106def nfc.flush(has: Bool, +s: U32, pend: String, out: String) -> String:
107  match has:
108    case True{}:
109      String.append(pend, SCon{Chr{s}, out})
110    case False{}:
111      String.append(pend, out)
112
113def nfc.miss(starter: Bool, +has: Bool, +s: U32, +last: U32, pend: String, out: String, +c: U32, +k: U32) -> Nfc:
114  match starter:
115    case True{}:
116      Nfc{True{}, c, 0, SNil{}, nfc.flush(has, s, pend, out)}
117    case False{}:
118      Nfc{has, s, k, SCon{Chr{c}, pend}, out}
119
120def nfc.hit(m: Maybe<&2, U32>, +has: Bool, +s: U32, +last: U32, pend: String, out: String, +c: U32, +k: U32) -> Nfc:
121  match m:
122    case Some{x}:
123      Nfc{has, x, last, pend, out}
124    case None{}:
125      nfc.miss(U32.is_eq(k, 0), has, s, last, pend, out, c, k)
126
127def nfc.try(free: Bool, +has: Bool, +s: U32, +last: U32, pend: String, out: String, +c: U32, +k: U32) -> Nfc:
128  match free:
129    case True{}:
130      nfc.hit(compose(s, c), has, s, last, pend, out, c, k)
131    case False{}:
132      nfc.miss(U32.is_eq(k, 0), has, s, last, pend, out, c, k)
133
134# c joins the starter unless a mark between them has class 0 or at least c's class.
135def nfc.step(z: Nfc, +c: U32) -> Nfc:
136  Nfc{+has, +s, +last, pend, out} = z
137  +k = T.ccc(c)
138  nfc.try(Bool.and(has, Bool.not(Bool.and(U32.is_ne(last, 0), U32.is_le(k, last)))), has, s, last, pend, out, c, k)
139
140def nfc.go(s: String, z: Nfc) -> String:
141  match s:
142    case SNil{}:
143      Nfc{+has, +st, last, pend, out} = z
144      String.reverse(nfc.flush(has, st, pend, out))
145    case SCon{Chr{+c}, t}:
146      nfc.go(t, nfc.step(z, c))
147
148# Normalization Form C (UAX #15): NFD, then canonical composition.
149def nfc(s: String) -> String:
150  nfc.go(nfd(s), Nfc{False{}, 0, 0, SNil{}, SNil{}})
151
152def fold.go(s: String, acc: String) -> String:
153  match s:
154    case SNil{}:
155      String.reverse(acc)
156    case SCon{Chr{+c}, t}:
157      fold.go(t, String.reverse.go(T.fold(c), acc))
158
159# Full case folding (CaseFolding.txt, statuses C and F): "Straße" folds to "strasse".
160def fold(s: String) -> String:
161  fold.go(s, SNil{})
162
163# T.gb packs Grapheme_Cluster_Break in bits 0-3 (Other 0, CR 1, LF 2, Control 3, Extend 4,
164# ZWJ 5, Regional_Indicator 6, Prepend 7, SpacingMark 8, L 9, V 10, T 11, LV 12, LVT 13),
165# Extended_Pictographic in bit 4, and InCB in bits 5-6 (Consonant 1, Extend 2, Linker 3).
166def gcb(+q: U32) -> U32:
167  (q .&. 15 : U32)
168
169def eq(+g: U32, +v: U32) -> Bool:
170  U32.is_eq(g, v)
171
172def hangul.join(+p: U32, +q: U32) -> Bool:
173  +l = eq(p, 9) && (eq(q, 9) || eq(q, 10) || eq(q, 12) || eq(q, 13))
174  +v = (eq(p, 12) || eq(p, 10)) && (eq(q, 10) || eq(q, 11))
175  l || v || ((eq(p, 13) || eq(p, 11)) && eq(q, 11))
176
177# UAX #29 GB3-GB13. ri: the regional indicators before q are odd in number.
178# ep: 1 after ExtPict Extend*, 2 after ExtPict Extend* ZWJ.
179# incb: 1 after an InCB consonant and extenders, 2 once a linker follows it.
180def brk(+p: U32, +q: U32, +ri: Bool, +ep: U32, +incb: U32) -> Bool:
181  +gp = gcb(p)
182  +gq = gcb(q)
183  +ctl = (U32.is_le(1, gp) && U32.is_le(gp, 3)) || (U32.is_le(1, gq) && U32.is_le(gq, 3))
184  +glue = hangul.join(gp, gq) || eq(gq, 4) || eq(gq, 5) || eq(gq, 8) || eq(gp, 7)
185  +conj = eq(incb, 2) && eq((q >> 5n : U32), 1)
186  +emoji = eq(ep, 2) && U32.is_ne((q .&. 16 : U32), 0)
187  +flag = eq(gp, 6) && eq(gq, 6) && ri
188  Bool.not((eq(gp, 1) && eq(gq, 2)) || (Bool.not(ctl) && (glue || conj || emoji || flag)))
189
190def next.ri(+p: U32, +q: U32, +ri: Bool) -> Bool:
191  eq(gcb(q), 6) && Bool.not(eq(gcb(p), 6) && ri)
192
193def next.ep(+q: U32, +ep: U32) -> U32:
194  +more = Bool.pick(U32, eq(ep, 1) && eq(gcb(q), 5), 2, Bool.pick(U32, eq(ep, 1) && eq(gcb(q), 4), 1, 0))
195  Bool.pick(U32, U32.is_ne((q .&. 16 : U32), 0), 1, more)
196
197def next.incb(+q: U32, +incb: U32) -> U32:
198  +i = (q >> 5n : U32)
199  +on = U32.is_ne(incb, 0)
200  +more = Bool.pick(U32, on && eq(i, 3), 2, Bool.pick(U32, on && eq(i, 2), incb, 0))
201  Bool.pick(U32, eq(i, 1), 1, more)
202
203# Segmentation state: the previous character's T.gb, ri, ep, incb, the current
204# cluster reversed, and the finished clusters reversed.
205type Gr is Data:
206  Gr{p: U32, ri: Bool, ep: U32, incb: U32, cur: String, out: List<&2, String>}
207
208def gr.cut(b: Bool, +q: U32, +ri: Bool, +ep: U32, +incb: U32, +c: U32, cur: String, out: List<&2, String>) -> Gr:
209  match b:
210    case True{}:
211      Gr{q, ri, ep, incb, SCon{Chr{c}, SNil{}}, String.reverse(cur) <> out}
212    case False{}:
213      Gr{q, ri, ep, incb, SCon{Chr{c}, cur}, out}
214
215def gr.step(z: Gr, +c: U32) -> Gr:
216  Gr{+p, +ri, +ep, +incb, cur, out} = z
217  +q = T.gb(c)
218  gr.cut(brk(p, q, ri, ep, incb), q, next.ri(p, q, ri), next.ep(q, ep), next.incb(q, incb), c, cur, out)
219
220def gr.go(s: String, z: Gr) -> List<&2, String>:
221  match s:
222    case SNil{}:
223      Gr{p, ri, ep, incb, cur, out} = z
224      List.reverse(&2, String, String.reverse(cur) <> out)
225    case SCon{Chr{+c}, t}:
226      gr.go(t, gr.step(z, c))
227
228# Extended grapheme clusters (UAX #29): "e\u{301}x" is ["e\u{301}", "x"].
229def graphemes(s: String) -> List<&2, String>:
230  match s:
231    case SNil{}:
232      Nil{}
233    case SCon{Chr{+c}, t}:
234      +q = T.gb(c)
235      gr.go(t, Gr{q, next.ri(0, q, False{}), next.ep(q, 0), next.incb(q, 0), SCon{Chr{c}, SNil{}}, Nil{}})