1	# Unicode 17.0 general category, NFC/NFD, full case folding, and grapheme clusters. Source: https://github.com/paymog/bend-kit/tree/main/unicode
2	import Base
3	import ./tables.bend as T
4	
5	# Text is a String of code points. Decode octets with the encoding package first.
6	
7	# General category as its two-letter UCD alias: "Lu", "Nd", "Zs", ... Unassigned is "Cn".
8	def category(c: Char) -> String:
9	  Chr{+u} = c
10	  T.gc(u)
11	
12	# Canonical combining class: 0 for a starter.
13	def ccc(c: Char) -> U32:
14	  Chr{+u} = c
15	  T.ccc(u)
16	
17	# Hangul syllables decompose and compose by formula (Unicode §3.12).
18	def hangul.dec(+s: U32) -> String:
19	  +t = (s % 28 : U32)
20	  tail = Bool.pick(String, U32.is_eq(t, 0), SNil{}, SCon{Chr{(4519 + t : U32)}, SNil{}})
21	  SCon{Chr{(4352 + (s / 588) : U32)}, SCon{Chr{(4449 + ((s % 588) / 28) : U32)}, tail}}
22	
23	def decomp.if(hangul: Bool, +s: U32, +c: U32) -> String:
24	  match hangul:
25	    case True{}:
26	      hangul.dec(s)
27	    case False{}:
28	      T.dec(c)
29	
30	# The full canonical decomposition of c; c itself when it has none.
31	def decomp(+c: U32) -> String:
32	  +s = (c - 44032 : U32)
33	  decomp.if(U32.is_lt(s, 11172), s, c)
34	
35	# Does the reversed output start with a mark of class above k?
36	def order.gt(acc: String, +k: U32) -> Bool:
37	  match acc:
38	    case SNil{}:
39	      False{}
40	    case SCon{Chr{+h}, t}:
41	      U32.is_lt(k, T.ccc(h))
42	
43	# Canonical ordering: c, of class k > 0, goes past each mark of a higher class. go says whether acc's head is one.
44	def order.put(acc: String, go: Bool, +c: U32, +k: U32) -> String:
45	  match acc:
46	    case SNil{}:
47	      SCon{Chr{c}, SNil{}}
48	    case SCon{Chr{+h}, +t}:
49	      match go:
50	        case True{}:
51	          SCon{Chr{h}, order.put(t, order.gt(t, k), c, k)}
52	        case False{}:
53	          SCon{Chr{c}, SCon{Chr{h}, t}}
54	
55	def nfd.put(starter: Bool, +acc: String, +c: U32, +k: U32) -> String:
56	  match starter:
57	    case True{}:
58	      SCon{Chr{c}, acc}
59	    case False{}:
60	      order.put(acc, order.gt(acc, k), c, k)
61	
62	def nfd.push(d: String, acc: String) -> String:
63	  match d:
64	    case SNil{}:
65	      acc
66	    case SCon{Chr{+c}, t}:
67	      +k = T.ccc(c)
68	      nfd.push(t, nfd.put(U32.is_eq(k, 0), acc, c, k))
69	
70	def nfd.go(s: String, acc: String) -> String:
71	  match s:
72	    case SNil{}:
73	      String.reverse(acc)
74	    case SCon{Chr{+c}, t}:
75	      nfd.go(t, nfd.push(decomp(c), acc))
76	
77	# Normalization Form D (UAX #15): full canonical decomposition, then canonical order.
78	def nfd(s: String) -> String:
79	  nfd.go(s, SNil{})
80	
81	def hangul.lvt(ok: Bool, +a: U32, +t: U32) -> Maybe<&2, U32>:
82	  match ok:
83	    case True{}:
84	      Some{(a + t : U32)}
85	    case False{}:
86	      T.comp((4519 + t : U32), a)
87	
88	def hangul.lv(ok: Bool, +a: U32, +b: U32) -> Maybe<&2, U32>:
89	  match ok:
90	    case True{}:
91	      Some{(44032 + ((((a - 4352) * 21) + (b - 4449)) * 28) : U32)}
92	    case False{}:
93	      +t = (b - 4519 : U32)
94	      +s = (a - 44032 : U32)
95	      hangul.lvt(Bool.and(Bool.and(U32.is_lt(s, 11172), U32.is_eq((s % 28 : U32), 0)), Bool.and(U32.is_lt(0, t), U32.is_lt(t, 28))), a, t)
96	
97	# The primary composite of a and b, if there is one.
98	def compose(+a: U32, +b: U32) -> Maybe<&2, U32>:
99	  hangul.lv(Bool.and(U32.is_lt((a - 4352 : U32), 19), U32.is_lt((b - 4449 : U32), 21)), a, b)
100	
101	# Composition state: is there a starter, the starter, the class of the last pending mark
102	# (0 when none), the marks after the starter reversed, and the output reversed.
103	type Nfc is Data:
104	  Nfc{has: Bool, st: U32, last: U32, pend: String, out: String}
105	
106	def nfc.flush(has: Bool, +s: U32, pend: String, out: String) -> String:
107	  match has:
108	    case True{}:
109	      String.append(pend, SCon{Chr{s}, out})
110	    case False{}:
111	      String.append(pend, out)
112	
113	def nfc.miss(starter: Bool, +has: Bool, +s: U32, +last: U32, pend: String, out: String, +c: U32, +k: U32) -> Nfc:
114	  match starter:
115	    case True{}:
116	      Nfc{True{}, c, 0, SNil{}, nfc.flush(has, s, pend, out)}
117	    case False{}:
118	      Nfc{has, s, k, SCon{Chr{c}, pend}, out}
119	
120	def nfc.hit(m: Maybe<&2, U32>, +has: Bool, +s: U32, +last: U32, pend: String, out: String, +c: U32, +k: U32) -> Nfc:
121	  match m:
122	    case Some{x}:
123	      Nfc{has, x, last, pend, out}
124	    case None{}:
125	      nfc.miss(U32.is_eq(k, 0), has, s, last, pend, out, c, k)
126	
127	def nfc.try(free: Bool, +has: Bool, +s: U32, +last: U32, pend: String, out: String, +c: U32, +k: U32) -> Nfc:
128	  match free:
129	    case True{}:
130	      nfc.hit(compose(s, c), has, s, last, pend, out, c, k)
131	    case False{}:
132	      nfc.miss(U32.is_eq(k, 0), has, s, last, pend, out, c, k)
133	
134	# c joins the starter unless a mark between them has class 0 or at least c's class.
135	def nfc.step(z: Nfc, +c: U32) -> Nfc:
136	  Nfc{+has, +s, +last, pend, out} = z
137	  +k = T.ccc(c)
138	  nfc.try(Bool.and(has, Bool.not(Bool.and(U32.is_ne(last, 0), U32.is_le(k, last)))), has, s, last, pend, out, c, k)
139	
140	def nfc.go(s: String, z: Nfc) -> String:
141	  match s:
142	    case SNil{}:
143	      Nfc{+has, +st, last, pend, out} = z
144	      String.reverse(nfc.flush(has, st, pend, out))
145	    case SCon{Chr{+c}, t}:
146	      nfc.go(t, nfc.step(z, c))
147	
148	# Normalization Form C (UAX #15): NFD, then canonical composition.
149	def nfc(s: String) -> String:
150	  nfc.go(nfd(s), Nfc{False{}, 0, 0, SNil{}, SNil{}})
151	
152	def fold.go(s: String, acc: String) -> String:
153	  match s:
154	    case SNil{}:
155	      String.reverse(acc)
156	    case SCon{Chr{+c}, t}:
157	      fold.go(t, String.reverse.go(T.fold(c), acc))
158	
159	# Full case folding (CaseFolding.txt, statuses C and F): "Straße" folds to "strasse".
160	def fold(s: String) -> String:
161	  fold.go(s, SNil{})
162	
163	# T.gb packs Grapheme_Cluster_Break in bits 0-3 (Other 0, CR 1, LF 2, Control 3, Extend 4,
164	# ZWJ 5, Regional_Indicator 6, Prepend 7, SpacingMark 8, L 9, V 10, T 11, LV 12, LVT 13),
165	# Extended_Pictographic in bit 4, and InCB in bits 5-6 (Consonant 1, Extend 2, Linker 3).
166	def gcb(+q: U32) -> U32:
167	  (q .&. 15 : U32)
168	
169	def eq(+g: U32, +v: U32) -> Bool:
170	  U32.is_eq(g, v)
171	
172	def hangul.join(+p: U32, +q: U32) -> Bool:
173	  +l = eq(p, 9) && (eq(q, 9) || eq(q, 10) || eq(q, 12) || eq(q, 13))
174	  +v = (eq(p, 12) || eq(p, 10)) && (eq(q, 10) || eq(q, 11))
175	  l || v || ((eq(p, 13) || eq(p, 11)) && eq(q, 11))
176	
177	# UAX #29 GB3-GB13. ri: the regional indicators before q are odd in number.
178	# ep: 1 after ExtPict Extend*, 2 after ExtPict Extend* ZWJ.
179	# incb: 1 after an InCB consonant and extenders, 2 once a linker follows it.
180	def brk(+p: U32, +q: U32, +ri: Bool, +ep: U32, +incb: U32) -> Bool:
181	  +gp = gcb(p)
182	  +gq = gcb(q)
183	  +ctl = (U32.is_le(1, gp) && U32.is_le(gp, 3)) || (U32.is_le(1, gq) && U32.is_le(gq, 3))
184	  +glue = hangul.join(gp, gq) || eq(gq, 4) || eq(gq, 5) || eq(gq, 8) || eq(gp, 7)
185	  +conj = eq(incb, 2) && eq((q >> 5n : U32), 1)
186	  +emoji = eq(ep, 2) && U32.is_ne((q .&. 16 : U32), 0)
187	  +flag = eq(gp, 6) && eq(gq, 6) && ri
188	  Bool.not((eq(gp, 1) && eq(gq, 2)) || (Bool.not(ctl) && (glue || conj || emoji || flag)))
189	
190	def next.ri(+p: U32, +q: U32, +ri: Bool) -> Bool:
191	  eq(gcb(q), 6) && Bool.not(eq(gcb(p), 6) && ri)
192	
193	def next.ep(+q: U32, +ep: U32) -> U32:
194	  +more = Bool.pick(U32, eq(ep, 1) && eq(gcb(q), 5), 2, Bool.pick(U32, eq(ep, 1) && eq(gcb(q), 4), 1, 0))
195	  Bool.pick(U32, U32.is_ne((q .&. 16 : U32), 0), 1, more)
196	
197	def next.incb(+q: U32, +incb: U32) -> U32:
198	  +i = (q >> 5n : U32)
199	  +on = U32.is_ne(incb, 0)
200	  +more = Bool.pick(U32, on && eq(i, 3), 2, Bool.pick(U32, on && eq(i, 2), incb, 0))
201	  Bool.pick(U32, eq(i, 1), 1, more)
202	
203	# Segmentation state: the previous character's T.gb, ri, ep, incb, the current
204	# cluster reversed, and the finished clusters reversed.
205	type Gr is Data:
206	  Gr{p: U32, ri: Bool, ep: U32, incb: U32, cur: String, out: List<&2, String>}
207	
208	def gr.cut(b: Bool, +q: U32, +ri: Bool, +ep: U32, +incb: U32, +c: U32, cur: String, out: List<&2, String>) -> Gr:
209	  match b:
210	    case True{}:
211	      Gr{q, ri, ep, incb, SCon{Chr{c}, SNil{}}, String.reverse(cur) <> out}
212	    case False{}:
213	      Gr{q, ri, ep, incb, SCon{Chr{c}, cur}, out}
214	
215	def gr.step(z: Gr, +c: U32) -> Gr:
216	  Gr{+p, +ri, +ep, +incb, cur, out} = z
217	  +q = T.gb(c)
218	  gr.cut(brk(p, q, ri, ep, incb), q, next.ri(p, q, ri), next.ep(q, ep), next.incb(q, incb), c, cur, out)
219	
220	def gr.go(s: String, z: Gr) -> List<&2, String>:
221	  match s:
222	    case SNil{}:
223	      Gr{p, ri, ep, incb, cur, out} = z
224	      List.reverse(&2, String, String.reverse(cur) <> out)
225	    case SCon{Chr{+c}, t}:
226	      gr.go(t, gr.step(z, c))
227	
228	# Extended grapheme clusters (UAX #29): "e\u{301}x" is ["e\u{301}", "x"].
229	def graphemes(s: String) -> List<&2, String>:
230	  match s:
231	    case SNil{}:
232	      Nil{}
233	    case SCon{Chr{+c}, t}:
234	      +q = T.gb(c)
235	      gr.go(t, Gr{q, next.ri(0, q, False{}), next.ep(q, 0), next.incb(q, 0), SCon{Chr{c}, SNil{}}, Nil{}})