1	# mlbend-datasets: the datasets of scikit-learn, fetched from the hub.
2	#
3	#   import mlbend-datasets@0.1.3.0/main.bend as Datasets
4	#
5	# The features of a dataset and its labels beside them: the rows of the
6	# features, and one label per row.
7	#
8	# The labels are numbers. A classification label is the index of the
9	# class, a regression label is the value; which one it is, the model
10	# decides.
11	#
12	# Every dataset here comes from HuggingFace, over the network, at run
13	# time: the effect is at the edge and the data that comes back is a
14	# value, so an estimator over it stays pure and a run is reproducible.
15	#
16	# Two shapes of file. The iris dataset is the one scikit-learn calls
17	# `load_iris`: 150 rows, four features, three classes, and no header,
18	#
19	#   5.1,3.5,1.4,0.2,Iris-setosa
20	#
21	# The others are CSV with a header, and the last column is the label:
22	#
23	#   fixed_acidity,volatile_acidity,...,alcohol,quality
24	#   7.4,0.7,0.0,1.9,0.076,11.0,34.0,0.9978,3.51,0.56,9.4,5,0
25	
26	import Base
27	import mlbend-huggingface@0.1.1.0/main.bend as Hf
28	
29	# A dataset: the rows of the features and the labels beside them.
30	type Table is Data:
31	  Table{rows: List<&2, List<&2, F32>>, labels: List<&2, F32>}
32	
33	# One row of the file: its features and its label.
34	type Row is Data:
35	  Row{features: List<&2, F32>, label: F32}
36	
37	# The rows read so far.
38	type Rows is Data:
39	  Rows{rows: List<&2, Row>, labels: List<&2, F32>}
40	
41	# The class of a row, by name: 0, 1 or 2 for the three iris species.
42	def add_features(acc: List<&2, List<&2, F32>>, r: Row) -> List<&2, List<&2, F32>>:
43	  match r:
44	    case Row{features, label}:
45	      List.append(&2, List<&2, F32>, acc, features <> Nil{})
46	
47	# The dataset a file of rows makes.
48	def add_row_of(acc: Rows, r: Row) -> Rows:
49	  match acc:
50	    case Rows{rows, labels}:
51	      match r:
52	        case Row{features, +label}:
53	          Rows{List.append(&2, Row, rows, r <> Nil{}),
54	            List.append(&2, F32, labels, label <> Nil{})}
55	
56	# The rows of a file.
57	def number_or_zero(m: Maybe<&2, F32>) -> F32:
58	  match m:
59	    case Some{v}:
60	      v
61	    case None{}:
62	      0.0
63	
64	# The text of a field of a row.
65	def cell(text: String) -> F32:
66	  match text:
67	    case "True":
68	      1.0
69	    case "False":
70	      0.0
71	    case _:
72	      number_or_zero(F32.read(text))
73	
74	# A number that may be there, or zero when it is not.
75	def cells_of(+xs: List<&2, String>) -> List<&2, F32>:
76	  match xs:
77	    case Nil{}:
78	      Nil{}
79	    case Con{h, t}:
80	      match t:
81	        case Nil{}:
82	          Nil{}
83	        case Con{h2, t2}:
84	          List.append(&2, F32, cell(h) <> Nil{}, cells_of(t))
85	
86	# The features of a row, from the first field to the one before the
87	# label.
88	def features_of(+xs: List<&2, String>) -> List<&2, F32>:
89	  cells_of(xs)
90	
91	# The features of a row, from the head: the last field is the label, so
92	# the walk stops one field short.
93	def class_of(name: String) -> F32:
94	  match name:
95	    case "Iris-setosa":
96	      0.0
97	    case "Iris-versicolor":
98	      1.0
99	    case "Iris-virginica":
100	      2.0
101	    case _:
102	      0.0
103	
104	# A field of a row, as a number: a number is itself, True is 1, False is
105	# 0, and anything else is 0.
106	def class_or_number(text: String) -> F32:
107	  match text:
108	    case "Iris-setosa":
109	      class_of(text)
110	    case "Iris-versicolor":
111	      class_of(text)
112	    case "Iris-virginica":
113	      class_of(text)
114	    case _:
115	      number_or_zero(F32.read(text))
116	
117	# One row of the file, from its fields.
118	def label_at(m: Maybe<&2, String>) -> F32:
119	  match m:
120	    case Some{s}:
121	      class_or_number(s)
122	    case None{}:
123	      0.0
124	
125	# The label of a row: a class name is its class, a number is itself.
126	def text_at(+xs: List<&2, String>, n: Nat) -> Maybe<&2, String>:
127	  List.get(&2, String, xs, n)
128	
129	# The features of a row: every field but the last.
130	def label_of(+xs: List<&2, String>) -> F32:
131	  label_at(text_at(xs, Nat.sub(List.length(&2, String, xs), 1n)))
132	
133	# The label of a row, when the field is there.
134	def row_of_parts(+fs: List<&2, F32>, l: F32) -> Row:
135	  Row{fs, l}
136	
137	# One row of the file.
138	def row_of(line: String) -> Row:
139	  +xs = String.split(line, Chr{44})
140	  row_of_parts(features_of(xs), label_of(xs))
141	
142	# The rows of a file, folded together.
143	def add_row(acc: Rows, line: String) -> Rows:
144	  add_row_of(acc, row_of(line))
145	
146	# The rows of a file, and the row one line makes.
147	def feature_rows(rs: List<&2, Row>) -> List<&2, List<&2, F32>>:
148	  List.foldl(&2, Row, List<&2, List<&2, F32>>, add_features, rs, Nil{})
149	
150	# One row's features, appended to the rows so far.
151	def data_of(acc: Rows) -> Table:
152	  match acc:
153	    case Rows{rows, labels}:
154	      Table{feature_rows(rows), labels}
155	
156	# The rows of the features, one after the other.
157	def not_empty(line: String) -> Bool:
158	  Bool.not(String.is_empty(line))
159	
160	# The iris dataset, fetched from the hub.
161	def drop_empty(lines: List<&2, String>) -> List<&2, String>:
162	  List.filter(String, not_empty, lines)
163	
164	# A line that holds something.
165	def rows_of(lines: List<&2, String>) -> Rows:
166	  List.foldl(&2, String, Rows, add_row, lines, Rows{Nil{}, Nil{}})
167	
168	# The dataset the rows make.
169	def build(text: String) -> Table:
170	  data_of(rows_of(drop_empty(String.lines(text))))
171	
172	# The dataset a file of CSV makes: the first line is the header and the
173	# last column of every other line is the label.
174	def build_csv_rows(lines: List<&2, String>) -> Table:
175	  data_of(rows_of(drop_empty(lines)))
176	
177	# The lines of a file without its first.
178	def drop_first(lines: List<&2, String>) -> List<&2, String>:
179	  List.drop(&2, String, lines, 1n)
180	
181	# The lines of a file without its empty ones: a file ends with a
182	# newline, and the line after it is empty.
183	def build_csv_lines(lines: List<&2, String>) -> Table:
184	  build_csv_rows(drop_first(lines))
185	
186	# The dataset the rows of a CSV make.
187	def build_csv(text: String) -> Table:
188	  build_csv_lines(String.lines(text))
189	
190	# The dataset the lines of a CSV make, without the header.
191	def csv_text(+text: String, header: Bool) -> Table:
192	  Bool.pick(Table, header, build_csv(text), build(text))
193	
194	def csv_of_text(t: Result<&1, &1, Hf.Err, String>, header: Bool) ->
195	  Result<&1, &1, Hf.Err, Table>:
196	  match t:
197	    case Fail{e}:
198	      Fail{e}
199	    case Done{text}:
200	      Done{csv_text(text, header)}
201	
202	# The dataset the text of a file makes.
203	def csv_of(repo: String, rev: String, path: String, header: Bool) ->
204	  IO(Result<&1, &1, Hf.Err, Table>):
205	  do IO<Result<&1, &1, Hf.Err, Table>>:
206	    t : Result<&1, &1, Hf.Err, String> <- Hf.file_text(repo, rev, path)
207	    return csv_of_text(t, header)
208	
209	# The dataset a file of text makes.
210	def cancer() -> IO(Result<&1, &1, Hf.Err, Table>):
211	  csv_of("mstz/breast", "main", "cancer/train.csv", True{})
212	
213	# The heart failure dataset: 299 rows, twelve features, and the label.
214	def cell_of(m: Maybe<&2, String>, rest: List<&2, F32>) -> List<&2, F32>:
215	  match m:
216	    case Some{s}:
217	      List.append(&2, F32, rest, cell(s) <> Nil{})
218	    case None{}:
219	      rest
220	
221	# The label of a row: the last field.
222	def heart() -> IO(Result<&1, &1, Hf.Err, Table>):
223	  csv_of("mstz/heart_failure", "main", "death/train.csv", True{})
224	
225	# A file of a dataset's repository, and the dataset it makes.
226	def iris() -> IO(Result<&1, &1, Hf.Err, Table>):
227	  csv_of("mstz/iris", "main", "iris.data", False{})
228	
229	# The wine dataset: 178 rows, thirteen features, and the quality.
230	def wine() -> IO(Result<&1, &1, Hf.Err, Table>):
231	  csv_of("mstz/wine", "main", "wine/train.csv", True{})
232	
233	# The breast cancer dataset: 569 rows, nine features, and the label.