main.bend
mlbend-datasets@0.1.3.0 · 233 lines · 0 definitions
1# mlbend-datasets: the datasets of scikit-learn, fetched from the hub.
2#
3# import mlbend-datasets@0.1.3.0/main.bend as Datasets
4#
5# The features of a dataset and its labels beside them: the rows of the
6# features, and one label per row.
7#
8# The labels are numbers. A classification label is the index of the
9# class, a regression label is the value; which one it is, the model
10# decides.
11#
12# Every dataset here comes from HuggingFace, over the network, at run
13# time: the effect is at the edge and the data that comes back is a
14# value, so an estimator over it stays pure and a run is reproducible.
15#
16# Two shapes of file. The iris dataset is the one scikit-learn calls
17# `load_iris`: 150 rows, four features, three classes, and no header,
18#
19# 5.1,3.5,1.4,0.2,Iris-setosa
20#
21# The others are CSV with a header, and the last column is the label:
22#
23# fixed_acidity,volatile_acidity,...,alcohol,quality
24# 7.4,0.7,0.0,1.9,0.076,11.0,34.0,0.9978,3.51,0.56,9.4,5,0
25
26import Base
27import mlbend-huggingface@0.1.1.0/main.bend as Hf
28
29# A dataset: the rows of the features and the labels beside them.
30type Table is Data:
31 Table{rows: List<&2, List<&2, F32>>, labels: List<&2, F32>}
32
33# One row of the file: its features and its label.
34type Row is Data:
35 Row{features: List<&2, F32>, label: F32}
36
37# The rows read so far.
38type Rows is Data:
39 Rows{rows: List<&2, Row>, labels: List<&2, F32>}
40
41# The class of a row, by name: 0, 1 or 2 for the three iris species.
42def add_features(acc: List<&2, List<&2, F32>>, r: Row) -> List<&2, List<&2, F32>>:
43 match r:
44 case Row{features, label}:
45 List.append(&2, List<&2, F32>, acc, features <> Nil{})
46
47# The dataset a file of rows makes.
48def add_row_of(acc: Rows, r: Row) -> Rows:
49 match acc:
50 case Rows{rows, labels}:
51 match r:
52 case Row{features, +label}:
53 Rows{List.append(&2, Row, rows, r <> Nil{}),
54 List.append(&2, F32, labels, label <> Nil{})}
55
56# The rows of a file.
57def number_or_zero(m: Maybe<&2, F32>) -> F32:
58 match m:
59 case Some{v}:
60 v
61 case None{}:
62 0.0
63
64# The text of a field of a row.
65def cell(text: String) -> F32:
66 match text:
67 case "True":
68 1.0
69 case "False":
70 0.0
71 case _:
72 number_or_zero(F32.read(text))
73
74# A number that may be there, or zero when it is not.
75def cells_of(+xs: List<&2, String>) -> List<&2, F32>:
76 match xs:
77 case Nil{}:
78 Nil{}
79 case Con{h, t}:
80 match t:
81 case Nil{}:
82 Nil{}
83 case Con{h2, t2}:
84 List.append(&2, F32, cell(h) <> Nil{}, cells_of(t))
85
86# The features of a row, from the first field to the one before the
87# label.
88def features_of(+xs: List<&2, String>) -> List<&2, F32>:
89 cells_of(xs)
90
91# The features of a row, from the head: the last field is the label, so
92# the walk stops one field short.
93def class_of(name: String) -> F32:
94 match name:
95 case "Iris-setosa":
96 0.0
97 case "Iris-versicolor":
98 1.0
99 case "Iris-virginica":
100 2.0
101 case _:
102 0.0
103
104# A field of a row, as a number: a number is itself, True is 1, False is
105# 0, and anything else is 0.
106def class_or_number(text: String) -> F32:
107 match text:
108 case "Iris-setosa":
109 class_of(text)
110 case "Iris-versicolor":
111 class_of(text)
112 case "Iris-virginica":
113 class_of(text)
114 case _:
115 number_or_zero(F32.read(text))
116
117# One row of the file, from its fields.
118def label_at(m: Maybe<&2, String>) -> F32:
119 match m:
120 case Some{s}:
121 class_or_number(s)
122 case None{}:
123 0.0
124
125# The label of a row: a class name is its class, a number is itself.
126def text_at(+xs: List<&2, String>, n: Nat) -> Maybe<&2, String>:
127 List.get(&2, String, xs, n)
128
129# The features of a row: every field but the last.
130def label_of(+xs: List<&2, String>) -> F32:
131 label_at(text_at(xs, Nat.sub(List.length(&2, String, xs), 1n)))
132
133# The label of a row, when the field is there.
134def row_of_parts(+fs: List<&2, F32>, l: F32) -> Row:
135 Row{fs, l}
136
137# One row of the file.
138def row_of(line: String) -> Row:
139 +xs = String.split(line, Chr{44})
140 row_of_parts(features_of(xs), label_of(xs))
141
142# The rows of a file, folded together.
143def add_row(acc: Rows, line: String) -> Rows:
144 add_row_of(acc, row_of(line))
145
146# The rows of a file, and the row one line makes.
147def feature_rows(rs: List<&2, Row>) -> List<&2, List<&2, F32>>:
148 List.foldl(&2, Row, List<&2, List<&2, F32>>, add_features, rs, Nil{})
149
150# One row's features, appended to the rows so far.
151def data_of(acc: Rows) -> Table:
152 match acc:
153 case Rows{rows, labels}:
154 Table{feature_rows(rows), labels}
155
156# The rows of the features, one after the other.
157def not_empty(line: String) -> Bool:
158 Bool.not(String.is_empty(line))
159
160# The iris dataset, fetched from the hub.
161def drop_empty(lines: List<&2, String>) -> List<&2, String>:
162 List.filter(String, not_empty, lines)
163
164# A line that holds something.
165def rows_of(lines: List<&2, String>) -> Rows:
166 List.foldl(&2, String, Rows, add_row, lines, Rows{Nil{}, Nil{}})
167
168# The dataset the rows make.
169def build(text: String) -> Table:
170 data_of(rows_of(drop_empty(String.lines(text))))
171
172# The dataset a file of CSV makes: the first line is the header and the
173# last column of every other line is the label.
174def build_csv_rows(lines: List<&2, String>) -> Table:
175 data_of(rows_of(drop_empty(lines)))
176
177# The lines of a file without its first.
178def drop_first(lines: List<&2, String>) -> List<&2, String>:
179 List.drop(&2, String, lines, 1n)
180
181# The lines of a file without its empty ones: a file ends with a
182# newline, and the line after it is empty.
183def build_csv_lines(lines: List<&2, String>) -> Table:
184 build_csv_rows(drop_first(lines))
185
186# The dataset the rows of a CSV make.
187def build_csv(text: String) -> Table:
188 build_csv_lines(String.lines(text))
189
190# The dataset the lines of a CSV make, without the header.
191def csv_text(+text: String, header: Bool) -> Table:
192 Bool.pick(Table, header, build_csv(text), build(text))
193
194def csv_of_text(t: Result<&1, &1, Hf.Err, String>, header: Bool) ->
195 Result<&1, &1, Hf.Err, Table>:
196 match t:
197 case Fail{e}:
198 Fail{e}
199 case Done{text}:
200 Done{csv_text(text, header)}
201
202# The dataset the text of a file makes.
203def csv_of(repo: String, rev: String, path: String, header: Bool) ->
204 IO(Result<&1, &1, Hf.Err, Table>):
205 do IO<Result<&1, &1, Hf.Err, Table>>:
206 t : Result<&1, &1, Hf.Err, String> <- Hf.file_text(repo, rev, path)
207 return csv_of_text(t, header)
208
209# The dataset a file of text makes.
210def cancer() -> IO(Result<&1, &1, Hf.Err, Table>):
211 csv_of("mstz/breast", "main", "cancer/train.csv", True{})
212
213# The heart failure dataset: 299 rows, twelve features, and the label.
214def cell_of(m: Maybe<&2, String>, rest: List<&2, F32>) -> List<&2, F32>:
215 match m:
216 case Some{s}:
217 List.append(&2, F32, rest, cell(s) <> Nil{})
218 case None{}:
219 rest
220
221# The label of a row: the last field.
222def heart() -> IO(Result<&1, &1, Hf.Err, Table>):
223 csv_of("mstz/heart_failure", "main", "death/train.csv", True{})
224
225# A file of a dataset's repository, and the dataset it makes.
226def iris() -> IO(Result<&1, &1, Hf.Err, Table>):
227 csv_of("mstz/iris", "main", "iris.data", False{})
228
229# The wine dataset: 178 rows, thirteen features, and the quality.
230def wine() -> IO(Result<&1, &1, Hf.Err, Table>):
231 csv_of("mstz/wine", "main", "wine/train.csv", True{})
232
233# The breast cancer dataset: 569 rows, nine features, and the label.