main.bend
mlbend-datasets@0.1.5.0 · 309 lines · 0 definitions
1# mlbend-datasets: the datasets of scikit-learn, fetched from the hub.
2#
3# import mlbend-datasets@0.1.5.0/main.bend as Datasets
4#
5# The features of a dataset and its labels beside them: the rows of the
6# features, and one label per row.
7#
8# The labels are numbers. A classification label is the index of the
9# class, a regression label is the value; which one it is, the model
10# decides.
11#
12# Every dataset here comes from HuggingFace, over the network, at run
13# time: the effect is at the edge and the data that comes back is a
14# value, so an estimator over it stays pure and a run is reproducible.
15#
16# Two shapes of file. The iris dataset is the one scikit-learn calls
17# `load_iris`: 150 rows, four features, three classes, and no header,
18#
19# 5.1,3.5,1.4,0.2,Iris-setosa
20#
21# The others are CSV with a header, and the label is one of the columns:
22#
23# fixed_acidity,volatile_acidity,...,alcohol,quality,is_red
24# 7.4,0.7,0.0,1.9,0.076,11.0,34.0,0.9978,3.51,0.56,9.4,5,0
25#
26# The loader is told how many columns to leave out at the end of a row:
27# the label is the column before them.
28
29import Base
30import mlbend-huggingface@0.1.1.0/main.bend as Hf
31
32# A dataset: the rows of the features and the labels beside them.
33
34type Table is Data:
35 Table{rows: List<&2, List<&2, F32>>, labels: List<&2, F32>}
36
37# One row of the file: its features and its label.
38type Row is Data:
39 Row{features: List<&2, F32>, label: F32}
40
41# The rows read so far.
42type Rows is Data:
43 Rows{rows: List<&2, Row>, labels: List<&2, F32>}
44
45# One row's features, appended to the rows so far.
46def add_features(acc: List<&2, List<&2, F32>>, r: Row) -> List<&2, List<&2, F32>>:
47 match r:
48 case Row{features, label}:
49 List.append(&2, List<&2, F32>, acc, Con{features, Nil{}})
50
51# The rows of a file, and the row one line makes.
52def add_row_of(acc: Rows, +r: Row) -> Rows:
53 match acc:
54 case Rows{rows, labels}:
55 match r:
56 case Row{features, +label}:
57 Rows{List.append(&2, Row, rows, r <> Nil{}),
58 List.append(&2, F32, labels, label <> Nil{})}
59
60# The rows of the features, one after the other.
61def feature_rows(rs: List<&2, Row>) -> List<&2, List<&2, F32>>:
62 List.foldl(&2, Row, List<&2, List<&2, F32>>, add_features, rs, Nil{})
63
64# The dataset the rows make.
65def data_of(acc: Rows) -> Table:
66 match acc:
67 case Rows{rows, labels}:
68 Table{feature_rows(rows), labels}
69
70# A line that holds something.
71def not_empty(line: String) -> Bool:
72 Bool.not(String.is_empty(line))
73
74# The lines of a file without its empty ones: a file ends with a
75# newline, and the line after it is empty.
76def drop_empty(lines: List<&2, String>) -> List<&2, String>:
77 List.filter(String, not_empty, lines)
78
79# A number that may be there, or zero when it is not.
80def number_or_zero(m: Maybe<&2, F32>) -> F32:
81 match m:
82 case Some{v}:
83 v
84 case None{}:
85 0.0
86
87# A field of a row, as a number: a number is itself, True is 1, False is
88# 0, and anything else is 0.
89def cell(+text: String) -> F32:
90 match text:
91 case "True":
92 1.0
93 case "False":
94 0.0
95 case _:
96 number_or_zero(F32.read(text))
97
98# The cells of a row, as numbers.
99def cells_of(+xs: List<&2, String>) -> List<&2, F32>:
100 match xs:
101 case Nil{}:
102 Nil{}
103 case Con{h, t}:
104 List.append(&2, F32, Nil{}, Con{cell(h), cells_of(t)})
105
106# The place of the label in a row: the last column, less the columns
107# that are left out at the end.
108def label_index(+xs: List<&2, String>, drop: Nat) -> Nat:
109 Nat.sub(List.length(&2, String, xs), Nat.add(1n, drop))
110
111# The fields before the label.
112def fields_before(+xs: List<&2, String>, drop: Nat) -> List<&2, String>:
113 List.take(&2, String, xs, label_index(xs, drop))
114
115# The features of a row: the fields before the label, as numbers.
116def features_of(+xs: List<&2, String>, drop: Nat) -> List<&2, F32>:
117 cells_of(fields_before(xs, drop))
118
119# The class of a row, by name: 0, 1 or 2 for the three iris species.
120def class_of(name: String) -> F32:
121 match name:
122 case "Iris-setosa":
123 0.0
124 case "Iris-versicolor":
125 1.0
126 case "Iris-virginica":
127 2.0
128 case _:
129 0.0
130
131# The label of a row: a class name is its class, a number is itself.
132def class_or_number(+text: String) -> F32:
133 match text:
134 case "Iris-setosa":
135 class_of(text)
136 case "Iris-versicolor":
137 class_of(text)
138 case "Iris-virginica":
139 class_of(text)
140 case _:
141 number_or_zero(F32.read(text))
142
143# The label of a row, when the field is there.
144def label_at(m: Maybe<&2, String>) -> F32:
145 match m:
146 case Some{s}:
147 class_or_number(s)
148 case None{}:
149 0.0
150
151# The text of a field of a row.
152def text_at(+xs: List<&2, String>, n: Nat) -> Maybe<&2, String>:
153 List.get(&2, String, xs, n)
154
155# The label of a row: the column at its place.
156def label_of(+xs: List<&2, String>, drop: Nat) -> F32:
157 label_at(text_at(xs, label_index(xs, drop)))
158
159# One row of the file, from its fields.
160def row_of_parts(+fs: List<&2, F32>, l: F32) -> Row:
161 Row{fs, l}
162
163# One row of the file.
164def row_of(+line: String, +drop: Nat) -> Row:
165 +xs = String.split(line, Chr{44})
166 row_of_parts(features_of(xs, drop), label_of(xs, drop))
167
168# The rows of a file, one line at a time.
169def rows_go(lines: List<&2, String>, +drop: Nat, +acc: Rows) -> Rows:
170 match lines:
171 case Nil{}:
172 acc
173 case Con{h, t}:
174 rows_go(t, drop, add_row_of(acc, row_of(h, drop)))
175
176# The rows of a file.
177def rows_of(lines: List<&2, String>, drop: Nat) -> Rows:
178 rows_go(lines, drop, Rows{Nil{}, Nil{}})
179
180# The dataset a file of rows makes.
181def build(text: String, drop: Nat) -> Table:
182 data_of(rows_of(drop_empty(String.lines(text)), drop))
183
184# The dataset the rows of a CSV make.
185def build_csv_rows(+lines: List<&2, String>, drop: Nat) -> Table:
186 data_of(rows_of(drop_empty(lines), drop))
187
188# The lines of a file without its first.
189def drop_first(+lines: List<&2, String>) -> List<&2, String>:
190 List.drop(&2, String, lines, 1n)
191
192# The dataset a file of CSV makes: the first line is the header and the
193# label is the column before the ones left out at the end.
194def build_csv(text: String, drop: Nat) -> Table:
195 build_csv_rows(drop_first(String.lines(text)), drop)
196
197# The dataset the text of a file makes.
198def csv_text(+text: String, header: Bool, +drop: Nat) -> Table:
199 Bool.pick(Table, header, build_csv(text, drop), build(text, drop))
200
201# The dataset a file of text makes.
202def csv_of_text(t: Result<&1, &1, Hf.Err, String>, header: Bool, drop: Nat) ->
203 Result<&1, &1, Hf.Err, Table>:
204 match t:
205 case Fail{e}:
206 Fail{e}
207 case Done{text}:
208 Done{csv_text(text, header, drop)}
209
210# ---- the local cache: MLBEND_DATA, a directory laid out like the hub ----
211#
212# The runner fills <dir>/<repo>/<path> with the exact bytes it hashed. When
213# the variable is set and the file is there, the program reads it and does
214# not touch the network; otherwise it falls back to the hub client.
215
216def cached_path(dir: String, repo: String, path: String) -> String:
217 String.append(dir, String.append("/", String.append(repo, String.append("/", path))))
218
219def read_result(result: Result<&1, &1, U32 & String, String>) -> IO(Maybe<&1, String>):
220 match result:
221 case Fail{e}:
222 IO.pure(Maybe<&1, String>, None{})
223 case Done{text}:
224 IO.pure(Maybe<&1, String>, Some{text})
225
226def read_done(pair: File & Result<&1, &1, U32 & String, String>) -> IO(Maybe<&1, String>):
227 (file, result) = pair
228 do IO<Maybe<&1, String>>:
229 File.close(file)
230 read_result(result)
231
232def sized_done(pair: File & Result<&1, &1, U32 & String, U32>) -> IO(Maybe<&1, String>):
233 (file, result) = pair
234 match result:
235 case Fail{e}:
236 do IO<Maybe<&1, String>>:
237 File.close(file)
238 return None{}
239 case Done{size}:
240 do IO<Maybe<&1, String>>:
241 pair : File & Result<&1, &1, U32 & String, String> <- File.read(file, size)
242 m : Maybe<&1, String> <- read_done(pair)
243 return m
244
245def opened(r: Result<&1, &1, U32 & String, File>) -> IO(Maybe<&1, String>):
246 match r:
247 case Fail{e}:
248 IO.pure(Maybe<&1, String>, None{})
249 case Done{file}:
250 do IO<Maybe<&1, String>>:
251 pair : File & Result<&1, &1, U32 & String, U32> <- File.size(file)
252 m : Maybe<&1, String> <- sized_done(pair)
253 return m
254
255# The text of a cached file, or None when it is not there.
256def read_cached(path: String) -> IO(Maybe<&1, String>):
257 do IO<Maybe<&1, String>>:
258 r : Result<&1, &1, U32 & String, File> <- File.open(path, "r")
259 opened(r)
260
261# A table from the cache when it is there, otherwise from the hub.
262def fetch_csv(repo: String, rev: String, path: String, header: Bool, drop: Nat) ->
263 IO(Result<&1, &1, Hf.Err, Table>):
264 do IO<Result<&1, &1, Hf.Err, Table>>:
265 t : Result<&1, &1, Hf.Err, String> <- Hf.file_text(repo, rev, path)
266 return csv_of_text(t, header, drop)
267
268def cached_or_fetch(m: Maybe<&1, String>, repo: String, rev: String, path: String,
269 header: Bool, drop: Nat) -> IO(Result<&1, &1, Hf.Err, Table>):
270 match m:
271 case Some{text}:
272 IO.pure(Result<&1, &1, Hf.Err, Table>, Done{csv_text(text, header, drop)})
273 case None{}:
274 fetch_csv(repo, rev, path, header, drop)
275
276def from_env(e: Result<&1, &1, U32 & String, String>, +repo: String, rev: String, +path: String,
277 header: Bool, drop: Nat) -> IO(Result<&1, &1, Hf.Err, Table>):
278 match e:
279 case Fail{code}:
280 fetch_csv(repo, rev, path, header, drop)
281 case Done{dir}:
282 do IO<Result<&1, &1, Hf.Err, Table>>:
283 m : Maybe<&1, String> <- read_cached(cached_path(dir, repo, path))
284 t : Result<&1, &1, Hf.Err, Table> <- cached_or_fetch(m, repo, rev, path, header, drop)
285 return t
286
287# A file of a dataset's repository, and the dataset it makes.
288def csv_of(repo: String, rev: String, path: String, header: Bool, drop: Nat) ->
289 IO(Result<&1, &1, Hf.Err, Table>):
290 do IO<Result<&1, &1, Hf.Err, Table>>:
291 e : Result<&1, &1, U32 & String, String> <- IO.get_env("MLBEND_DATA")
292 t : Result<&1, &1, Hf.Err, Table> <- from_env(e, repo, rev, path, header, drop)
293 return t
294
295# The breast cancer dataset: 683 rows, nine features, and the label.
296def cancer() -> IO(Result<&1, &1, Hf.Err, Table>):
297 csv_of("mstz/breast", "main", "cancer/train.csv", True{}, 0n)
298
299# The heart failure dataset: 299 rows, twelve features, and the label.
300def heart() -> IO(Result<&1, &1, Hf.Err, Table>):
301 csv_of("mstz/heart_failure", "main", "death/train.csv", True{}, 0n)
302
303# The iris dataset, fetched from the hub.
304def iris() -> IO(Result<&1, &1, Hf.Err, Table>):
305 csv_of("mstz/iris", "main", "iris.data", False{}, 0n)
306
307# The wine dataset: 6497 rows, eleven features, and the quality.
308def wine() -> IO(Result<&1, &1, Hf.Err, Table>):
309 csv_of("mstz/wine", "main", "wine/train.csv", True{}, 1n)