main.bend

mlbend-datasets@0.1.6.0 · 314 lines · 44 definitions

1# mlbend-datasets: the datasets of scikit-learn, fetched from the hub.
2#
3#   import mlbend-datasets@0.1.6.0/main.bend as Datasets
4#
5# The features of a dataset and its labels beside them: the rows of the
6# features, and one label per row.
7#
8# The labels are numbers. A classification label is the index of the
9# class, a regression label is the value; which one it is, the model
10# decides.
11#
12# Every dataset here comes from HuggingFace, over the network, at run
13# time: the effect is at the edge and the data that comes back is a
14# value, so an estimator over it stays pure and a run is reproducible.
15#
16# Two shapes of file. The iris dataset is the one scikit-learn calls
17# `load_iris`: 150 rows, four features, three classes, and no header,
18#
19#   5.1,3.5,1.4,0.2,Iris-setosa
20#
21# The others are CSV with a header, and the label is one of the columns:
22#
23#   fixed_acidity,volatile_acidity,...,alcohol,quality,is_red
24#   7.4,0.7,0.0,1.9,0.076,11.0,34.0,0.9978,3.51,0.56,9.4,5,0
25#
26# The loader is told how many columns to leave out at the end of a row:
27# the label is the column before them.
28
29import Base
30import mlbend-huggingface@0.1.1.0/main.bend as Hf
31
32# A dataset: the rows of the features and the labels beside them.
33
34type Table is Data:
35  Table{rows: List<&2, List<&2, F32>>, labels: List<&2, F32>}
36
37# One row of the file: its features and its label.
38type Row is Data:
39  Row{features: List<&2, F32>, label: F32}
40
41# The rows read so far.
42type Rows is Data:
43  Rows{rows: List<&2, Row>, labels: List<&2, F32>}
44
45# One row's features, appended to the rows so far.
46def add_features(acc: List<&2, List<&2, F32>>, r: Row) -> List<&2, List<&2, F32>>:
47  match r:
48    case Row{features, label}:
49      List.append(&2, List<&2, F32>, acc, Con{features, Nil{}})
50
51# The rows of a file, and the row one line makes.
52def add_row_of(acc: Rows, +r: Row) -> Rows:
53  match acc:
54    case Rows{rows, labels}:
55      match r:
56        case Row{features, +label}:
57          Rows{List.append(&2, Row, rows, r <> Nil{}),
58            List.append(&2, F32, labels, label <> Nil{})}
59
60# The rows of the features, one after the other.
61def feature_rows(rs: List<&2, Row>) -> List<&2, List<&2, F32>>:
62  List.foldl(&2, Row, List<&2, List<&2, F32>>, add_features, rs, Nil{})
63
64# The dataset the rows make.
65def data_of(acc: Rows) -> Table:
66  match acc:
67    case Rows{rows, labels}:
68      Table{feature_rows(rows), labels}
69
70# A line that holds something.
71def not_empty(line: String) -> Bool:
72  Bool.not(String.is_empty(line))
73
74# The lines of a file without its empty ones: a file ends with a
75# newline, and the line after it is empty.
76def drop_empty(lines: List<&2, String>) -> List<&2, String>:
77  List.filter(String, not_empty, lines)
78
79# A number that may be there, or zero when it is not.
80def number_or_zero(m: Maybe<&2, F32>) -> F32:
81  match m:
82    case Some{v}:
83      v
84    case None{}:
85      0.0
86
87# A field of a row, as a number: a number is itself, True is 1, False is
88# 0, and anything else is 0.
89def cell(+text: String) -> F32:
90  match text:
91    case "True":
92      1.0
93    case "False":
94      0.0
95    case _:
96      number_or_zero(F32.read(text))
97
98# The cells of a row, as numbers.
99def cells_of(+xs: List<&2, String>) -> List<&2, F32>:
100  match xs:
101    case Nil{}:
102      Nil{}
103    case Con{h, t}:
104      List.append(&2, F32, Nil{}, Con{cell(h), cells_of(t)})
105
106# The place of the label in a row: the last column, less the columns
107# that are left out at the end.
108def label_index(+xs: List<&2, String>, drop: Nat) -> Nat:
109  Nat.sub(List.length(&2, String, xs), Nat.add(1n, drop))
110
111# The fields before the label.
112def fields_before(+xs: List<&2, String>, drop: Nat) -> List<&2, String>:
113  List.take(&2, String, xs, label_index(xs, drop))
114
115# The features of a row: the fields before the label, as numbers.
116def features_of(+xs: List<&2, String>, drop: Nat) -> List<&2, F32>:
117  cells_of(fields_before(xs, drop))
118
119# The class of a row, by name: 0, 1 or 2 for the three iris species.
120def class_of(name: String) -> F32:
121  match name:
122    case "Iris-setosa":
123      0.0
124    case "Iris-versicolor":
125      1.0
126    case "Iris-virginica":
127      2.0
128    case _:
129      0.0
130
131# The label of a row: a class name is its class, a number is itself.
132def class_or_number(+text: String) -> F32:
133  match text:
134    case "Iris-setosa":
135      class_of(text)
136    case "Iris-versicolor":
137      class_of(text)
138    case "Iris-virginica":
139      class_of(text)
140    case _:
141      number_or_zero(F32.read(text))
142
143# The label of a row, when the field is there.
144def label_at(m: Maybe<&2, String>) -> F32:
145  match m:
146    case Some{s}:
147      class_or_number(s)
148    case None{}:
149      0.0
150
151# The text of a field of a row.
152def text_at(+xs: List<&2, String>, n: Nat) -> Maybe<&2, String>:
153  List.get(&2, String, xs, n)
154
155# The label of a row: the column at its place.
156def label_of(+xs: List<&2, String>, drop: Nat) -> F32:
157  label_at(text_at(xs, label_index(xs, drop)))
158
159# One row of the file, from its fields.
160def row_of_parts(+fs: List<&2, F32>, l: F32) -> Row:
161  Row{fs, l}
162
163# One row of the file.
164def row_of(+line: String, +drop: Nat) -> Row:
165  +xs = String.split(line, Chr{44})
166  row_of_parts(features_of(xs, drop), label_of(xs, drop))
167
168# The rows of a file, one line at a time.
169def rows_go(lines: List<&2, String>, +drop: Nat, +acc: Rows) -> Rows:
170  match lines:
171    case Nil{}:
172      acc
173    case Con{h, t}:
174      rows_go(t, drop, add_row_of(acc, row_of(h, drop)))
175
176# The rows of a file.
177def rows_of(lines: List<&2, String>, drop: Nat) -> Rows:
178  rows_go(lines, drop, Rows{Nil{}, Nil{}})
179
180# The dataset a file of rows makes.
181def build(text: String, drop: Nat) -> Table:
182  data_of(rows_of(drop_empty(String.lines(text)), drop))
183
184# The dataset the rows of a CSV make.
185def build_csv_rows(+lines: List<&2, String>, drop: Nat) -> Table:
186  data_of(rows_of(drop_empty(lines), drop))
187
188# The lines of a file without its first.
189def drop_first(+lines: List<&2, String>) -> List<&2, String>:
190  List.drop(&2, String, lines, 1n)
191
192# The dataset a file of CSV makes: the first line is the header and the
193# label is the column before the ones left out at the end.
194def build_csv(text: String, drop: Nat) -> Table:
195  build_csv_rows(drop_first(String.lines(text)), drop)
196
197# The dataset the text of a file makes.
198def csv_text(+text: String, header: Bool, +drop: Nat) -> Table:
199  Bool.pick(Table, header, build_csv(text, drop), build(text, drop))
200
201# The dataset a file of text makes.
202def csv_of_text(t: Result<&1, &1, Hf.Err, String>, header: Bool, drop: Nat) ->
203  Result<&1, &1, Hf.Err, Table>:
204  match t:
205    case Fail{e}:
206      Fail{e}
207    case Done{text}:
208      Done{csv_text(text, header, drop)}
209
210# ---- the local cache: MLBEND_DATA, a directory laid out like the hub ----
211#
212# The runner fills <dir>/<repo>/<path> with the exact bytes it hashed. When
213# the variable is set and the file is there, the program reads it and does
214# not touch the network; otherwise it falls back to the hub client.
215
216def cached_path(dir: String, repo: String, path: String) -> String:
217  String.append(dir, String.append("/", String.append(repo, String.append("/", path))))
218
219def read_result(result: Result<&1, &1, U32 & String, String>) -> IO(Maybe<&1, String>):
220  match result:
221    case Fail{e}:
222      IO.pure(Maybe<&1, String>, None{})
223    case Done{text}:
224      IO.pure(Maybe<&1, String>, Some{text})
225
226def read_done(pair: File & Result<&1, &1, U32 & String, String>) -> IO(Maybe<&1, String>):
227  (file, result) = pair
228  do IO<Maybe<&1, String>>:
229    File.close(file)
230    read_result(result)
231
232def sized_done(pair: File & Result<&1, &1, U32 & String, U32>) -> IO(Maybe<&1, String>):
233  (file, result) = pair
234  match result:
235    case Fail{e}:
236      do IO<Maybe<&1, String>>:
237        File.close(file)
238        return None{}
239    case Done{size}:
240      do IO<Maybe<&1, String>>:
241        pair : File & Result<&1, &1, U32 & String, String> <- File.read(file, size)
242        m : Maybe<&1, String> <- read_done(pair)
243        return m
244
245def opened(r: Result<&1, &1, U32 & String, File>) -> IO(Maybe<&1, String>):
246  match r:
247    case Fail{e}:
248      IO.pure(Maybe<&1, String>, None{})
249    case Done{file}:
250      do IO<Maybe<&1, String>>:
251        pair : File & Result<&1, &1, U32 & String, U32> <- File.size(file)
252        m : Maybe<&1, String> <- sized_done(pair)
253        return m
254
255# The text of a cached file, or None when it is not there.
256def read_cached(path: String) -> IO(Maybe<&1, String>):
257  do IO<Maybe<&1, String>>:
258    r : Result<&1, &1, U32 & String, File> <- File.open(path, "r")
259    opened(r)
260
261# A table from the cache when it is there. With MLBEND_DATA set the cache is
262# the source of the run: a miss is an error, not a quiet fetch, or the
263# manifest would hash one file and the program would read another.
264def cached_or_fail(m: Maybe<&1, String>, +repo: String, +path: String, header: Bool,
265  drop: Nat) -> IO(Result<&1, &1, Hf.Err, Table>):
266  match m:
267    case Some{text}:
268      IO.pure(Result<&1, &1, Hf.Err, Table>, Done{csv_text(text, header, drop)})
269    case None{}:
270      IO.pure(Result<&1, &1, Hf.Err, Table>,
271        Fail{Hf.ErrNet{0, String.append("MLBEND_DATA: no readable file at ",
272          String.append(repo, String.append("/", path)))}})
273
274# A table from the hub, when there is no cache to read.
275def fetch_csv(repo: String, rev: String, path: String, header: Bool, drop: Nat) ->
276  IO(Result<&1, &1, Hf.Err, Table>):
277  do IO<Result<&1, &1, Hf.Err, Table>>:
278    t : Result<&1, &1, Hf.Err, String> <- Hf.file_text(repo, rev, path)
279    return csv_of_text(t, header, drop)
280
281def from_env(e: Result<&1, &1, U32 & String, String>, +repo: String, rev: String, +path: String,
282  header: Bool, drop: Nat) -> IO(Result<&1, &1, Hf.Err, Table>):
283  match e:
284    case Fail{code}:
285      fetch_csv(repo, rev, path, header, drop)
286    case Done{dir}:
287      do IO<Result<&1, &1, Hf.Err, Table>>:
288        m : Maybe<&1, String> <- read_cached(cached_path(dir, repo, path))
289        t : Result<&1, &1, Hf.Err, Table> <- cached_or_fail(m, repo, path, header, drop)
290        return t
291
292# A file of a dataset's repository, and the dataset it makes.
293def csv_of(repo: String, rev: String, path: String, header: Bool, drop: Nat) ->
294  IO(Result<&1, &1, Hf.Err, Table>):
295  do IO<Result<&1, &1, Hf.Err, Table>>:
296    e : Result<&1, &1, U32 & String, String> <- IO.get_env("MLBEND_DATA")
297    t : Result<&1, &1, Hf.Err, Table> <- from_env(e, repo, rev, path, header, drop)
298    return t
299
300# The breast cancer dataset: 683 rows, nine features, and the label.
301def cancer() -> IO(Result<&1, &1, Hf.Err, Table>):
302  csv_of("mstz/breast", "3cc7c6eef1160389f64f8760d71cba2b789603ec", "cancer/train.csv", True{}, 0n)
303
304# The heart failure dataset: 299 rows, twelve features, and the label.
305def heart() -> IO(Result<&1, &1, Hf.Err, Table>):
306  csv_of("mstz/heart_failure", "57c2e1a4b3bb2f28e766c6d3b10bc0ced972c857", "death/train.csv", True{}, 0n)
307
308# The iris dataset, fetched from the hub.
309def iris() -> IO(Result<&1, &1, Hf.Err, Table>):
310  csv_of("mstz/iris", "e99306d5df05d2ea66a188ea936037f6588a20e9", "iris.data", False{}, 0n)
311
312# The wine dataset: 6497 rows, eleven features, and the quality.
313def wine() -> IO(Result<&1, &1, Hf.Err, Table>):
314  csv_of("mstz/wine", "2042c73ed074afafc54264ba6ef0ad0450fa1e04", "wine/train.csv", True{}, 1n)