1 # mlbend-datasets: the datasets of scikit-learn, fetched from the hub. 2 # 3 # import mlbend-datasets@0.1.4.0/main.bend as Datasets 4 # 5 # The features of a dataset and its labels beside them: the rows of the 6 # features, and one label per row. 7 # 8 # The labels are numbers. A classification label is the index of the 9 # class, a regression label is the value; which one it is, the model 10 # decides. 11 # 12 # Every dataset here comes from HuggingFace, over the network, at run 13 # time: the effect is at the edge and the data that comes back is a 14 # value, so an estimator over it stays pure and a run is reproducible. 15 # 16 # Two shapes of file. The iris dataset is the one scikit-learn calls 17 # `load_iris`: 150 rows, four features, three classes, and no header, 18 # 19 # 5.1,3.5,1.4,0.2,Iris-setosa 20 # 21 # The others are CSV with a header, and the label is one of the columns: 22 # 23 # fixed_acidity,volatile_acidity,...,alcohol,quality,is_red 24 # 7.4,0.7,0.0,1.9,0.076,11.0,34.0,0.9978,3.51,0.56,9.4,5,0 25 # 26 # The loader is told how many columns to leave out at the end of a row: 27 # the label is the column before them. 28 29 import Base 30 import mlbend-huggingface@0.1.1.0/main.bend as Hf 31 32 # A dataset: the rows of the features and the labels beside them. 33 34 type Table is Data: 35 Table{rows: List<&2, List<&2, F32>>, labels: List<&2, F32>} 36 37 # One row of the file: its features and its label. 38 type Row is Data: 39 Row{features: List<&2, F32>, label: F32} 40 41 # The rows read so far. 42 type Rows is Data: 43 Rows{rows: List<&2, Row>, labels: List<&2, F32>} 44 45 # One row's features, appended to the rows so far. 46 def add_features(acc: List<&2, List<&2, F32>>, r: Row) -> List<&2, List<&2, F32>>: 47 match r: 48 case Row{features, label}: 49 List.append(&2, List<&2, F32>, acc, Con{features, Nil{}}) 50 51 # The rows of a file, and the row one line makes. 52 def add_row_of(acc: Rows, +r: Row) -> Rows: 53 match acc: 54 case Rows{rows, labels}: 55 match r: 56 case Row{features, +label}: 57 Rows{List.append(&2, Row, rows, r <> Nil{}), 58 List.append(&2, F32, labels, label <> Nil{})} 59 60 # The rows of the features, one after the other. 61 def feature_rows(rs: List<&2, Row>) -> List<&2, List<&2, F32>>: 62 List.foldl(&2, Row, List<&2, List<&2, F32>>, add_features, rs, Nil{}) 63 64 # The dataset the rows make. 65 def data_of(acc: Rows) -> Table: 66 match acc: 67 case Rows{rows, labels}: 68 Table{feature_rows(rows), labels} 69 70 # A line that holds something. 71 def not_empty(line: String) -> Bool: 72 Bool.not(String.is_empty(line)) 73 74 # The lines of a file without its empty ones: a file ends with a 75 # newline, and the line after it is empty. 76 def drop_empty(lines: List<&2, String>) -> List<&2, String>: 77 List.filter(String, not_empty, lines) 78 79 # A number that may be there, or zero when it is not. 80 def number_or_zero(m: Maybe<&2, F32>) -> F32: 81 match m: 82 case Some{v}: 83 v 84 case None{}: 85 0.0 86 87 # A field of a row, as a number: a number is itself, True is 1, False is 88 # 0, and anything else is 0. 89 def cell(+text: String) -> F32: 90 match text: 91 case "True": 92 1.0 93 case "False": 94 0.0 95 case _: 96 number_or_zero(F32.read(text)) 97 98 # The cells of a row, as numbers. 99 def cells_of(+xs: List<&2, String>) -> List<&2, F32>: 100 match xs: 101 case Nil{}: 102 Nil{} 103 case Con{h, t}: 104 List.append(&2, F32, Nil{}, Con{cell(h), cells_of(t)}) 105 106 # The place of the label in a row: the last column, less the columns 107 # that are left out at the end. 108 def label_index(+xs: List<&2, String>, drop: Nat) -> Nat: 109 Nat.sub(List.length(&2, String, xs), Nat.add(1n, drop)) 110 111 # The fields before the label. 112 def fields_before(+xs: List<&2, String>, drop: Nat) -> List<&2, String>: 113 List.take(&2, String, xs, label_index(xs, drop)) 114 115 # The features of a row: the fields before the label, as numbers. 116 def features_of(+xs: List<&2, String>, drop: Nat) -> List<&2, F32>: 117 cells_of(fields_before(xs, drop)) 118 119 # The class of a row, by name: 0, 1 or 2 for the three iris species. 120 def class_of(name: String) -> F32: 121 match name: 122 case "Iris-setosa": 123 0.0 124 case "Iris-versicolor": 125 1.0 126 case "Iris-virginica": 127 2.0 128 case _: 129 0.0 130 131 # The label of a row: a class name is its class, a number is itself. 132 def class_or_number(+text: String) -> F32: 133 match text: 134 case "Iris-setosa": 135 class_of(text) 136 case "Iris-versicolor": 137 class_of(text) 138 case "Iris-virginica": 139 class_of(text) 140 case _: 141 number_or_zero(F32.read(text)) 142 143 # The label of a row, when the field is there. 144 def label_at(m: Maybe<&2, String>) -> F32: 145 match m: 146 case Some{s}: 147 class_or_number(s) 148 case None{}: 149 0.0 150 151 # The text of a field of a row. 152 def text_at(+xs: List<&2, String>, n: Nat) -> Maybe<&2, String>: 153 List.get(&2, String, xs, n) 154 155 # The label of a row: the column at its place. 156 def label_of(+xs: List<&2, String>, drop: Nat) -> F32: 157 label_at(text_at(xs, label_index(xs, drop))) 158 159 # One row of the file, from its fields. 160 def row_of_parts(+fs: List<&2, F32>, l: F32) -> Row: 161 Row{fs, l} 162 163 # One row of the file. 164 def row_of(+line: String, +drop: Nat) -> Row: 165 +xs = String.split(line, Chr{44}) 166 row_of_parts(features_of(xs, drop), label_of(xs, drop)) 167 168 # The rows of a file, one line at a time. 169 def rows_go(lines: List<&2, String>, +drop: Nat, +acc: Rows) -> Rows: 170 match lines: 171 case Nil{}: 172 acc 173 case Con{h, t}: 174 rows_go(t, drop, add_row_of(acc, row_of(h, drop))) 175 176 # The rows of a file. 177 def rows_of(lines: List<&2, String>, drop: Nat) -> Rows: 178 rows_go(lines, drop, Rows{Nil{}, Nil{}}) 179 180 # The dataset a file of rows makes. 181 def build(text: String, drop: Nat) -> Table: 182 data_of(rows_of(drop_empty(String.lines(text)), drop)) 183 184 # The dataset the rows of a CSV make. 185 def build_csv_rows(+lines: List<&2, String>, drop: Nat) -> Table: 186 data_of(rows_of(drop_empty(lines), drop)) 187 188 # The lines of a file without its first. 189 def drop_first(+lines: List<&2, String>) -> List<&2, String>: 190 List.drop(&2, String, lines, 1n) 191 192 # The dataset a file of CSV makes: the first line is the header and the 193 # label is the column before the ones left out at the end. 194 def build_csv(text: String, drop: Nat) -> Table: 195 build_csv_rows(drop_first(String.lines(text)), drop) 196 197 # The dataset the text of a file makes. 198 def csv_text(+text: String, header: Bool, +drop: Nat) -> Table: 199 Bool.pick(Table, header, build_csv(text, drop), build(text, drop)) 200 201 # The dataset a file of text makes. 202 def csv_of_text(t: Result<&1, &1, Hf.Err, String>, header: Bool, drop: Nat) -> 203 Result<&1, &1, Hf.Err, Table>: 204 match t: 205 case Fail{e}: 206 Fail{e} 207 case Done{text}: 208 Done{csv_text(text, header, drop)} 209 210 # A file of a dataset's repository, and the dataset it makes. 211 def csv_of(repo: String, rev: String, path: String, header: Bool, drop: Nat) -> 212 IO(Result<&1, &1, Hf.Err, Table>): 213 do IO>: 214 t : Result<&1, &1, Hf.Err, String> <- Hf.file_text(repo, rev, path) 215 return csv_of_text(t, header, drop) 216 217 # The breast cancer dataset: 683 rows, nine features, and the label. 218 def cancer() -> IO(Result<&1, &1, Hf.Err, Table>): 219 csv_of("mstz/breast", "main", "cancer/train.csv", True{}, 0n) 220 221 # The heart failure dataset: 299 rows, twelve features, and the label. 222 def heart() -> IO(Result<&1, &1, Hf.Err, Table>): 223 csv_of("mstz/heart_failure", "main", "death/train.csv", True{}, 0n) 224 225 # The iris dataset, fetched from the hub. 226 def iris() -> IO(Result<&1, &1, Hf.Err, Table>): 227 csv_of("mstz/iris", "main", "iris.data", False{}, 0n) 228 229 # The wine dataset: 6497 rows, eleven features, and the quality. 230 def wine() -> IO(Result<&1, &1, Hf.Err, Table>): 231 csv_of("mstz/wine", "main", "wine/train.csv", True{}, 1n)