main.bend

mlbend-huggingface@0.1.1.0 · 278 lines · 38 definitions

1# mlbend-huggingface: a HuggingFace client for Bend 2.
2#
3#   import mlbend-hf@0.1.0.0/main.bend as Hf
4#
5# Two APIs, both JSON over HTTP.
6#
7# The dataset's own host:
8#
9#   GET huggingface.co/api/datasets/<name>             the dataset's page
10#   GET huggingface.co/api/datasets/<name>/tree/<rev>  the files in it
11#   GET huggingface.co/datasets/<name>/resolve/<rev>/<path>  a file's bytes
12#
13# And the rows server, which answers a dataset that is stored as data as
14# JSON rows, with the features beside them:
15#
16#   GET datasets-server.huggingface.co/rows?dataset=<name>&config=<c>&split=<s>&offset=<n>&length=<k>
17#
18# A dataset that is a script rather than data cannot be read this way. The
19# rows server says so, and its message says what to do about it: convert the
20# dataset to Parquet. The file tree and the raw file download work for those
21# anyway.
22#
23# Everything is an effect at the edge, so the estimators that use a dataset
24# stay pure and a run is reproducible.
25#
26# A note on the shape of the code: Bend's checker will not let a match
27# scrutinize a computed value, so every helper that unwraps a Maybe takes it
28# as a parameter and matches on that. The callers get one call each.
29
30import Base
31import bend-kit-http@0.36.0.0/http.bend as Http
32import bend-kit-json@0.6.1.1/json.bend as Json
33import bend-kit-bytes@0.3.2.0/bytes.bend as Bytes
34
35# Why an answer did not come.
36type Err is Data:
37  ErrNet{code: U32, why: String}
38  ErrStatus{status: U32}
39  ErrJson{}
40
41# The hosts.
42def host() -> String:
43  "https://huggingface.co"
44
45def rows_host() -> String:
46  "https://datasets-server.huggingface.co"
47
48# The HTTP kit's error, as ours.
49def why_of(e: Http.Err) -> Err:
50  match e:
51    case Http.ErrConnect{code, why}:
52      ErrNet{code, why}
53    case Http.ErrTls{code, why}:
54      ErrNet{code, why}
55    case Http.ErrRead{code, why}:
56      ErrNet{code, why}
57    case Http.ErrWrite{code, why}:
58      ErrNet{code, why}
59    case Http.ErrUrl{}:
60      ErrNet{0, "a url bend cannot open"}
61    case Http.ErrDns{}:
62      ErrNet{0, "a name that does not resolve"}
63    case Http.ErrTimeout{}:
64      ErrNet{0, "the answer did not come in time"}
65    case Http.ErrRedirect{}:
66      ErrNet{0, "a redirect bend did not follow"}
67    case Http.ErrBad{}:
68      ErrNet{0, "a malformed answer"}
69
70# The JSON of a parsed body.
71def json_of_maybe(m: Maybe<&1, Json.Val>) -> Result<&1, &1, Err, Json.Val>:
72  match m:
73    case Some{v}:
74      Done{v}
75    case None{}:
76      Fail{ErrJson{}}
77
78# The JSON of a body, or the reason there is none.
79def json_of_body(body: Bytes.Bytes) -> Result<&1, &1, Err, Json.Val>:
80  json_of_maybe(Json.parse.bytes(body))
81
82# The JSON of a response, or the reason there is none.
83def json_of(r: Result<&1, &1, Http.Err, Http.Res>) -> Result<&1, &1, Err, Json.Val>:
84  match r:
85    case Fail{e}:
86      Fail{why_of(e)}
87    case Done{Http.Res{+status, headers, body}}:
88      Bool.pick(Result<&1, &1, Err, Json.Val>, U32.is_lt(status, 400),
89        json_of_body(body),
90        Fail{ErrStatus{status}})
91
92# The bytes of a response, or the reason there are none.
93def bytes_of(r: Result<&1, &1, Http.Err, Http.Res>) -> Result<&1, &1, Err, Bytes.Bytes>:
94  match r:
95    case Fail{e}:
96      Fail{why_of(e)}
97    case Done{Http.Res{+status, headers, body}}:
98      Bool.pick(Result<&1, &1, Err, Bytes.Bytes>, U32.is_lt(status, 400),
99        Done{body},
100        Fail{ErrStatus{status}})
101
102# A GET, and the JSON it answered with.
103def get_json(url: String) -> IO(Result<&1, &1, Err, Json.Val>):
104  do IO<Result<&1, &1, Err, Json.Val>>:
105    r : Result<&1, &1, Http.Err, Http.Res> <- Http.get(url)
106    return json_of(r)
107
108# A GET, and the bytes it answered with.
109def get_bytes(url: String) -> IO(Result<&1, &1, Err, Bytes.Bytes>):
110  do IO<Result<&1, &1, Err, Bytes.Bytes>>:
111    r : Result<&1, &1, Http.Err, Http.Res> <- Http.get(url)
112    return bytes_of(r)
113
114# The page of a dataset: its name, its revision, and what it says about
115# itself.
116def dataset(name: String) -> IO(Result<&1, &1, Err, Json.Val>):
117  get_json(String.append(String.append(host(), "/api/datasets/"), name))
118
119# The files of a dataset, at a revision.
120def tree(name: String, rev: String) -> IO(Result<&1, &1, Err, Json.Val>):
121  get_json(String.append(String.append(String.append(host(), "/api/datasets/"), name),
122    String.append("/tree/", rev)))
123
124# A file of a dataset, at a revision: its bytes.
125def file(name: String, rev: String, path: String) -> IO(Result<&1, &1, Err, Bytes.Bytes>):
126  get_bytes(String.append(String.append(String.append(String.append(host(), "/datasets/"), name),
127    String.append("/resolve/", rev)), String.append("/", path)))
128
129# The rest of a query string, each pair behind an "&".
130def query.go(pairs: List<&2, String>, acc: String) -> String:
131  match pairs:
132    case Nil{}:
133      acc
134    case Con{h, t}:
135      query.go(t, String.append(acc, String.append("&", h)))
136
137# A url with a query string: the base, the first pair, and the rest.
138def query(base: String, first: String, rest: List<&2, String>) -> String:
139  query.go(rest, String.append(base, first))
140
141# The url of the rows server's answer for a dataset, a config and a
142# split, a window of rows.
143def rows_url(name: String, config: String, split: String, offset: Nat, length: Nat) -> String:
144  query(String.append(rows_host(), "/rows?dataset="), name, [
145    String.append("config=", config),
146    String.append("split=", split),
147    String.append("offset=", U32.show(U32.from_nat(offset))),
148    String.append("length=", U32.show(U32.from_nat(length)))])
149
150# The text of a file that may be there.
151def text_of_file(r: Result<&1, &1, Err, Bytes.Bytes>) -> Result<&1, &1, Err, String>:
152  match r:
153    case Fail{e}:
154      Fail{e}
155    case Done{b}:
156      Done{Bytes.to_string(b)}
157
158# A file of a dataset, at a revision, as text.
159def file_text(name: String, rev: String, path: String) -> IO(Result<&1, &1, Err, String>):
160  do IO<Result<&1, &1, Err, String>>:
161    b : Result<&1, &1, Err, Bytes.Bytes> <- file(name, rev, path)
162    return text_of_file(b)
163
164# The rows of a dataset, at a config and a split: the features beside them.
165def rows(name: String, config: String, split: String, offset: Nat, length: Nat) ->
166  IO(Result<&1, &1, Err, Json.Val>):
167  get_json(rows_url(name, config, split, offset, length))
168
169# Reading the JSON
170# ----------------
171
172# A field of a JSON object.
173def field(v: Json.Val, key: String) -> Maybe<&1, Json.Val>:
174  Json.get(v, key)
175
176# An item of a JSON array.
177def item(v: Json.Val, n: Nat) -> Maybe<&1, Json.Val>:
178  Json.at(v, n)
179
180# The text of a JSON value, when it is a string or a number.
181def text_value(v: Json.Val) -> Maybe<&1, String>:
182  match v:
183    case Json.Str{s}:
184      Some{Bytes.to_string(s)}
185    case Json.Num{s}:
186      Some{Bytes.to_string(s)}
187    case _:
188      None{}
189
190# The text of a JSON string, or of a JSON number: the number keeps its
191# exact text, so nothing is lost on the way.
192def text_of(v: Json.Val) -> Maybe<&1, String>:
193  text_value(v)
194
195# The text of a value that may be there.
196def text_of_value(m: Maybe<&1, Json.Val>) -> Maybe<&1, String>:
197  match m:
198    case Some{v}:
199      text_value(v)
200    case None{}:
201      None{}
202
203# The text of some text, or "" when there is none.
204def text_maybe(m: Maybe<&1, String>) -> String:
205  match m:
206    case Some{s}:
207      s
208    case None{}:
209      ""
210
211# The field of an item of a JSON array, when the item is there.
212def field_of(m: Maybe<&1, Json.Val>, key: String) -> Maybe<&1, Json.Val>:
213  match m:
214    case Some{it}:
215      field(it, key)
216    case None{}:
217      None{}
218
219# The field of a field of an item of a JSON array, when the item is there.
220def field_of_field(m: Maybe<&1, Json.Val>, key1: String, key2: String) -> Maybe<&1, Json.Val>:
221  match m:
222    case Some{it}:
223      field_of(field(it, key1), key2)
224    case None{}:
225      None{}
226
227# The text of a field of a JSON object, or "" when it is not there.
228def text_field(v: Json.Val, key: String) -> String:
229  text_maybe(text_of_value(field(v, key)))
230
231# The text of an item of a JSON array, or "" when it is not there.
232def text_item(v: Json.Val, n: Nat) -> String:
233  text_maybe(text_of_value(item(v, n)))
234
235# The field of an item of a JSON array.
236def field_item(v: Json.Val, n: Nat, key: String) -> Maybe<&1, Json.Val>:
237  field_of(item(v, n), key)
238
239# The text of a field of an item of a JSON array.
240def text_item_field(v: Json.Val, n: Nat, key: String) -> String:
241  text_maybe(text_of_value(field_item(v, n, key)))
242
243# The field of a field of an item of a JSON array: the row of the rows
244# server, which is two fields down from the row itself.
245def field_item_field(v: Json.Val, n: Nat, key1: String, key2: String) -> Maybe<&1, Json.Val>:
246  field_of_field(item(v, n), key1, key2)
247
248# The text of a field of a field of an item of a JSON array.
249def text_item_field_field(v: Json.Val, n: Nat, key1: String, key2: String) -> String:
250  text_maybe(text_of_value(field_item_field(v, n, key1, key2)))
251
252# A field of a JSON object, when it is there.
253def array_of(m: Maybe<&1, Json.Val>) -> Json.Val:
254  match m:
255    case Some{a}:
256      a
257    case None{}:
258      Json.Arr{Nil{}}
259
260# The field of a JSON object, or an empty array when it is not there.
261def array_field(v: Json.Val, key: String) -> Json.Val:
262  array_of(field(v, key))
263
264# The number some text holds, when it is one that fits.
265def number_of_text(t: Maybe<&1, String>) -> Maybe<&2, F32>:
266  match t:
267    case Some{s}:
268      F32.read(s)
269    case None{}:
270      None{}
271
272# The number a JSON number holds, when it is a whole number that fits.
273def number_of(v: Json.Val) -> Maybe<&2, F32>:
274  number_of_text(text_of(v))
275
276# The number a field of a JSON object holds, when it is one that fits.
277def number_field(v: Json.Val, key: String) -> Maybe<&2, F32>:
278  number_of_text(text_of_value(field(v, key)))