src/ez/test.bend
ezx@1.7.0.0 · 140 lines · 0 definitions
1# ez/test: the test runner. Bend has none, so a test is a `*/tests/*.bend` file
2# that prints, and states in a `#|` trailer the lines its run must print. Each
3# test is its own `bend` run, since what these drive is git, nix and bend
4# itself, and each wants its own process, scratch directory and port. A run
5# disagreeing with its trailer is a failure, and any failure is a non-zero
6# exit.
7#
8# Tests are not the proof gate. `ez prove` is, and nothing here runs a proof.
9import Base
10import snap@1.3.0.0/main.bend as R
11import ezx@1.7.0.0/src/io/file.bend as F
12import ezx@1.7.0.0/src/share/env.bend as Env
13import ezx@1.7.0.0/src/share/cap.bend as Cap
14import ezx@1.7.0.0/src/ez/gate.bend as G
15import ezx@1.7.0.0/src/ez/quiet.bend as Q
16import ezx@1.7.0.0/src/ez/clock.bend as Clock
17import ezx@1.7.0.0/src/ez/timer.bend as Timer
18import ezx@1.7.0.0/src/ez/prove.bend as P
19import ezx@1.7.0.0/src/ez/ends.bend as E
20import ezx@1.7.0.0/src/lock/run.bend as Run
21
22# how many comparisons were made, and how many of them held
23type Tally is Data:
24 Tally{pass: Nat, total: Nat}
25
26# a comparison that held says nothing. One that did not prints both sides in
27# full: a diff is real work in Bend, and for a handful of short lines the pair
28# reads better than one would.
29def say(ok: Bool, name: String, want: String, got: String) -> IO(Unit):
30 match ok:
31 case True{}:
32 IO.pure(Unit, Unit{})
33 case False{}:
34 IO.write("FAIL: " ++ name ++ "\n expected:\n" ++ P.indent(want) ++
35 " observed:\n" ++ P.indent(got))
36
37# one more comparison, and one more pass when it held
38def tally.go(ok: Bool, passed: Nat, total: Nat) -> Tally:
39 match ok:
40 case True{}:
41 Tally{(1n + passed : Nat), (1n + total : Nat)}
42 case False{}:
43 Tally{passed, (1n + total : Nat)}
44
45# the tally with this comparison's verdict in it
46def tally(ok: Bool, score: Tally) -> Tally:
47 Tally{p, n} = score
48 tally.go(ok, p, n)
49
50# what was wanted against what happened, reported and counted
51def check(name: String, +wanted: String, +got: String, score: Tally) -> IO(Tally):
52 do IO<Tally>:
53 say(String.eq(wanted, got), name, wanted, got)
54 return tally(String.eq(wanted, got), score)
55
56# one test against its own trailer
57def one(+path: String, +output: String, score: Tally) -> IO(Tally):
58 do IO<Tally>:
59 src : Maybe<&2, String> <- F.read(path)
60 check(path, G.trailer(F.text_of(src)), Q.filter(R.text(output)), score)
61
62# every test the answers ran out before. The two lists are the same length by
63# construction and this should never have one to report, but the alternative to
64# saying so is a green count over fewer tests than the tree holds, which is the
65# worst answer a gate can give.
66def short(ts: List<&2, String>, score: Tally) -> IO(Tally):
67 match ts:
68 case Nil{}:
69 IO.pure(Tally, score)
70 case Con{_h, rest}:
71 do IO<Tally>:
72 IO.write("FAIL: the run planned a test it never got an answer for\n")
73 short(rest, tally(False{}, score))
74
75# every test, in the tree's order however the run happened to interleave
76def report(ts: List<&2, String>, as: List<&2, String>, score: Tally) -> IO(Tally):
77 match ts as:
78 case Con{+h, tt} Con{+x, xt}:
79 do IO<Tally>:
80 next : Tally <- one(h, x, score)
81 report(tt, xt, next)
82 case rest _x:
83 short(rest, score)
84
85# every test file
86def find() -> IO(List<&2, String>):
87 P.find.at("*/tests/*.bend", "-path")
88
89# the count, and the exit status it implies. The reminder sits above the count
90# and never below it: a run's last line is its verdict, and something reads it.
91def done.go(+passed: Nat, +total: Nat) -> IO(Unit):
92 do IO<Unit>:
93 IO.print("tests are a last resort for what you cannot prove")
94 IO.print("PASS: " ++ Nat.show(passed) ++ " / " ++ Nat.show(total))
95 Run.end(E.counted(passed, total))
96
97# what a whole run amounts to
98def done(score: Tally) -> IO(Unit):
99 Tally{p, n} = score
100 done.go(p, n)
101
102# where the jobs of a run write what they print
103def out.at() -> String:
104 ".ez/run/test"
105
106# the budget checked the way every other thing here is checked. A run that
107# overran is a run that failed, and it says so in the count rather than merely
108# taking a long time.
109def budget(late: Bool, +secs: String, score: Tally) -> IO(Tally):
110 match late:
111 case False{}:
112 IO.pure(Tally, score)
113 case True{}:
114 do IO<Tally>:
115 IO.write("FAIL: the gate took longer than its " ++ secs ++ " second " ++
116 "budget\n set EZ_DEADLINE higher if this tree honestly needs " ++
117 "longer, or to 0 to stop asking\n")
118 return tally(False{}, score)
119
120# `ez test`: every test in the tree, all of them at once. Nothing here is
121# ordered against anything else, since each test has its own scratch directory
122# and port, and the report is still walked in the tree's order whatever order
123# the jobs finished in.
124def run() -> IO(Unit):
125 do IO<Unit>:
126 +cap : Bool <- Cap.ok()
127 Cap.warn(cap)
128 +secs : String <- Timer.seconds()
129 +deadline : String <- Timer.start(secs)
130 +g : String <- Cap.gb()
131 +at : String <- Env.lib()
132 +jobs : String <- P.width()
133 +ts : List<&2, String> <- find()
134 P.fresh.dir(out.at())
135 +by : String <- IO.pure(String, Clock.by.effect(deadline))
136 as : List<&2, String> <- R.par(P.cmds(ts, cap, g, at), jobs, g, out.at(), by)
137 every : Tally <- report(ts, as, Tally{0n, 0n})
138 +late : Bool <- Timer.over(deadline)
139 kept : Tally <- budget(late, secs, every)
140 done(kept)