src/ez/test.bend

ezx@1.11.0.0 · 140 lines · 14 definitions

1# ez/test: the test runner. Bend has none, so a test is a `*/tests/*.bend` file
2# that prints, and states in a `#|` trailer the lines its run must print. Each
3# test is its own `bend` run, since what these drive is git, nix and bend
4# itself, and each wants its own process, scratch directory and port. A run
5# disagreeing with its trailer is a failure, and any failure is a non-zero
6# exit.
7#
8# Tests are not the proof gate. `ez prove` is, and nothing here runs a proof.
9import Base
10import snap@1.4.0.0/main.bend as R
11import ezx@1.11.0.0/src/io/file.bend as F
12import ezx@1.11.0.0/src/share/env.bend as Env
13import ezx@1.11.0.0/src/share/cap.bend as Cap
14import ezx@1.11.0.0/src/ez/gate.bend as G
15import ezx@1.11.0.0/src/ez/quiet.bend as Q
16import ezx@1.11.0.0/src/ez/clock.bend as Clock
17import ezx@1.11.0.0/src/ez/timer.bend as Timer
18import ezx@1.11.0.0/src/ez/prove.bend as P
19import ezx@1.11.0.0/src/ez/ends.bend as E
20import ezx@1.11.0.0/src/lock/run.bend as Run
21
22# how many comparisons were made, and how many of them held
23type Tally is Data:
24  Tally{pass: Nat, total: Nat}
25
26# a comparison that held says nothing. One that did not prints both sides in
27# full: a diff is real work in Bend, and for a handful of short lines the pair
28# reads better than one would.
29def say(ok: Bool, name: String, want: String, got: String) -> IO(Unit):
30  match ok:
31    case True{}:
32      IO.pure(Unit, Unit{})
33    case False{}:
34      IO.write("FAIL: " ++ name ++ "\n  expected:\n" ++ P.indent(want) ++
35        "  observed:\n" ++ P.indent(got))
36
37# one more comparison, and one more pass when it held
38def tally.go(ok: Bool, passed: Nat, total: Nat) -> Tally:
39  match ok:
40    case True{}:
41      Tally{(1n + passed : Nat), (1n + total : Nat)}
42    case False{}:
43      Tally{passed, (1n + total : Nat)}
44
45# the tally with this comparison's verdict in it
46def tally(ok: Bool, score: Tally) -> Tally:
47  Tally{p, n} = score
48  tally.go(ok, p, n)
49
50# what was wanted against what happened, reported and counted
51def check(name: String, +wanted: String, +got: String, score: Tally) -> IO(Tally):
52  do IO<Tally>:
53    say(String.eq(wanted, got), name, wanted, got)
54    return tally(String.eq(wanted, got), score)
55
56# one test against its own trailer
57def one(+path: String, +output: String, score: Tally) -> IO(Tally):
58  do IO<Tally>:
59    src : Maybe<&2, String> <- F.read(path)
60    check(path, G.trailer(F.text_of(src)), Q.filter(R.text(output)), score)
61
62# every test the answers ran out before. The two lists are the same length by
63# construction and this should never have one to report, but the alternative to
64# saying so is a green count over fewer tests than the tree holds, which is the
65# worst answer a gate can give.
66def short(ts: List<&2, String>, score: Tally) -> IO(Tally):
67  match ts:
68    case Nil{}:
69      IO.pure(Tally, score)
70    case Con{_h, rest}:
71      do IO<Tally>:
72        IO.write("FAIL: the run planned a test it never got an answer for\n")
73        short(rest, tally(False{}, score))
74
75# every test, in the tree's order however the run happened to interleave
76def report(ts: List<&2, String>, as: List<&2, String>, score: Tally) -> IO(Tally):
77  match ts as:
78    case Con{+h, tt} Con{+x, xt}:
79      do IO<Tally>:
80        next : Tally <- one(h, x, score)
81        report(tt, xt, next)
82    case rest _x:
83      short(rest, score)
84
85# every test file
86def find() -> IO(List<&2, String>):
87  P.find.at("*/tests/*.bend", "-path")
88
89# the count, and the exit status it implies. The reminder sits above the count
90# and never below it: a run's last line is its verdict, and something reads it.
91def done.go(+passed: Nat, +total: Nat) -> IO(Unit):
92  do IO<Unit>:
93    IO.print("tests are a last resort for what you cannot prove")
94    IO.print("PASS: " ++ Nat.show(passed) ++ " / " ++ Nat.show(total))
95    Run.end(E.counted(passed, total))
96
97# what a whole run amounts to
98def done(score: Tally) -> IO(Unit):
99  Tally{p, n} = score
100  done.go(p, n)
101
102# where the jobs of a run write what they print
103def out.at() -> String:
104  ".ez/run/test"
105
106# the budget checked the way every other thing here is checked. A run that
107# overran is a run that failed, and it says so in the count rather than merely
108# taking a long time.
109def budget(late: Bool, +secs: String, score: Tally) -> IO(Tally):
110  match late:
111    case False{}:
112      IO.pure(Tally, score)
113    case True{}:
114      do IO<Tally>:
115        IO.write("FAIL: the gate took longer than its " ++ secs ++ " second " ++
116          "budget\n  set EZ_DEADLINE higher if this tree honestly needs " ++
117          "longer, or to 0 to stop asking\n")
118        return tally(False{}, score)
119
120# `ez test`: every test in the tree, all of them at once. Nothing here is
121# ordered against anything else, since each test has its own scratch directory
122# and port, and the report is still walked in the tree's order whatever order
123# the jobs finished in.
124def run() -> IO(Unit):
125  do IO<Unit>:
126    +cap : Bool <- Cap.ok()
127    Cap.warn(cap)
128    +secs : String <- Timer.seconds()
129    +deadline : String <- Timer.start(secs)
130    +g : String <- Cap.gb()
131    +at : String <- Env.lib()
132    +jobs : String <- P.width()
133    +ts : List<&2, String> <- find()
134    P.fresh.dir(out.at())
135    +by : String <- IO.pure(String, Clock.by.effect(deadline))
136    as : List<&2, String> <- R.par(P.cmds(ts, cap, g, at), jobs, g, out.at(), by)
137    every : Tally <- report(ts, as, Tally{0n, 0n})
138    +late : Bool <- Timer.over(deadline)
139    kept : Tally <- budget(late, secs, every)
140    done(kept)