1	# ez/test: the test runner. Bend has none, so a test is a `*/tests/*.bend` file
2	# that prints, and states in a `#|` trailer the lines its run must print. Each
3	# test is its own `bend` run, since what these drive is git, nix and bend
4	# itself, and each wants its own process, scratch directory and port. A run
5	# disagreeing with its trailer is a failure, and any failure is a non-zero
6	# exit.
7	#
8	# Tests are not the proof gate. `ez prove` is, and nothing here runs a proof.
9	import Base
10	import 0xabe575924687afad4cee1a2c1194d639/main.bend as R
11	import ../io/file.bend as F
12	import ../share/env.bend as Env
13	import ../share/cap.bend as Cap
14	import ./gate.bend as G
15	import ./quiet.bend as Q
16	import ./clock.bend as Clock
17	import ./timer.bend as Timer
18	import ./prove.bend as P
19	import ../ez/ends.bend as E
20	import ../lock/run.bend as Run
21	
22	# how many comparisons were made, and how many of them held
23	type Tally is Data:
24	  Tally{pass: Nat, total: Nat}
25	
26	# a comparison that held says nothing. One that did not prints both sides in
27	# full: a diff is real work in Bend, and for a handful of short lines the pair
28	# reads better than one would.
29	def say(ok: Bool, name: String, want: String, got: String) -> IO(Unit):
30	  match ok:
31	    case True{}:
32	      IO.pure(Unit, Unit{})
33	    case False{}:
34	      IO.write("FAIL: " ++ name ++ "\n  expected:\n" ++ P.indent(want) ++
35	        "  observed:\n" ++ P.indent(got))
36	
37	# one more comparison, and one more pass when it held
38	def tally.go(ok: Bool, passed: Nat, total: Nat) -> Tally:
39	  match ok:
40	    case True{}:
41	      Tally{(1n + passed : Nat), (1n + total : Nat)}
42	    case False{}:
43	      Tally{passed, (1n + total : Nat)}
44	
45	# the tally with this comparison's verdict in it
46	def tally(ok: Bool, score: Tally) -> Tally:
47	  Tally{p, n} = score
48	  tally.go(ok, p, n)
49	
50	# what was wanted against what happened, reported and counted
51	def check(name: String, +wanted: String, +got: String, score: Tally) -> IO(Tally):
52	  do IO<Tally>:
53	    say(String.eq(wanted, got), name, wanted, got)
54	    return tally(String.eq(wanted, got), score)
55	
56	# one test against its own trailer
57	def one(+path: String, +output: String, score: Tally) -> IO(Tally):
58	  do IO<Tally>:
59	    src : Maybe<&2, String> <- F.read(path)
60	    check(path, G.trailer(F.text_of(src)), Q.filter(R.text(output)), score)
61	
62	# every test the answers ran out before. The two lists are the same length by
63	# construction and this should never have one to report, but the alternative to
64	# saying so is a green count over fewer tests than the tree holds, which is the
65	# worst answer a gate can give.
66	def short(ts: List<&2, String>, score: Tally) -> IO(Tally):
67	  match ts:
68	    case Nil{}:
69	      IO.pure(Tally, score)
70	    case Con{_h, rest}:
71	      do IO<Tally>:
72	        IO.write("FAIL: the run planned a test it never got an answer for\n")
73	        short(rest, tally(False{}, score))
74	
75	# every test, in the tree's order however the run happened to interleave
76	def report(ts: List<&2, String>, as: List<&2, String>, score: Tally) -> IO(Tally):
77	  match ts as:
78	    case Con{+h, tt} Con{+x, xt}:
79	      do IO<Tally>:
80	        next : Tally <- one(h, x, score)
81	        report(tt, xt, next)
82	    case rest _x:
83	      short(rest, score)
84	
85	# every test file
86	def find() -> IO(List<&2, String>):
87	  P.find.at("*/tests/*.bend", "-path")
88	
89	# the count, and the exit status it implies. The reminder sits above the count
90	# and never below it: a run's last line is its verdict, and something reads it.
91	def done.go(+passed: Nat, +total: Nat) -> IO(Unit):
92	  do IO<Unit>:
93	    IO.print("tests are a last resort for what you cannot prove")
94	    IO.print("PASS: " ++ Nat.show(passed) ++ " / " ++ Nat.show(total))
95	    Run.end(E.counted(passed, total))
96	
97	# what a whole run amounts to
98	def done(score: Tally) -> IO(Unit):
99	  Tally{p, n} = score
100	  done.go(p, n)
101	
102	# where the jobs of a run write what they print
103	def out.at() -> String:
104	  ".ez/run/test"
105	
106	# the budget checked the way every other thing here is checked. A run that
107	# overran is a run that failed, and it says so in the count rather than merely
108	# taking a long time.
109	def budget(late: Bool, +secs: String, score: Tally) -> IO(Tally):
110	  match late:
111	    case False{}:
112	      IO.pure(Tally, score)
113	    case True{}:
114	      do IO<Tally>:
115	        IO.write("FAIL: the gate took longer than its " ++ secs ++ " second " ++
116	          "budget\n  set EZ_DEADLINE higher if this tree honestly needs " ++
117	          "longer, or to 0 to stop asking\n")
118	        return tally(False{}, score)
119	
120	# `ez test`: every test in the tree, all of them at once. Nothing here is
121	# ordered against anything else, since each test has its own scratch directory
122	# and port, and the report is still walked in the tree's order whatever order
123	# the jobs finished in.
124	def run() -> IO(Unit):
125	  do IO<Unit>:
126	    +cap : Bool <- Cap.ok()
127	    Cap.warn(cap)
128	    +secs : String <- Timer.seconds()
129	    +deadline : String <- Timer.start(secs)
130	    +g : String <- Cap.gb()
131	    +at : String <- Env.lib()
132	    +jobs : String <- P.width()
133	    +ts : List<&2, String> <- find()
134	    P.fresh.dir(out.at())
135	    +by : String <- IO.pure(String, Clock.by.effect(deadline))
136	    as : List<&2, String> <- R.par(P.cmds(ts, cap, g, at), jobs, g, out.at(), by)
137	    every : Tally <- report(ts, as, Tally{0n, 0n})
138	    +late : Bool <- Timer.over(deadline)
139	    kept : Tally <- budget(late, secs, every)
140	    done(kept)