gabriel / musehub public
test_intel_outcome_reporting.py python
46 lines 2.4 KB
Raw
sha256:ad83bfa8d46d529c26bc603709ff7471ac571b967e87521eb0dc4d662a78a2a1 fix(intel): the run-all report classifies every step instea… Sonnet 5.5 patch 4 days ago
1 """The run-all report must say what each step actually was — musehub#8 release gate (empty outcomes).
2
3 A step that finished ``done`` with outcome ``empty`` (nothing could be analysed, nothing published) was reported as OK because only ``status == "done"`` was
4 checked. Persistence, job status and report now agree on five distinct things: ok · empty · skipped · blocked · failed (plus a truncation flag on an ok result).
5
6 Run (no database):: python -P -m pytest tests/test_intel_outcome_reporting.py
7 """
8 from __future__ import annotations
9
10 import pytest
11
12 from musehub.intel_experiment import classify_step, summarise_steps
13
14
15 @pytest.mark.parametrize("step,verdict", [
16 ({"status": "done", "outcome": "computed"}, "ok"),
17 ({"status": "done", "outcome": "already_current"}, "ok"),
18 ({"status": "done", "outcome": "empty"}, "empty"),
19 ({"status": "done", "outcome": "stale"}, "stale"),
20 ({"status": "done"}, "unknown"), # a done step that does not say what it did is never "ok"
21 ({"status": "skipped", "reason": "superseded"}, "skipped"),
22 ({"status": "blocked", "reason": "index not current"}, "blocked"),
23 ({"status": "failed", "error": "boom"}, "failed"),
24 ({"status": "not_run"}, "not_run"),
25 ])
26 def test_each_step_is_classified_by_what_it_was(step, verdict) -> None:
27 assert classify_step(step) == verdict
28
29
30 def test_the_report_is_ok_only_when_every_step_is_ok_and_names_the_rest() -> None:
31 steps = [
32 {"provider": "intel.code", "status": "done", "outcome": "computed"},
33 {"provider": "intel.code.dead", "status": "done", "outcome": "empty"},
34 {"provider": "intel.code.type", "status": "blocked", "reason": "x"},
35 ]
36 published = {"intel.code.coupling": {"data": {"count": 3, "walk_truncated": True}}, "intel.code.type": {"data": {"count": 1, "truncated": False}}}
37 summary = summarise_steps(steps, published)
38 assert summary["ok"] is False
39 assert summary["not_ok"] == {"intel.code.dead": "empty", "intel.code.type": "blocked"}
40 assert summary["truncated"] == ["intel.code.coupling"]
41
42
43 def test_an_all_ok_run_with_a_truncated_result_is_ok_but_says_truncated() -> None:
44 steps = [{"provider": "intel.code.coupling", "status": "done", "outcome": "computed"}]
45 summary = summarise_steps(steps, {"intel.code.coupling": {"data": {"truncated": True}}})
46 assert summary["ok"] is True and summary["not_ok"] == {} and summary["truncated"] == ["intel.code.coupling"]
File History 1 commit
sha256:ad83bfa8d46d529c26bc603709ff7471ac571b967e87521eb0dc4d662a78a2a1 fix(intel): the run-all report classifies every step instea… Sonnet 5.5 patch 4 days ago