test_intel_outcome_reporting.py
python
sha256:ad83bfa8d46d529c26bc603709ff7471ac571b967e87521eb0dc4d662a78a2a1
fix(intel): the run-all report classifies every step instea…
Sonnet 5.5
patch
4 days ago
| 1 | """The run-all report must say what each step actually was — musehub#8 release gate (empty outcomes). |
| 2 | |
| 3 | A step that finished ``done`` with outcome ``empty`` (nothing could be analysed, nothing published) was reported as OK because only ``status == "done"`` was |
| 4 | checked. Persistence, job status and report now agree on five distinct things: ok · empty · skipped · blocked · failed (plus a truncation flag on an ok result). |
| 5 | |
| 6 | Run (no database):: python -P -m pytest tests/test_intel_outcome_reporting.py |
| 7 | """ |
| 8 | from __future__ import annotations |
| 9 | |
| 10 | import pytest |
| 11 | |
| 12 | from musehub.intel_experiment import classify_step, summarise_steps |
| 13 | |
| 14 | |
| 15 | @pytest.mark.parametrize("step,verdict", [ |
| 16 | ({"status": "done", "outcome": "computed"}, "ok"), |
| 17 | ({"status": "done", "outcome": "already_current"}, "ok"), |
| 18 | ({"status": "done", "outcome": "empty"}, "empty"), |
| 19 | ({"status": "done", "outcome": "stale"}, "stale"), |
| 20 | ({"status": "done"}, "unknown"), # a done step that does not say what it did is never "ok" |
| 21 | ({"status": "skipped", "reason": "superseded"}, "skipped"), |
| 22 | ({"status": "blocked", "reason": "index not current"}, "blocked"), |
| 23 | ({"status": "failed", "error": "boom"}, "failed"), |
| 24 | ({"status": "not_run"}, "not_run"), |
| 25 | ]) |
| 26 | def test_each_step_is_classified_by_what_it_was(step, verdict) -> None: |
| 27 | assert classify_step(step) == verdict |
| 28 | |
| 29 | |
| 30 | def test_the_report_is_ok_only_when_every_step_is_ok_and_names_the_rest() -> None: |
| 31 | steps = [ |
| 32 | {"provider": "intel.code", "status": "done", "outcome": "computed"}, |
| 33 | {"provider": "intel.code.dead", "status": "done", "outcome": "empty"}, |
| 34 | {"provider": "intel.code.type", "status": "blocked", "reason": "x"}, |
| 35 | ] |
| 36 | published = {"intel.code.coupling": {"data": {"count": 3, "walk_truncated": True}}, "intel.code.type": {"data": {"count": 1, "truncated": False}}} |
| 37 | summary = summarise_steps(steps, published) |
| 38 | assert summary["ok"] is False |
| 39 | assert summary["not_ok"] == {"intel.code.dead": "empty", "intel.code.type": "blocked"} |
| 40 | assert summary["truncated"] == ["intel.code.coupling"] |
| 41 | |
| 42 | |
| 43 | def test_an_all_ok_run_with_a_truncated_result_is_ok_but_says_truncated() -> None: |
| 44 | steps = [{"provider": "intel.code.coupling", "status": "done", "outcome": "computed"}] |
| 45 | summary = summarise_steps(steps, {"intel.code.coupling": {"data": {"truncated": True}}}) |
| 46 | assert summary["ok"] is True and summary["not_ok"] == {} and summary["truncated"] == ["intel.code.coupling"] |
File History
1 commit
sha256:ad83bfa8d46d529c26bc603709ff7471ac571b967e87521eb0dc4d662a78a2a1
fix(intel): the run-all report classifies every step instea…
Sonnet 5.5
patch
4 days ago