test_core_test_history.py
python
sha256:51ce277f663e01a43eaffbe77509b1de7ac2d4251b55d23306304bcdeb92c90d
feat(pack): delta-encode snapshots in MPackBundle wire format
Sonnet 4.6
minor
⚠ breaking
120 days ago
| 1 | """Tests for muse.core.test_history — persistent test-run history. |
| 2 | |
| 3 | Coverage: |
| 4 | - Unit tests for serialisation (_record_to_msgpack / _record_from_msgpack). |
| 5 | - Round-trip tests: save + load round-trips for RunRecord. |
| 6 | - load_history returns empty list when file missing. |
| 7 | - append_run adds one record. |
| 8 | - summarize computes correct counts, flaky flag, and fail_streak. |
| 9 | - flaky_tests returns only flaky tests, sorted by fail_count. |
| 10 | - prioritize_targets puts streaky/flaky tests first. |
| 11 | - Corrupt file handling: load_history returns empty list on corruption. |
| 12 | - iso_now returns a valid ISO 8601 string. |
| 13 | - make_run_id returns a unique UUID. |
| 14 | """ |
| 15 | |
| 16 | from __future__ import annotations |
| 17 | |
| 18 | import pathlib |
| 19 | |
| 20 | import pytest |
| 21 | |
| 22 | from muse.core.paths import muse_dir, test_history_path as _test_history_path |
| 23 | from muse.core.test_history import ( |
| 24 | HistorySummary, |
| 25 | RunRecord, |
| 26 | CaseRecord, |
| 27 | _record_from_msgpack, |
| 28 | _record_to_msgpack, |
| 29 | append_run, |
| 30 | flaky_tests, |
| 31 | iso_now, |
| 32 | load_history, |
| 33 | make_run_id, |
| 34 | prioritize_targets, |
| 35 | save_history, |
| 36 | summarize, |
| 37 | ) |
| 38 | |
| 39 | |
| 40 | # --------------------------------------------------------------------------- |
| 41 | # Fixtures |
| 42 | # --------------------------------------------------------------------------- |
| 43 | |
| 44 | |
| 45 | def _make_record( |
| 46 | run_id: str = "run-1", |
| 47 | *, |
| 48 | passed: int = 2, |
| 49 | failed: int = 0, |
| 50 | results: list[CaseRecord] | None = None, |
| 51 | ) -> RunRecord: |
| 52 | if results is None: |
| 53 | results = [ |
| 54 | CaseRecord( |
| 55 | node_id="tests/test_foo.py::test_a", |
| 56 | outcome="passed", |
| 57 | duration_ms=10.0, |
| 58 | symbol_addresses=[], |
| 59 | ), |
| 60 | CaseRecord( |
| 61 | node_id="tests/test_foo.py::test_b", |
| 62 | outcome="passed", |
| 63 | duration_ms=20.0, |
| 64 | symbol_addresses=[], |
| 65 | ), |
| 66 | ] |
| 67 | return RunRecord( |
| 68 | run_id=run_id, |
| 69 | timestamp="2026-03-26T12:00:00Z", |
| 70 | commit_id="abc123", |
| 71 | branch="main", |
| 72 | results=results, |
| 73 | total=len(results), |
| 74 | passed=passed, |
| 75 | failed=failed, |
| 76 | errored=0, |
| 77 | skipped=0, |
| 78 | ) |
| 79 | |
| 80 | |
| 81 | # --------------------------------------------------------------------------- |
| 82 | # Unit tests — serialisation |
| 83 | # --------------------------------------------------------------------------- |
| 84 | |
| 85 | |
| 86 | class TestRecordSerialization: |
| 87 | def test_round_trip(self) -> None: |
| 88 | """A RunRecord serialises and deserialises back to an equal value.""" |
| 89 | import msgpack |
| 90 | record = _make_record() |
| 91 | # Round-trip through msgpack bytes to get a MsgpackValue for _record_from_msgpack. |
| 92 | doc = _record_to_msgpack(record) |
| 93 | raw_bytes = msgpack.packb(doc, use_bin_type=True) |
| 94 | raw_value = msgpack.unpackb(raw_bytes, raw=False) |
| 95 | restored = _record_from_msgpack(raw_value) |
| 96 | assert restored is not None |
| 97 | assert restored["run_id"] == record["run_id"] |
| 98 | assert restored["timestamp"] == record["timestamp"] |
| 99 | assert restored["commit_id"] == record["commit_id"] |
| 100 | assert restored["branch"] == record["branch"] |
| 101 | assert restored["total"] == record["total"] |
| 102 | assert restored["passed"] == record["passed"] |
| 103 | assert len(restored["results"]) == len(record["results"]) |
| 104 | |
| 105 | def test_longrepr_round_trip(self) -> None: |
| 106 | """longrepr is preserved across serialisation.""" |
| 107 | import msgpack as _msgpack |
| 108 | result = CaseRecord( |
| 109 | node_id="tests/test_foo.py::test_fail", |
| 110 | outcome="failed", |
| 111 | duration_ms=5.0, |
| 112 | symbol_addresses=[], |
| 113 | ) |
| 114 | result["longrepr"] = "AssertionError: expected 1, got 2" |
| 115 | |
| 116 | record = _make_record( |
| 117 | failed=1, passed=0, results=[result] |
| 118 | ) |
| 119 | doc = _record_to_msgpack(record) |
| 120 | raw_bytes = _msgpack.packb(doc, use_bin_type=True) |
| 121 | raw_value = _msgpack.unpackb(raw_bytes, raw=False) |
| 122 | restored = _record_from_msgpack(raw_value) |
| 123 | assert restored is not None |
| 124 | restored_result = restored["results"][0] |
| 125 | assert restored_result.get("longrepr") == "AssertionError: expected 1, got 2" |
| 126 | |
| 127 | def test_none_fields_preserved(self) -> None: |
| 128 | """commit_id=None and branch=None survive round-trip.""" |
| 129 | import msgpack as _msgpack |
| 130 | record = _make_record() |
| 131 | record["commit_id"] = None |
| 132 | record["branch"] = None |
| 133 | doc = _record_to_msgpack(record) |
| 134 | raw_bytes = _msgpack.packb(doc, use_bin_type=True) |
| 135 | raw_value = _msgpack.unpackb(raw_bytes, raw=False) |
| 136 | restored = _record_from_msgpack(raw_value) |
| 137 | assert restored is not None |
| 138 | assert restored["commit_id"] is None |
| 139 | assert restored["branch"] is None |
| 140 | |
| 141 | def test_invalid_input_returns_none(self) -> None: |
| 142 | """_record_from_msgpack returns None for non-dict input.""" |
| 143 | assert _record_from_msgpack("not a dict") is None |
| 144 | assert _record_from_msgpack([]) is None |
| 145 | assert _record_from_msgpack(None) is None |
| 146 | |
| 147 | |
| 148 | # --------------------------------------------------------------------------- |
| 149 | # I/O tests — load_history / save_history / append_run |
| 150 | # --------------------------------------------------------------------------- |
| 151 | |
| 152 | |
| 153 | class TestLoadSave: |
| 154 | def test_load_missing_file(self, tmp_path: pathlib.Path) -> None: |
| 155 | """load_history returns [] when history file does not exist.""" |
| 156 | muse_dir(tmp_path).mkdir() |
| 157 | records = load_history(tmp_path) |
| 158 | assert records == [] |
| 159 | |
| 160 | def test_save_and_load(self, tmp_path: pathlib.Path) -> None: |
| 161 | """save_history + load_history is a faithful round-trip.""" |
| 162 | muse_dir(tmp_path).mkdir() |
| 163 | rec1 = _make_record("r1") |
| 164 | rec2 = _make_record("r2", passed=1, failed=1) |
| 165 | save_history(tmp_path, [rec1, rec2]) |
| 166 | loaded = load_history(tmp_path) |
| 167 | assert len(loaded) == 2 |
| 168 | assert loaded[0]["run_id"] == "r1" |
| 169 | assert loaded[1]["run_id"] == "r2" |
| 170 | |
| 171 | def test_append_run(self, tmp_path: pathlib.Path) -> None: |
| 172 | """append_run adds one record to the history.""" |
| 173 | muse_dir(tmp_path).mkdir() |
| 174 | save_history(tmp_path, [_make_record("r1")]) |
| 175 | append_run(tmp_path, _make_record("r2")) |
| 176 | loaded = load_history(tmp_path) |
| 177 | assert len(loaded) == 2 |
| 178 | assert loaded[-1]["run_id"] == "r2" |
| 179 | |
| 180 | def test_load_corrupt_file_returns_empty(self, tmp_path: pathlib.Path) -> None: |
| 181 | """Corrupt msgpack file returns empty list without raising.""" |
| 182 | hist_path = _test_history_path(tmp_path) |
| 183 | hist_path.parent.mkdir(parents=True, exist_ok=True) |
| 184 | hist_path.write_bytes(b"\xff\xfe garbage bytes that are not valid msgpack") |
| 185 | records = load_history(tmp_path) |
| 186 | assert records == [] |
| 187 | |
| 188 | def test_atomic_write(self, tmp_path: pathlib.Path) -> None: |
| 189 | """save_history writes to a .tmp file then renames (no partial writes).""" |
| 190 | muse_dir(tmp_path).mkdir() |
| 191 | save_history(tmp_path, [_make_record()]) |
| 192 | tmp_files = list(muse_dir(tmp_path).glob("*.tmp")) |
| 193 | assert tmp_files == [], "Temp file should be removed after atomic write" |
| 194 | |
| 195 | |
| 196 | # --------------------------------------------------------------------------- |
| 197 | # Analytics — summarize |
| 198 | # --------------------------------------------------------------------------- |
| 199 | |
| 200 | |
| 201 | class TestSummarize: |
| 202 | def test_empty_records(self) -> None: |
| 203 | """summarize returns empty dict for empty input.""" |
| 204 | assert summarize([]) == {} |
| 205 | |
| 206 | def test_all_passed(self) -> None: |
| 207 | """All-pass history: pass_count = total_runs, fail_count = 0.""" |
| 208 | results = [ |
| 209 | CaseRecord( |
| 210 | node_id="tests/test_foo.py::test_a", |
| 211 | outcome="passed", |
| 212 | duration_ms=10.0, |
| 213 | symbol_addresses=[], |
| 214 | ) |
| 215 | ] |
| 216 | record = _make_record(passed=1, failed=0, results=results) |
| 217 | sums = summarize([record]) |
| 218 | s = sums["tests/test_foo.py::test_a"] |
| 219 | assert s["pass_count"] == 1 |
| 220 | assert s["fail_count"] == 0 |
| 221 | assert s["flaky"] is False |
| 222 | assert s["fail_streak"] == 0 |
| 223 | assert s["last_outcome"] == "passed" |
| 224 | |
| 225 | def test_all_failed(self) -> None: |
| 226 | """All-fail history: fail_count = total_runs, fail_streak = total_runs.""" |
| 227 | results = [ |
| 228 | CaseRecord( |
| 229 | node_id="tests/test_foo.py::test_a", |
| 230 | outcome="failed", |
| 231 | duration_ms=5.0, |
| 232 | symbol_addresses=[], |
| 233 | ) |
| 234 | ] |
| 235 | records = [ |
| 236 | RunRecord( |
| 237 | run_id=f"r{i}", |
| 238 | timestamp=f"2026-03-{i+1:02d}T00:00:00Z", |
| 239 | commit_id=None, |
| 240 | branch=None, |
| 241 | results=results, |
| 242 | total=1, |
| 243 | passed=0, |
| 244 | failed=1, |
| 245 | errored=0, |
| 246 | skipped=0, |
| 247 | ) |
| 248 | for i in range(3) |
| 249 | ] |
| 250 | sums = summarize(records) |
| 251 | s = sums["tests/test_foo.py::test_a"] |
| 252 | assert s["fail_count"] == 3 |
| 253 | assert s["pass_count"] == 0 |
| 254 | assert s["flaky"] is False |
| 255 | assert s["fail_streak"] == 3 |
| 256 | |
| 257 | def test_flaky_detection(self) -> None: |
| 258 | """A test that both passes and fails is flagged as flaky.""" |
| 259 | pass_res = CaseRecord( |
| 260 | node_id="tests/test_foo.py::test_flaky", |
| 261 | outcome="passed", |
| 262 | duration_ms=10.0, |
| 263 | symbol_addresses=[], |
| 264 | ) |
| 265 | fail_res = CaseRecord( |
| 266 | node_id="tests/test_foo.py::test_flaky", |
| 267 | outcome="failed", |
| 268 | duration_ms=10.0, |
| 269 | symbol_addresses=[], |
| 270 | ) |
| 271 | records = [ |
| 272 | _make_record("r1", passed=1, failed=0, results=[pass_res]), |
| 273 | _make_record("r2", passed=0, failed=1, results=[fail_res]), |
| 274 | ] |
| 275 | sums = summarize(records) |
| 276 | s = sums["tests/test_foo.py::test_flaky"] |
| 277 | assert s["flaky"] is True |
| 278 | assert s["pass_count"] == 1 |
| 279 | assert s["fail_count"] == 1 |
| 280 | |
| 281 | def test_fail_streak_stops_on_pass(self) -> None: |
| 282 | """fail_streak resets when the most recent run passes.""" |
| 283 | results_fail = [ |
| 284 | CaseRecord( |
| 285 | node_id="tests/t.py::test_x", |
| 286 | outcome="failed", |
| 287 | duration_ms=5.0, |
| 288 | symbol_addresses=[], |
| 289 | ) |
| 290 | ] |
| 291 | results_pass = [ |
| 292 | CaseRecord( |
| 293 | node_id="tests/t.py::test_x", |
| 294 | outcome="passed", |
| 295 | duration_ms=5.0, |
| 296 | symbol_addresses=[], |
| 297 | ) |
| 298 | ] |
| 299 | records = [ |
| 300 | _make_record("r1", passed=0, failed=1, results=results_fail), |
| 301 | _make_record("r2", passed=0, failed=1, results=results_fail), |
| 302 | _make_record("r3", passed=1, failed=0, results=results_pass), |
| 303 | ] |
| 304 | sums = summarize(records) |
| 305 | s = sums["tests/t.py::test_x"] |
| 306 | assert s["fail_streak"] == 0 # Most recent run passed. |
| 307 | |
| 308 | def test_avg_duration_excludes_skipped(self) -> None: |
| 309 | """avg_duration_ms excludes skipped tests from the mean.""" |
| 310 | results = [ |
| 311 | CaseRecord( |
| 312 | node_id="tests/t.py::test_x", |
| 313 | outcome="passed", |
| 314 | duration_ms=100.0, |
| 315 | symbol_addresses=[], |
| 316 | ), |
| 317 | CaseRecord( |
| 318 | node_id="tests/t.py::test_x", |
| 319 | outcome="skipped", |
| 320 | duration_ms=0.0, |
| 321 | symbol_addresses=[], |
| 322 | ), |
| 323 | ] |
| 324 | records = [ |
| 325 | _make_record("r1", passed=1, results=[results[0]]), |
| 326 | _make_record("r2", passed=0, results=[results[1]]), |
| 327 | ] |
| 328 | sums = summarize(records) |
| 329 | s = sums["tests/t.py::test_x"] |
| 330 | assert s["avg_duration_ms"] == 100.0 |
| 331 | |
| 332 | |
| 333 | # --------------------------------------------------------------------------- |
| 334 | # Analytics — flaky_tests |
| 335 | # --------------------------------------------------------------------------- |
| 336 | |
| 337 | |
| 338 | class TestFlakyTests: |
| 339 | def test_returns_only_flaky(self) -> None: |
| 340 | """flaky_tests returns only tests with both passes and failures.""" |
| 341 | pass_res = CaseRecord( |
| 342 | node_id="tests/t.py::test_stable", |
| 343 | outcome="passed", |
| 344 | duration_ms=10.0, |
| 345 | symbol_addresses=[], |
| 346 | ) |
| 347 | flaky_res_pass = CaseRecord( |
| 348 | node_id="tests/t.py::test_flaky", |
| 349 | outcome="passed", |
| 350 | duration_ms=10.0, |
| 351 | symbol_addresses=[], |
| 352 | ) |
| 353 | flaky_res_fail = CaseRecord( |
| 354 | node_id="tests/t.py::test_flaky", |
| 355 | outcome="failed", |
| 356 | duration_ms=10.0, |
| 357 | symbol_addresses=[], |
| 358 | ) |
| 359 | records = [ |
| 360 | _make_record("r1", passed=2, results=[pass_res, flaky_res_pass]), |
| 361 | _make_record("r2", passed=1, failed=1, results=[pass_res, flaky_res_fail]), |
| 362 | ] |
| 363 | result = flaky_tests(records) |
| 364 | node_ids = {s["node_id"] for s in result} |
| 365 | assert "tests/t.py::test_flaky" in node_ids |
| 366 | assert "tests/t.py::test_stable" not in node_ids |
| 367 | |
| 368 | def test_empty_returns_empty(self) -> None: |
| 369 | assert flaky_tests([]) == [] |
| 370 | |
| 371 | |
| 372 | # --------------------------------------------------------------------------- |
| 373 | # Analytics — prioritize_targets |
| 374 | # --------------------------------------------------------------------------- |
| 375 | |
| 376 | |
| 377 | class TestPrioritizeTargets: |
| 378 | def test_unknown_targets_returned_in_some_order(self) -> None: |
| 379 | """Unknown targets (not in history) are returned (order unspecified).""" |
| 380 | targets = ["tests/t.py::test_a", "tests/t.py::test_b"] |
| 381 | result = prioritize_targets(targets, []) |
| 382 | assert sorted(result) == sorted(targets) |
| 383 | |
| 384 | def test_streaky_test_comes_first(self) -> None: |
| 385 | """A test with a recent failure streak is sorted before stable tests.""" |
| 386 | fail_res = CaseRecord( |
| 387 | node_id="tests/t.py::test_fail", |
| 388 | outcome="failed", |
| 389 | duration_ms=5.0, |
| 390 | symbol_addresses=[], |
| 391 | ) |
| 392 | pass_res = CaseRecord( |
| 393 | node_id="tests/t.py::test_pass", |
| 394 | outcome="passed", |
| 395 | duration_ms=5.0, |
| 396 | symbol_addresses=[], |
| 397 | ) |
| 398 | records = [ |
| 399 | _make_record("r1", passed=0, failed=1, results=[fail_res]), |
| 400 | _make_record("r2", passed=1, failed=0, results=[pass_res]), |
| 401 | ] |
| 402 | targets = ["tests/t.py::test_pass", "tests/t.py::test_fail"] |
| 403 | ordered = prioritize_targets(targets, records) |
| 404 | assert ordered[0] == "tests/t.py::test_fail" |
| 405 | |
| 406 | def test_empty_targets(self) -> None: |
| 407 | assert prioritize_targets([], []) == [] |
| 408 | |
| 409 | |
| 410 | # --------------------------------------------------------------------------- |
| 411 | # Utilities |
| 412 | # --------------------------------------------------------------------------- |
| 413 | |
| 414 | |
| 415 | class TestUtilities: |
| 416 | def test_iso_now_format(self) -> None: |
| 417 | """iso_now returns an ISO 8601 UTC string.""" |
| 418 | ts = iso_now() |
| 419 | assert "T" in ts |
| 420 | assert ts.endswith("Z") |
| 421 | assert len(ts) == 20 # "YYYY-MM-DDTHH:MM:SSZ" |
| 422 | |
| 423 | def test_make_run_id_is_unique(self) -> None: |
| 424 | """make_run_id returns a different sha256: ID each time.""" |
| 425 | ids = {make_run_id() for _ in range(100)} |
| 426 | assert len(ids) == 100 |
| 427 | |
| 428 | def test_make_run_id_is_sha256(self) -> None: |
| 429 | """make_run_id returns a sha256: content-addressed ID.""" |
| 430 | run_id = make_run_id() |
| 431 | assert run_id.startswith("sha256:"), f"expected sha256: prefix, got {run_id!r}" |
| 432 | assert len(run_id) == 71 |
File History
1 commit
sha256:51ce277f663e01a43eaffbe77509b1de7ac2d4251b55d23306304bcdeb92c90d
feat(pack): delta-encode snapshots in MPackBundle wire format
Sonnet 4.6
minor
⚠
120 days ago