test_intel_child_lifecycle.py
python
sha256:4489fc9ba7823051e175113ea3b5d87046623736e8e7e2cf1ad1ddf8e9b88e48
test: test_n1b counts the unreaped original child by pid (p…
Sonnet 5.5
3 days ago
| 1 | """Intel child lifecycle — musehub#8 N1 (reference kept until reaped) and N2 (no idle child survives a job). |
| 2 | |
| 3 | Real ``IntelChild`` and real subprocesses (the protocol stub), no database. |
| 4 | |
| 5 | N2 An idle child of one worker process could overlap another process's prebuild: the serial barrier sees only *running* job rows. The child is therefore |
| 6 | one-shot: it is killed and reaped when its job ends, whatever the way it ends, so nothing is resident between jobs. |
| 7 | N1 ``kill()`` used to clear its process reference before the kill and swallow a failed ``wait()``, so a child that might still be alive was forgotten and |
| 8 | a second one could start beside it. The reference is now kept until the reap is confirmed, and a failed reap raises. |
| 9 | """ |
| 10 | from __future__ import annotations |
| 11 | |
| 12 | import asyncio |
| 13 | import sys |
| 14 | from pathlib import Path |
| 15 | |
| 16 | import psutil |
| 17 | import pytest |
| 18 | |
| 19 | from musehub import intel_supervisor as S |
| 20 | |
| 21 | STUB = str(Path(__file__).with_name("intel_child_stub.py")) |
| 22 | |
| 23 | |
| 24 | def child(**kw) -> S.IntelChild: |
| 25 | base = dict(memory_bytes=600 * 2**20, cpu_seconds=30, wall_seconds=20, poll_seconds=0.05) |
| 26 | base.update(kw) |
| 27 | return S.IntelChild(S.IntelChildLimits(**base), command=lambda: [sys.executable, "-P", STUB]) |
| 28 | |
| 29 | |
| 30 | def alive(pid: int) -> bool: |
| 31 | try: |
| 32 | return psutil.Process(pid).status() != psutil.STATUS_ZOMBIE |
| 33 | except psutil.NoSuchProcess: |
| 34 | return False |
| 35 | |
| 36 | |
| 37 | @pytest.mark.asyncio |
| 38 | async def test_n2a_no_child_is_resident_after_a_successful_job() -> None: |
| 39 | c = child() |
| 40 | try: |
| 41 | terminal, info = await c.run("ok", "repo", {}) |
| 42 | assert terminal == "done" |
| 43 | assert not alive(info["pid"]) and c.pid is None and c._proc is None |
| 44 | finally: |
| 45 | await c.close() |
| 46 | |
| 47 | |
| 48 | @pytest.mark.asyncio |
| 49 | async def test_n2b_no_child_is_resident_after_a_job_error(tmp_path, monkeypatch) -> None: |
| 50 | pid_file = tmp_path / "pid" |
| 51 | monkeypatch.setenv("STUB_PID_FILE", str(pid_file)) |
| 52 | c = child() |
| 53 | try: |
| 54 | with pytest.raises(S.IntelChildError, match="boom"): |
| 55 | await c.run("fail", "repo", {}) |
| 56 | assert not alive(int(pid_file.read_text())) and c.pid is None and c._proc is None |
| 57 | finally: |
| 58 | await c.close() |
| 59 | |
| 60 | |
| 61 | @pytest.mark.asyncio |
| 62 | async def test_n2c_consecutive_jobs_each_get_a_fresh_child_and_leave_none_behind() -> None: |
| 63 | c = child() |
| 64 | try: |
| 65 | _, first = await c.run("ok", "repo", {}) |
| 66 | _, second = await c.run("ok", "repo", {}) |
| 67 | assert first["pid"] != second["pid"] |
| 68 | assert not alive(first["pid"]) and not alive(second["pid"]) |
| 69 | finally: |
| 70 | await c.close() |
| 71 | |
| 72 | |
| 73 | @pytest.mark.asyncio |
| 74 | async def test_n2d_the_supervisor_has_no_idle_mode() -> None: |
| 75 | assert not hasattr(S.IntelChild, "reap_if_idle") |
| 76 | assert "idle_seconds" not in S.IntelChildLimits.__dataclass_fields__ |
| 77 | |
| 78 | |
| 79 | @pytest.mark.asyncio |
| 80 | async def test_n1a_a_failed_reap_keeps_the_reference_and_raises() -> None: |
| 81 | c = child() |
| 82 | proc = await c._ensure() |
| 83 | real_wait = proc.wait |
| 84 | |
| 85 | async def failing_wait() -> int: |
| 86 | raise OSError("cannot reap") |
| 87 | |
| 88 | proc.wait = failing_wait # type: ignore[method-assign] |
| 89 | try: |
| 90 | with pytest.raises(OSError, match="cannot reap"): |
| 91 | await c.kill() |
| 92 | assert c._proc is proc, "the reference to a child that may still be alive must survive a failed reap" |
| 93 | del proc.wait # the next attempt reaps for real |
| 94 | await c.kill() # retry works |
| 95 | assert c._proc is None and not alive(proc.pid) |
| 96 | finally: |
| 97 | proc.wait = real_wait # type: ignore[method-assign] |
| 98 | await c.close() |
| 99 | |
| 100 | |
| 101 | @pytest.mark.asyncio |
| 102 | async def test_n1b_no_second_child_starts_beside_one_that_could_not_be_reaped(tmp_path, monkeypatch) -> None: |
| 103 | c = child() |
| 104 | proc = await c._ensure() |
| 105 | |
| 106 | async def failing_wait() -> int: |
| 107 | raise OSError("cannot reap") |
| 108 | |
| 109 | proc.wait = failing_wait # type: ignore[method-assign] |
| 110 | try: |
| 111 | with pytest.raises(OSError, match="cannot reap"): |
| 112 | await c.run("ok", "repo", {}) # the job's own end-of-job reap fails... or the pre-spawn cleanup does; either way no second child |
| 113 | assert c._proc is proc |
| 114 | children = [] |
| 115 | for p in psutil.Process().children(): |
| 116 | try: |
| 117 | is_stub = STUB in p.cmdline() |
| 118 | except psutil.ZombieProcess: |
| 119 | # The unreaped original exits into a zombie, which has no readable command line (racy: it may |
| 120 | # exit between children() and cmdline()). Identify it by pid so it still counts as the one child. |
| 121 | is_stub = p.pid == proc.pid |
| 122 | except psutil.NoSuchProcess: |
| 123 | continue |
| 124 | if is_stub: |
| 125 | children.append(p) |
| 126 | assert len(children) <= 1, "a second child was spawned beside the unreaped one" |
| 127 | finally: |
| 128 | del proc.wait |
| 129 | await c.close() |
| 130 | assert not alive(proc.pid) |
| 131 | |
| 132 | |
| 133 | @pytest.mark.asyncio |
| 134 | async def test_n1c_cancellation_still_ends_and_reaps_the_child() -> None: |
| 135 | c = child() |
| 136 | task = asyncio.create_task(c.run("hang", "repo", {})) |
| 137 | await asyncio.sleep(0.5) |
| 138 | pid = c._proc.pid |
| 139 | task.cancel() |
| 140 | with pytest.raises(asyncio.CancelledError): |
| 141 | await task |
| 142 | assert not alive(pid) and c._proc is None |
| 143 | |
| 144 | |
| 145 | @pytest.mark.asyncio |
| 146 | async def test_n1d_a_reap_that_hangs_times_out_with_a_named_failure_and_keeps_the_reference(monkeypatch) -> None: |
| 147 | monkeypatch.setattr(S, "_REAP_TIMEOUT_SECONDS", 0.5) |
| 148 | c = child() |
| 149 | proc = await c._ensure() |
| 150 | real_wait = proc.wait |
| 151 | |
| 152 | async def hanging_wait() -> int: |
| 153 | await asyncio.sleep(3600) |
| 154 | return 0 |
| 155 | |
| 156 | proc.wait = hanging_wait # type: ignore[method-assign] |
| 157 | try: |
| 158 | with pytest.raises(S.IntelChildFailure, match=f"pid={proc.pid}"): |
| 159 | await c.kill() |
| 160 | assert c._proc is proc |
| 161 | del proc.wait |
| 162 | await c.kill() |
| 163 | assert c._proc is None and not alive(proc.pid) |
| 164 | finally: |
| 165 | proc.wait = real_wait # type: ignore[method-assign] |
| 166 | await c.close() |
| 167 | |
| 168 | |
| 169 | @pytest.mark.asyncio |
| 170 | async def test_n1e_using_a_supervisor_with_a_live_child_from_another_loop_fails_closed() -> None: |
| 171 | c = child() |
| 172 | proc = await c._ensure() |
| 173 | try: |
| 174 | c._loop = object() # type: ignore[assignment] # as if spawned by a different event loop |
| 175 | with pytest.raises(RuntimeError, match="different event loop"): |
| 176 | await c.kill() |
| 177 | assert c._proc is proc and alive(proc.pid), "a live child must be neither killed blind nor forgotten" |
| 178 | finally: |
| 179 | c._loop = asyncio.get_running_loop() |
| 180 | await c.close() |
File History
3 commits
sha256:4489fc9ba7823051e175113ea3b5d87046623736e8e7e2cf1ad1ddf8e9b88e48
test: test_n1b counts the unreaped original child by pid (p…
Sonnet 5.5
3 days ago
sha256:c3e18b15a2c0061e5bf0cf1130ec534cfad3582e0abaf5750ae68b8093a1258d
fix(intel): head-keyed rebuild dedup, named reap failure, f…
Sonnet 5.5
minor
⚠
3 days ago
sha256:3733f942311ecd1beecb0d6f112f3ebd87b2801a5a410bf35b2415a84a238f1b
fix(intel): one-shot child, reap confirmed before the refer…
Sonnet 5.5
minor
⚠
3 days ago