gabriel / musehub public
test_intel_child_lifecycle.py python
180 lines 6.3 KB
Raw
sha256:4489fc9ba7823051e175113ea3b5d87046623736e8e7e2cf1ad1ddf8e9b88e48 test: test_n1b counts the unreaped original child by pid (p… Sonnet 5.5 3 days ago
1 """Intel child lifecycle — musehub#8 N1 (reference kept until reaped) and N2 (no idle child survives a job).
2
3 Real ``IntelChild`` and real subprocesses (the protocol stub), no database.
4
5 N2 An idle child of one worker process could overlap another process's prebuild: the serial barrier sees only *running* job rows. The child is therefore
6 one-shot: it is killed and reaped when its job ends, whatever the way it ends, so nothing is resident between jobs.
7 N1 ``kill()`` used to clear its process reference before the kill and swallow a failed ``wait()``, so a child that might still be alive was forgotten and
8 a second one could start beside it. The reference is now kept until the reap is confirmed, and a failed reap raises.
9 """
10 from __future__ import annotations
11
12 import asyncio
13 import sys
14 from pathlib import Path
15
16 import psutil
17 import pytest
18
19 from musehub import intel_supervisor as S
20
21 STUB = str(Path(__file__).with_name("intel_child_stub.py"))
22
23
24 def child(**kw) -> S.IntelChild:
25 base = dict(memory_bytes=600 * 2**20, cpu_seconds=30, wall_seconds=20, poll_seconds=0.05)
26 base.update(kw)
27 return S.IntelChild(S.IntelChildLimits(**base), command=lambda: [sys.executable, "-P", STUB])
28
29
30 def alive(pid: int) -> bool:
31 try:
32 return psutil.Process(pid).status() != psutil.STATUS_ZOMBIE
33 except psutil.NoSuchProcess:
34 return False
35
36
37 @pytest.mark.asyncio
38 async def test_n2a_no_child_is_resident_after_a_successful_job() -> None:
39 c = child()
40 try:
41 terminal, info = await c.run("ok", "repo", {})
42 assert terminal == "done"
43 assert not alive(info["pid"]) and c.pid is None and c._proc is None
44 finally:
45 await c.close()
46
47
48 @pytest.mark.asyncio
49 async def test_n2b_no_child_is_resident_after_a_job_error(tmp_path, monkeypatch) -> None:
50 pid_file = tmp_path / "pid"
51 monkeypatch.setenv("STUB_PID_FILE", str(pid_file))
52 c = child()
53 try:
54 with pytest.raises(S.IntelChildError, match="boom"):
55 await c.run("fail", "repo", {})
56 assert not alive(int(pid_file.read_text())) and c.pid is None and c._proc is None
57 finally:
58 await c.close()
59
60
61 @pytest.mark.asyncio
62 async def test_n2c_consecutive_jobs_each_get_a_fresh_child_and_leave_none_behind() -> None:
63 c = child()
64 try:
65 _, first = await c.run("ok", "repo", {})
66 _, second = await c.run("ok", "repo", {})
67 assert first["pid"] != second["pid"]
68 assert not alive(first["pid"]) and not alive(second["pid"])
69 finally:
70 await c.close()
71
72
73 @pytest.mark.asyncio
74 async def test_n2d_the_supervisor_has_no_idle_mode() -> None:
75 assert not hasattr(S.IntelChild, "reap_if_idle")
76 assert "idle_seconds" not in S.IntelChildLimits.__dataclass_fields__
77
78
79 @pytest.mark.asyncio
80 async def test_n1a_a_failed_reap_keeps_the_reference_and_raises() -> None:
81 c = child()
82 proc = await c._ensure()
83 real_wait = proc.wait
84
85 async def failing_wait() -> int:
86 raise OSError("cannot reap")
87
88 proc.wait = failing_wait # type: ignore[method-assign]
89 try:
90 with pytest.raises(OSError, match="cannot reap"):
91 await c.kill()
92 assert c._proc is proc, "the reference to a child that may still be alive must survive a failed reap"
93 del proc.wait # the next attempt reaps for real
94 await c.kill() # retry works
95 assert c._proc is None and not alive(proc.pid)
96 finally:
97 proc.wait = real_wait # type: ignore[method-assign]
98 await c.close()
99
100
101 @pytest.mark.asyncio
102 async def test_n1b_no_second_child_starts_beside_one_that_could_not_be_reaped(tmp_path, monkeypatch) -> None:
103 c = child()
104 proc = await c._ensure()
105
106 async def failing_wait() -> int:
107 raise OSError("cannot reap")
108
109 proc.wait = failing_wait # type: ignore[method-assign]
110 try:
111 with pytest.raises(OSError, match="cannot reap"):
112 await c.run("ok", "repo", {}) # the job's own end-of-job reap fails... or the pre-spawn cleanup does; either way no second child
113 assert c._proc is proc
114 children = []
115 for p in psutil.Process().children():
116 try:
117 is_stub = STUB in p.cmdline()
118 except psutil.ZombieProcess:
119 # The unreaped original exits into a zombie, which has no readable command line (racy: it may
120 # exit between children() and cmdline()). Identify it by pid so it still counts as the one child.
121 is_stub = p.pid == proc.pid
122 except psutil.NoSuchProcess:
123 continue
124 if is_stub:
125 children.append(p)
126 assert len(children) <= 1, "a second child was spawned beside the unreaped one"
127 finally:
128 del proc.wait
129 await c.close()
130 assert not alive(proc.pid)
131
132
133 @pytest.mark.asyncio
134 async def test_n1c_cancellation_still_ends_and_reaps_the_child() -> None:
135 c = child()
136 task = asyncio.create_task(c.run("hang", "repo", {}))
137 await asyncio.sleep(0.5)
138 pid = c._proc.pid
139 task.cancel()
140 with pytest.raises(asyncio.CancelledError):
141 await task
142 assert not alive(pid) and c._proc is None
143
144
145 @pytest.mark.asyncio
146 async def test_n1d_a_reap_that_hangs_times_out_with_a_named_failure_and_keeps_the_reference(monkeypatch) -> None:
147 monkeypatch.setattr(S, "_REAP_TIMEOUT_SECONDS", 0.5)
148 c = child()
149 proc = await c._ensure()
150 real_wait = proc.wait
151
152 async def hanging_wait() -> int:
153 await asyncio.sleep(3600)
154 return 0
155
156 proc.wait = hanging_wait # type: ignore[method-assign]
157 try:
158 with pytest.raises(S.IntelChildFailure, match=f"pid={proc.pid}"):
159 await c.kill()
160 assert c._proc is proc
161 del proc.wait
162 await c.kill()
163 assert c._proc is None and not alive(proc.pid)
164 finally:
165 proc.wait = real_wait # type: ignore[method-assign]
166 await c.close()
167
168
169 @pytest.mark.asyncio
170 async def test_n1e_using_a_supervisor_with_a_live_child_from_another_loop_fails_closed() -> None:
171 c = child()
172 proc = await c._ensure()
173 try:
174 c._loop = object() # type: ignore[assignment] # as if spawned by a different event loop
175 with pytest.raises(RuntimeError, match="different event loop"):
176 await c.kill()
177 assert c._proc is proc and alive(proc.pid), "a live child must be neither killed blind nor forgotten"
178 finally:
179 c._loop = asyncio.get_running_loop()
180 await c.close()
File History 3 commits
sha256:4489fc9ba7823051e175113ea3b5d87046623736e8e7e2cf1ad1ddf8e9b88e48 test: test_n1b counts the unreaped original child by pid (p… Sonnet 5.5 3 days ago
sha256:c3e18b15a2c0061e5bf0cf1130ec534cfad3582e0abaf5750ae68b8093a1258d fix(intel): head-keyed rebuild dedup, named reap failure, f… Sonnet 5.5 minor ⚠ 3 days ago
sha256:3733f942311ecd1beecb0d6f112f3ebd87b2801a5a410bf35b2415a84a238f1b fix(intel): one-shot child, reap confirmed before the refer… Sonnet 5.5 minor ⚠ 3 days ago