gabriel / musehub public
test_intel_supervisor.py python
146 lines 4.8 KB
Raw
sha256:3733f942311ecd1beecb0d6f112f3ebd87b2801a5a410bf35b2415a84a238f1b fix(intel): one-shot child, reap confirmed before the refer… Sonnet 5.5 minor ⚠ breaking 3 days ago
1 """The bounded intel child — musehub#8 release gate R1.
2
3 Run ONLY this file (no database needed)::
4
5 python -P -m pytest tests/test_intel_supervisor.py
6
7 A stub child speaks the real protocol and misbehaves on request. Rules under test:
8
9 J1 A job's result comes back unchanged, and one child serves consecutive jobs (the caches live in it).
10 J2 A job that raises in the child fails with the traceback and the child stays healthy for the next job.
11 J3 The wall-clock limit kills a hung child; the worker survives and the next job gets a fresh child.
12 J4 The memory ceiling kills a child that grows past it, before the host OOM killer decides.
13 J5 The CPU budget kills a child that spins.
14 J6 A child that dies on its own (crash, OOM-kill) is reported, not hung on, and replaced.
15 J7 Cancelling the job kills the child: no execution outlives its job.
16 J8 An idle child is ended; a busy one is not.
17 J9 The real command line carries no credentials, and closing kills the child.
18 """
19 from __future__ import annotations
20
21 import asyncio
22 import os
23 import sys
24 from pathlib import Path
25
26 import psutil
27 import pytest
28
29 from musehub import intel_supervisor as S
30
31 STUB = str(Path(__file__).with_name("intel_child_stub.py"))
32
33
34 def limits(**kw) -> S.IntelChildLimits:
35 base = dict(memory_bytes=600 * 2**20, cpu_seconds=30, wall_seconds=20, poll_seconds=0.05)
36 base.update(kw)
37 return S.IntelChildLimits(**base)
38
39
40 def child(**kw) -> S.IntelChild:
41 return S.IntelChild(limits(**kw), command=lambda: [sys.executable, "-P", STUB])
42
43
44 def alive(pid: int) -> bool:
45 try:
46 return psutil.Process(pid).status() != psutil.STATUS_ZOMBIE
47 except psutil.NoSuchProcess:
48 return False
49
50
51 @pytest.mark.asyncio
52 async def test_j1_results_come_back_and_each_job_runs_in_its_own_child() -> None:
53 c = child()
54 try:
55 t1, r1 = await c.run("ok", "repo", {"n": 1})
56 t2, r2 = await c.run("ok", "repo", {"n": 2})
57 assert (t1, r1["echo"]) == ("done", {"n": 1}) and r2["echo"] == {"n": 2}
58 assert r1["pid"] != r2["pid"] and c.pid is None
59 finally:
60 await c.close()
61
62
63 @pytest.mark.asyncio
64 async def test_j2_a_job_failure_carries_the_traceback_and_the_next_job_still_runs() -> None:
65 c = child()
66 try:
67 with pytest.raises(S.IntelChildError, match="boom"):
68 await c.run("fail", "repo", {})
69 _, again = await c.run("ok", "repo", {})
70 assert again["echo"] == {}
71 finally:
72 await c.close()
73
74
75 @pytest.mark.asyncio
76 async def test_j3_the_wall_clock_limit_kills_a_hung_child_and_the_next_job_gets_a_fresh_one() -> None:
77 c = child(wall_seconds=0.7)
78 try:
79 with pytest.raises(S.IntelChildTimeout):
80 await c.run("hang", "repo", {})
81 assert c.pid is None and c._proc is None
82 _, after = await c.run("ok", "repo", {})
83 assert after["pid"] > 0
84 finally:
85 await c.close()
86
87
88 @pytest.mark.asyncio
89 async def test_j4_the_memory_ceiling_kills_a_growing_child() -> None:
90 c = child(memory_bytes=120 * 2**20)
91 try:
92 with pytest.raises(S.IntelChildMemoryExceeded):
93 await c.run("alloc", "repo", {"mib": 400})
94 assert c.pid is None and c._proc is None
95 finally:
96 await c.close()
97
98
99 @pytest.mark.asyncio
100 async def test_j5_the_cpu_budget_kills_a_spinning_child() -> None:
101 c = child(cpu_seconds=1.0)
102 try:
103 with pytest.raises(S.IntelChildCpuExceeded):
104 await c.run("burn", "repo", {})
105 assert c.pid is None and c._proc is None
106 finally:
107 await c.close()
108
109
110 @pytest.mark.asyncio
111 async def test_j6_a_child_that_dies_on_its_own_is_reported_and_replaced() -> None:
112 c = child()
113 try:
114 with pytest.raises(S.IntelChildDied, match="exit code 9"):
115 await c.run("die", "repo", {})
116 _, after = await c.run("ok", "repo", {})
117 assert after["pid"] > 0
118 finally:
119 await c.close()
120
121
122 @pytest.mark.asyncio
123 async def test_j7_cancelling_the_job_kills_the_child() -> None:
124 c = child()
125 try:
126 task = asyncio.create_task(c.run("hang", "repo", {}))
127 await asyncio.sleep(0.5)
128 pid = c._proc.pid
129 task.cancel()
130 with pytest.raises(asyncio.CancelledError):
131 await task
132 assert not alive(pid) and c.pid is None
133 finally:
134 await c.close()
135
136
137 @pytest.mark.asyncio
138 async def test_j9_the_real_command_line_carries_no_credentials_and_close_kills_the_child(monkeypatch) -> None:
139 monkeypatch.setenv("DATABASE_URL", "postgresql+asyncpg://user:s3cret-password@db:5432/x")
140 argv = S.default_command()
141 assert argv == [sys.executable, "-m", "musehub.intel_child", str(os.getpid())]
142 assert "s3cret-password" not in " ".join(argv)
143 c = child()
144 proc = await c._ensure() # a live child, as between spawn and the end of a job
145 await c.close()
146 assert not alive(proc.pid)
File History 2 commits
sha256:3733f942311ecd1beecb0d6f112f3ebd87b2801a5a410bf35b2415a84a238f1b fix(intel): one-shot child, reap confirmed before the refer… Sonnet 5.5 minor ⚠ 3 days ago
sha256:f0ce9f515a530043854b2549a902d0df78837898e9e9439fe32abeee180bef85 feat(worker): run intel jobs in a bounded child process; th… Sonnet 5.5 patch 4 days ago