test_intel_supervisor.py
python
sha256:3733f942311ecd1beecb0d6f112f3ebd87b2801a5a410bf35b2415a84a238f1b
fix(intel): one-shot child, reap confirmed before the refer…
Sonnet 5.5
minor
⚠ breaking
3 days ago
| 1 | """The bounded intel child — musehub#8 release gate R1. |
| 2 | |
| 3 | Run ONLY this file (no database needed):: |
| 4 | |
| 5 | python -P -m pytest tests/test_intel_supervisor.py |
| 6 | |
| 7 | A stub child speaks the real protocol and misbehaves on request. Rules under test: |
| 8 | |
| 9 | J1 A job's result comes back unchanged, and one child serves consecutive jobs (the caches live in it). |
| 10 | J2 A job that raises in the child fails with the traceback and the child stays healthy for the next job. |
| 11 | J3 The wall-clock limit kills a hung child; the worker survives and the next job gets a fresh child. |
| 12 | J4 The memory ceiling kills a child that grows past it, before the host OOM killer decides. |
| 13 | J5 The CPU budget kills a child that spins. |
| 14 | J6 A child that dies on its own (crash, OOM-kill) is reported, not hung on, and replaced. |
| 15 | J7 Cancelling the job kills the child: no execution outlives its job. |
| 16 | J8 An idle child is ended; a busy one is not. |
| 17 | J9 The real command line carries no credentials, and closing kills the child. |
| 18 | """ |
| 19 | from __future__ import annotations |
| 20 | |
| 21 | import asyncio |
| 22 | import os |
| 23 | import sys |
| 24 | from pathlib import Path |
| 25 | |
| 26 | import psutil |
| 27 | import pytest |
| 28 | |
| 29 | from musehub import intel_supervisor as S |
| 30 | |
| 31 | STUB = str(Path(__file__).with_name("intel_child_stub.py")) |
| 32 | |
| 33 | |
| 34 | def limits(**kw) -> S.IntelChildLimits: |
| 35 | base = dict(memory_bytes=600 * 2**20, cpu_seconds=30, wall_seconds=20, poll_seconds=0.05) |
| 36 | base.update(kw) |
| 37 | return S.IntelChildLimits(**base) |
| 38 | |
| 39 | |
| 40 | def child(**kw) -> S.IntelChild: |
| 41 | return S.IntelChild(limits(**kw), command=lambda: [sys.executable, "-P", STUB]) |
| 42 | |
| 43 | |
| 44 | def alive(pid: int) -> bool: |
| 45 | try: |
| 46 | return psutil.Process(pid).status() != psutil.STATUS_ZOMBIE |
| 47 | except psutil.NoSuchProcess: |
| 48 | return False |
| 49 | |
| 50 | |
| 51 | @pytest.mark.asyncio |
| 52 | async def test_j1_results_come_back_and_each_job_runs_in_its_own_child() -> None: |
| 53 | c = child() |
| 54 | try: |
| 55 | t1, r1 = await c.run("ok", "repo", {"n": 1}) |
| 56 | t2, r2 = await c.run("ok", "repo", {"n": 2}) |
| 57 | assert (t1, r1["echo"]) == ("done", {"n": 1}) and r2["echo"] == {"n": 2} |
| 58 | assert r1["pid"] != r2["pid"] and c.pid is None |
| 59 | finally: |
| 60 | await c.close() |
| 61 | |
| 62 | |
| 63 | @pytest.mark.asyncio |
| 64 | async def test_j2_a_job_failure_carries_the_traceback_and_the_next_job_still_runs() -> None: |
| 65 | c = child() |
| 66 | try: |
| 67 | with pytest.raises(S.IntelChildError, match="boom"): |
| 68 | await c.run("fail", "repo", {}) |
| 69 | _, again = await c.run("ok", "repo", {}) |
| 70 | assert again["echo"] == {} |
| 71 | finally: |
| 72 | await c.close() |
| 73 | |
| 74 | |
| 75 | @pytest.mark.asyncio |
| 76 | async def test_j3_the_wall_clock_limit_kills_a_hung_child_and_the_next_job_gets_a_fresh_one() -> None: |
| 77 | c = child(wall_seconds=0.7) |
| 78 | try: |
| 79 | with pytest.raises(S.IntelChildTimeout): |
| 80 | await c.run("hang", "repo", {}) |
| 81 | assert c.pid is None and c._proc is None |
| 82 | _, after = await c.run("ok", "repo", {}) |
| 83 | assert after["pid"] > 0 |
| 84 | finally: |
| 85 | await c.close() |
| 86 | |
| 87 | |
| 88 | @pytest.mark.asyncio |
| 89 | async def test_j4_the_memory_ceiling_kills_a_growing_child() -> None: |
| 90 | c = child(memory_bytes=120 * 2**20) |
| 91 | try: |
| 92 | with pytest.raises(S.IntelChildMemoryExceeded): |
| 93 | await c.run("alloc", "repo", {"mib": 400}) |
| 94 | assert c.pid is None and c._proc is None |
| 95 | finally: |
| 96 | await c.close() |
| 97 | |
| 98 | |
| 99 | @pytest.mark.asyncio |
| 100 | async def test_j5_the_cpu_budget_kills_a_spinning_child() -> None: |
| 101 | c = child(cpu_seconds=1.0) |
| 102 | try: |
| 103 | with pytest.raises(S.IntelChildCpuExceeded): |
| 104 | await c.run("burn", "repo", {}) |
| 105 | assert c.pid is None and c._proc is None |
| 106 | finally: |
| 107 | await c.close() |
| 108 | |
| 109 | |
| 110 | @pytest.mark.asyncio |
| 111 | async def test_j6_a_child_that_dies_on_its_own_is_reported_and_replaced() -> None: |
| 112 | c = child() |
| 113 | try: |
| 114 | with pytest.raises(S.IntelChildDied, match="exit code 9"): |
| 115 | await c.run("die", "repo", {}) |
| 116 | _, after = await c.run("ok", "repo", {}) |
| 117 | assert after["pid"] > 0 |
| 118 | finally: |
| 119 | await c.close() |
| 120 | |
| 121 | |
| 122 | @pytest.mark.asyncio |
| 123 | async def test_j7_cancelling_the_job_kills_the_child() -> None: |
| 124 | c = child() |
| 125 | try: |
| 126 | task = asyncio.create_task(c.run("hang", "repo", {})) |
| 127 | await asyncio.sleep(0.5) |
| 128 | pid = c._proc.pid |
| 129 | task.cancel() |
| 130 | with pytest.raises(asyncio.CancelledError): |
| 131 | await task |
| 132 | assert not alive(pid) and c.pid is None |
| 133 | finally: |
| 134 | await c.close() |
| 135 | |
| 136 | |
| 137 | @pytest.mark.asyncio |
| 138 | async def test_j9_the_real_command_line_carries_no_credentials_and_close_kills_the_child(monkeypatch) -> None: |
| 139 | monkeypatch.setenv("DATABASE_URL", "postgresql+asyncpg://user:s3cret-password@db:5432/x") |
| 140 | argv = S.default_command() |
| 141 | assert argv == [sys.executable, "-m", "musehub.intel_child", str(os.getpid())] |
| 142 | assert "s3cret-password" not in " ".join(argv) |
| 143 | c = child() |
| 144 | proc = await c._ensure() # a live child, as between spawn and the end of a job |
| 145 | await c.close() |
| 146 | assert not alive(proc.pid) |
File History
2 commits
sha256:3733f942311ecd1beecb0d6f112f3ebd87b2801a5a410bf35b2415a84a238f1b
fix(intel): one-shot child, reap confirmed before the refer…
Sonnet 5.5
minor
⚠
3 days ago
sha256:f0ce9f515a530043854b2549a902d0df78837898e9e9439fe32abeee180bef85
feat(worker): run intel jobs in a bounded child process; th…
Sonnet 5.5
patch
4 days ago