gabriel / musehub public
test_prebuild_serialization_331.py python
307 lines 13.9 KB
Raw
sha256:cb4b05e9d6caea5f351e3cb452ee6a1db8856169328ad96bfba9ee8e04b8afa6 fix(jobs): fence every execution of a job; an expired or su… Sonnet 5.5 minor ⚠ breaking 4 days ago
1 """musehub#331 — prebuilds are serialised host-wide and their memory is returned to the OS after every job.
2
3 Found live: production fell to 119 MB MemAvailable (2026-11-12, near-miss) and staging to 171 MB during the rc.5
4 push-triggered prebuild. Every app slot runs an embedded job worker next to the dedicated worker container
5 (test_db_connection_budget), so up to three independent claimers poll one queue. ``claim_next_job``'s
6 ``FOR UPDATE SKIP LOCKED`` only stops two claimers taking the *same* row: two pending prebuilds for different
7 repositories were claimed concurrently, one per worker process, each holding ~1.4 GB. Separately, the worker
8 process that ran a prebuild kept ~0.9 GB of it afterwards (allocator retention, musehub#315 Phase 2). Contract:
9
10 * PBS_01 a running prebuild blocks the claim of every other pending prebuild, in any repository;
11 * PBS_02 other job types are claimed as before while a prebuild runs (the barrier is prebuild-only);
12 * PBS_03 the barrier lifts when the running prebuild completes, fails, or is reclaimed as stale;
13 * PBS_04 two claims racing in separate transactions never both take a prebuild (admission is serialised);
14 * PBS_05 the worker runs a prebuild in a short-lived child process, never in its own process;
15 * PBS_06 a failing child fails the job through ``fail_job`` (retry bookkeeping unchanged) and leaves no process;
16 * PBS_07 cancelling the worker kills the child; a child past the lease cap is killed (so a reclaim cannot overlap it);
17 * PBS_08 a child killed by the kernel (OOM) is an error, not a silent success;
18 * PBS_09 a child whose worker is gone exits by itself.
19 """
20 from __future__ import annotations
21
22 import asyncio
23 import os
24 import sys
25 import textwrap
26 from datetime import datetime, timedelta, timezone
27 from pathlib import Path
28 from collections.abc import AsyncGenerator
29
30 import pytest
31 from sqlalchemy import select
32 from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
33
34 from musehub.core.genesis import compute_job_id
35 from musehub.db import database
36 from musehub.db.musehub_jobs_models import MusehubBackgroundJob
37 from musehub.services import musehub_jobs
38 from musehub.services.musehub_jobs import (
39 _STALE_CLAIM_MINUTES,
40 claim_next_job,
41 complete_job,
42 fail_job,
43 reclaim_stale_jobs,
44 )
45 from tests.job_execution import current_attempt, start_execution # noqa: F401
46 from tests.factories import create_repo
47
48 _PREBUILD = "fetch.mpack.prebuild"
49
50
51 def _now() -> datetime:
52 return datetime.now(tz=timezone.utc)
53
54
55 async def _job(
56 session: AsyncSession, repo_id: str, job_type: str, status: str, seed: str, *, age_s: int = 0, claimed_min_ago: int = 0,
57 ) -> str:
58 created = _now() - timedelta(seconds=age_s)
59 job_id = compute_job_id(repo_id, job_type, created.isoformat() + seed)
60 session.add(MusehubBackgroundJob(
61 job_id=job_id, repo_id=repo_id, job_type=job_type, payload={}, status=status, created_at=created,
62 claimed_at=_now() - timedelta(minutes=claimed_min_ago) if status == "running" else None,
63 attempt=1 if status == "running" else 0,
64 ))
65 await session.commit()
66 return job_id
67
68
69 async def _status(session: AsyncSession, job_id: str) -> str:
70 session.expire_all()
71 return (await session.execute(select(MusehubBackgroundJob.status).where(MusehubBackgroundJob.job_id == job_id))).scalar_one()
72
73
74 async def _claim(session: AsyncSession) -> MusehubBackgroundJob | None:
75 job = await claim_next_job(session)
76 await session.commit()
77 return job
78
79
80 # ── admission: one prebuild host-wide ───────────────────────────────────────────────────────────────────────────
81
82
83 @pytest.mark.asyncio
84 async def test_pbs_01_running_prebuild_blocks_every_other_prebuild(db_session: AsyncSession) -> None:
85 repo_a, repo_b = await create_repo(db_session), await create_repo(db_session)
86 await _job(db_session, repo_a.repo_id, _PREBUILD, "running", "a")
87 pending_b = await _job(db_session, repo_b.repo_id, _PREBUILD, "pending", "b")
88 assert await _claim(db_session) is None, "a second prebuild (another repository) must wait for the running one"
89 assert await _status(db_session, pending_b) == "pending"
90
91
92 @pytest.mark.asyncio
93 async def test_pbs_02_other_job_types_are_not_blocked(db_session: AsyncSession) -> None:
94 repo_a, repo_b, repo_c = await create_repo(db_session), await create_repo(db_session), await create_repo(db_session)
95 await _job(db_session, repo_a.repo_id, _PREBUILD, "running", "a")
96 await _job(db_session, repo_b.repo_id, _PREBUILD, "pending", "b", age_s=60) # older, so it would win by created_at
97 index = await _job(db_session, repo_c.repo_id, "mpack.index", "pending", "c")
98 claimed = await _claim(db_session)
99 assert claimed is not None and claimed.job_id == index, "the barrier is prebuild-only; the queue keeps moving"
100
101
102 @pytest.mark.asyncio
103 @pytest.mark.parametrize("how", ["done", "failed", "retry", "reclaimed"])
104 async def test_pbs_03_barrier_lifts_when_the_running_prebuild_ends(db_session: AsyncSession, how: str) -> None:
105 repo_a, repo_b = await create_repo(db_session), await create_repo(db_session)
106 running = await _job(db_session, repo_a.repo_id, _PREBUILD, "running", "a", claimed_min_ago=_STALE_CLAIM_MINUTES + 5 if how == "reclaimed" else 0)
107 pending_b = await _job(db_session, repo_b.repo_id, _PREBUILD, "pending", "b")
108 assert await _claim(db_session) is None
109
110 if how == "done":
111 await complete_job(db_session, running, attempt=await current_attempt(db_session, running))
112 elif how == "failed":
113 row = (await db_session.execute(select(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == running))).scalar_one()
114 row.attempt = musehub_jobs._MAX_ATTEMPTS
115 await db_session.flush()
116 await fail_job(db_session, running, "boom", attempt=musehub_jobs._MAX_ATTEMPTS)
117 elif how == "retry":
118 await fail_job(db_session, running, "boom", attempt=await current_attempt(db_session, running)) # back to pending (attempt < max); the older job now wins
119 else:
120 await reclaim_stale_jobs(db_session)
121 await db_session.commit()
122
123 claimed = await _claim(db_session)
124 assert claimed is not None and claimed.job_type == _PREBUILD
125 assert claimed.job_id in {pending_b, running}, "after the running prebuild ends, a pending prebuild is claimable again"
126 assert (await _status(db_session, claimed.job_id)) == "running"
127
128
129 @pytest.mark.asyncio
130 async def test_pbs_04_racing_claims_never_both_take_a_prebuild(
131 db_session: AsyncSession, session_factory: async_sessionmaker[AsyncSession],
132 ) -> None:
133 """Two workers claim at the same moment: A's claim is not committed yet when B claims.
134
135 Without admission serialisation B cannot see A's 'running' row, so both get a prebuild.
136 """
137 repo_a, repo_b = await create_repo(db_session), await create_repo(db_session)
138 await _job(db_session, repo_a.repo_id, _PREBUILD, "pending", "a", age_s=30)
139 await _job(db_session, repo_b.repo_id, _PREBUILD, "pending", "b")
140
141 async with session_factory() as sess_a, session_factory() as sess_b:
142 first = await claim_next_job(sess_a) # in flight: uncommitted
143 second = await claim_next_job(sess_b)
144 await sess_a.commit()
145 await sess_b.commit()
146
147 claimed = [j for j in (first, second) if j is not None]
148 assert len(claimed) == 1 and first is not None, "exactly one prebuild is admitted"
149 # the loser is simply retried on its next poll, and then finds the barrier instead of the lock
150 async with session_factory() as sess_c:
151 assert await _claim(sess_c) is None
152
153
154 # ── the worker runs a prebuild in a child process ──────────────────────────────────────────────────────────────
155
156
157 @pytest.fixture
158 async def worker_db(db_session: AsyncSession, monkeypatch: pytest.MonkeyPatch) -> AsyncGenerator[AsyncSession, None]:
159 """The production engine (``init_db``) pointed at the test database, for ``worker._process_one``.
160
161 The child process builds its own engine from the environment, so the test URL is exported too.
162 """
163 from musehub.config import settings
164 from tests.conftest import _TEST_DATABASE_URL
165
166 monkeypatch.setattr(settings, "database_url", _TEST_DATABASE_URL)
167 monkeypatch.setenv("DATABASE_URL", _TEST_DATABASE_URL)
168 await database.init_db(pool_size=1, max_overflow=1)
169 try:
170 yield db_session
171 finally:
172 await database.close_db()
173
174
175 @pytest.mark.asyncio
176 async def test_pbs_05_prebuild_runs_in_a_child_process_not_in_the_worker(
177 worker_db: AsyncSession, monkeypatch: pytest.MonkeyPatch,
178 ) -> None:
179 import musehub.worker as worker
180 from musehub.services import musehub_wire_fetch
181
182 async def must_not_run_in_the_worker(*_a: object, **_k: object) -> object:
183 raise AssertionError("the prebuild ran inside the worker process")
184
185 monkeypatch.setattr(musehub_wire_fetch, "process_fetch_mpack_prebuild_job", must_not_run_in_the_worker)
186 repo = await create_repo(worker_db) # no branches: the job body returns zeros after the real DB round trip
187 job_id = await _job(worker_db, repo.repo_id, _PREBUILD, "pending", "child")
188
189 assert await worker._process_one() is True
190 assert await _status(worker_db, job_id) == "done"
191
192
193 @pytest.mark.asyncio
194 async def test_pbs_06_a_failing_child_fails_the_job_through_fail_job(
195 worker_db: AsyncSession, monkeypatch: pytest.MonkeyPatch,
196 ) -> None:
197 import musehub.worker as worker
198
199 pids: list[int] = []
200 real = asyncio.create_subprocess_exec
201
202 async def spy(*args: object, **kwargs: object) -> asyncio.subprocess.Process:
203 proc = await real(*args, **kwargs) # type: ignore[arg-type]
204 pids.append(proc.pid)
205 return proc
206
207 monkeypatch.setattr(asyncio, "create_subprocess_exec", spy)
208 # the child cannot find its job: process_fetch_mpack_prebuild_job raises ValueError inside the child
209 real_command = worker._child_command
210 monkeypatch.setattr(worker, "_child_command", lambda job_id: real_command("sha256:" + "0" * 64))
211 repo = await create_repo(worker_db)
212 job_id = await _job(worker_db, repo.repo_id, _PREBUILD, "pending", "fail")
213
214 assert await worker._process_one() is True
215 assert await _status(worker_db, job_id) == "pending", "fail_job's retry bookkeeping is unchanged (attempt 1 of 3)"
216 row = (await worker_db.execute(select(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == job_id))).scalar_one()
217 assert row.attempt == 1 and row.error and "ValueError" in row.error and "prebuild job not found" in row.error
218 assert len(pids) == 1
219 with pytest.raises(ProcessLookupError):
220 os.kill(pids[0], 0) # reaped: no process left behind
221
222
223 def _script(tmp_path: Path, body: str) -> list[str]:
224 path = tmp_path / "child.py"
225 path.write_text(textwrap.dedent(body))
226 return [sys.executable, str(path)]
227
228
229 async def _wait_for(path: Path, timeout: float = 10.0) -> str:
230 deadline = asyncio.get_running_loop().time() + timeout
231 while asyncio.get_running_loop().time() < deadline:
232 if path.exists() and path.read_text().strip():
233 return path.read_text().strip()
234 await asyncio.sleep(0.02)
235 raise AssertionError(f"{path} never written")
236
237
238 def _alive(pid: int) -> bool:
239 try:
240 os.kill(pid, 0)
241 except ProcessLookupError:
242 return False
243 return True
244
245
246 @pytest.mark.asyncio
247 async def test_pbs_07_cancellation_and_the_lease_cap_kill_the_child(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
248 import musehub.worker as worker
249
250 pid_file = tmp_path / "pid"
251 sleeper = _script(tmp_path, f"""
252 import os, time
253 open({str(pid_file)!r}, "w").write(str(os.getpid()))
254 time.sleep(120)
255 """)
256 monkeypatch.setattr(worker, "_child_command", lambda job_id: sleeper)
257
258 task = asyncio.create_task(worker._run_prebuild_child("sha256:job"))
259 pid = int(await _wait_for(pid_file))
260 assert _alive(pid)
261 task.cancel()
262 with pytest.raises(asyncio.CancelledError):
263 await task
264 assert not _alive(pid), "cancelling the worker must not leave the child building"
265
266 pid_file.unlink()
267 monkeypatch.setattr(worker, "_CLAIM_MAX_SECONDS", 0.5)
268 with pytest.raises(RuntimeError, match="lease cap"):
269 await worker._run_prebuild_child("sha256:job")
270 assert not _alive(int(pid_file.read_text())), "the child dies at the cap, before reclaim_stale_jobs could start a second build"
271
272
273 @pytest.mark.asyncio
274 async def test_pbs_08_a_child_killed_by_the_kernel_is_an_error(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
275 import musehub.worker as worker
276
277 oom = _script(tmp_path, """
278 import os, signal
279 os.kill(os.getpid(), signal.SIGKILL)
280 """)
281 monkeypatch.setattr(worker, "_child_command", lambda job_id: oom)
282 with pytest.raises(RuntimeError, match="signal 9"):
283 await worker._run_prebuild_child("sha256:job")
284
285 silent_exit = _script(tmp_path, "import sys; sys.exit(0)") # exit 0 but no result line: not a success
286 monkeypatch.setattr(worker, "_child_command", lambda job_id: silent_exit)
287 with pytest.raises(RuntimeError, match="no result"):
288 await worker._run_prebuild_child("sha256:job")
289
290
291 @pytest.mark.asyncio
292 async def test_pbs_09_a_child_whose_worker_is_gone_exits(monkeypatch: pytest.MonkeyPatch) -> None:
293 from musehub import prebuild_child
294
295 died: list[int] = []
296 monkeypatch.setattr(prebuild_child, "_die", lambda code: died.append(code))
297 watcher = asyncio.create_task(prebuild_child._watch_parent(parent_pid=os.getpid() + 1, interval=0.01))
298 await asyncio.wait_for(watcher, timeout=2)
299 assert died == [1]
300
301 died.clear()
302 live = asyncio.create_task(prebuild_child._watch_parent(parent_pid=os.getppid(), interval=0.01))
303 await asyncio.sleep(0.1)
304 live.cancel()
305 with pytest.raises(asyncio.CancelledError):
306 await live
307 assert died == [], "a child whose worker is alive keeps running"
File History 2 commits
sha256:cb4b05e9d6caea5f351e3cb452ee6a1db8856169328ad96bfba9ee8e04b8afa6 fix(jobs): fence every execution of a job; an expired or su… Sonnet 5.5 minor ⚠ 4 days ago
sha256:217096701012b41f6362eec2e887220098537b3fc89eedd49d41eb11bb9e5e41 fix(worker): one prebuild host-wide, run in a short-lived c… Sonnet 5.5 patch 4 days ago