test_prebuild_serialization_331.py
python
sha256:cb4b05e9d6caea5f351e3cb452ee6a1db8856169328ad96bfba9ee8e04b8afa6
fix(jobs): fence every execution of a job; an expired or su…
Sonnet 5.5
minor
⚠ breaking
4 days ago
| 1 | """musehub#331 — prebuilds are serialised host-wide and their memory is returned to the OS after every job. |
| 2 | |
| 3 | Found live: production fell to 119 MB MemAvailable (2026-11-12, near-miss) and staging to 171 MB during the rc.5 |
| 4 | push-triggered prebuild. Every app slot runs an embedded job worker next to the dedicated worker container |
| 5 | (test_db_connection_budget), so up to three independent claimers poll one queue. ``claim_next_job``'s |
| 6 | ``FOR UPDATE SKIP LOCKED`` only stops two claimers taking the *same* row: two pending prebuilds for different |
| 7 | repositories were claimed concurrently, one per worker process, each holding ~1.4 GB. Separately, the worker |
| 8 | process that ran a prebuild kept ~0.9 GB of it afterwards (allocator retention, musehub#315 Phase 2). Contract: |
| 9 | |
| 10 | * PBS_01 a running prebuild blocks the claim of every other pending prebuild, in any repository; |
| 11 | * PBS_02 other job types are claimed as before while a prebuild runs (the barrier is prebuild-only); |
| 12 | * PBS_03 the barrier lifts when the running prebuild completes, fails, or is reclaimed as stale; |
| 13 | * PBS_04 two claims racing in separate transactions never both take a prebuild (admission is serialised); |
| 14 | * PBS_05 the worker runs a prebuild in a short-lived child process, never in its own process; |
| 15 | * PBS_06 a failing child fails the job through ``fail_job`` (retry bookkeeping unchanged) and leaves no process; |
| 16 | * PBS_07 cancelling the worker kills the child; a child past the lease cap is killed (so a reclaim cannot overlap it); |
| 17 | * PBS_08 a child killed by the kernel (OOM) is an error, not a silent success; |
| 18 | * PBS_09 a child whose worker is gone exits by itself. |
| 19 | """ |
| 20 | from __future__ import annotations |
| 21 | |
| 22 | import asyncio |
| 23 | import os |
| 24 | import sys |
| 25 | import textwrap |
| 26 | from datetime import datetime, timedelta, timezone |
| 27 | from pathlib import Path |
| 28 | from collections.abc import AsyncGenerator |
| 29 | |
| 30 | import pytest |
| 31 | from sqlalchemy import select |
| 32 | from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker |
| 33 | |
| 34 | from musehub.core.genesis import compute_job_id |
| 35 | from musehub.db import database |
| 36 | from musehub.db.musehub_jobs_models import MusehubBackgroundJob |
| 37 | from musehub.services import musehub_jobs |
| 38 | from musehub.services.musehub_jobs import ( |
| 39 | _STALE_CLAIM_MINUTES, |
| 40 | claim_next_job, |
| 41 | complete_job, |
| 42 | fail_job, |
| 43 | reclaim_stale_jobs, |
| 44 | ) |
| 45 | from tests.job_execution import current_attempt, start_execution # noqa: F401 |
| 46 | from tests.factories import create_repo |
| 47 | |
| 48 | _PREBUILD = "fetch.mpack.prebuild" |
| 49 | |
| 50 | |
| 51 | def _now() -> datetime: |
| 52 | return datetime.now(tz=timezone.utc) |
| 53 | |
| 54 | |
| 55 | async def _job( |
| 56 | session: AsyncSession, repo_id: str, job_type: str, status: str, seed: str, *, age_s: int = 0, claimed_min_ago: int = 0, |
| 57 | ) -> str: |
| 58 | created = _now() - timedelta(seconds=age_s) |
| 59 | job_id = compute_job_id(repo_id, job_type, created.isoformat() + seed) |
| 60 | session.add(MusehubBackgroundJob( |
| 61 | job_id=job_id, repo_id=repo_id, job_type=job_type, payload={}, status=status, created_at=created, |
| 62 | claimed_at=_now() - timedelta(minutes=claimed_min_ago) if status == "running" else None, |
| 63 | attempt=1 if status == "running" else 0, |
| 64 | )) |
| 65 | await session.commit() |
| 66 | return job_id |
| 67 | |
| 68 | |
| 69 | async def _status(session: AsyncSession, job_id: str) -> str: |
| 70 | session.expire_all() |
| 71 | return (await session.execute(select(MusehubBackgroundJob.status).where(MusehubBackgroundJob.job_id == job_id))).scalar_one() |
| 72 | |
| 73 | |
| 74 | async def _claim(session: AsyncSession) -> MusehubBackgroundJob | None: |
| 75 | job = await claim_next_job(session) |
| 76 | await session.commit() |
| 77 | return job |
| 78 | |
| 79 | |
| 80 | # ── admission: one prebuild host-wide ─────────────────────────────────────────────────────────────────────────── |
| 81 | |
| 82 | |
| 83 | @pytest.mark.asyncio |
| 84 | async def test_pbs_01_running_prebuild_blocks_every_other_prebuild(db_session: AsyncSession) -> None: |
| 85 | repo_a, repo_b = await create_repo(db_session), await create_repo(db_session) |
| 86 | await _job(db_session, repo_a.repo_id, _PREBUILD, "running", "a") |
| 87 | pending_b = await _job(db_session, repo_b.repo_id, _PREBUILD, "pending", "b") |
| 88 | assert await _claim(db_session) is None, "a second prebuild (another repository) must wait for the running one" |
| 89 | assert await _status(db_session, pending_b) == "pending" |
| 90 | |
| 91 | |
| 92 | @pytest.mark.asyncio |
| 93 | async def test_pbs_02_other_job_types_are_not_blocked(db_session: AsyncSession) -> None: |
| 94 | repo_a, repo_b, repo_c = await create_repo(db_session), await create_repo(db_session), await create_repo(db_session) |
| 95 | await _job(db_session, repo_a.repo_id, _PREBUILD, "running", "a") |
| 96 | await _job(db_session, repo_b.repo_id, _PREBUILD, "pending", "b", age_s=60) # older, so it would win by created_at |
| 97 | index = await _job(db_session, repo_c.repo_id, "mpack.index", "pending", "c") |
| 98 | claimed = await _claim(db_session) |
| 99 | assert claimed is not None and claimed.job_id == index, "the barrier is prebuild-only; the queue keeps moving" |
| 100 | |
| 101 | |
| 102 | @pytest.mark.asyncio |
| 103 | @pytest.mark.parametrize("how", ["done", "failed", "retry", "reclaimed"]) |
| 104 | async def test_pbs_03_barrier_lifts_when_the_running_prebuild_ends(db_session: AsyncSession, how: str) -> None: |
| 105 | repo_a, repo_b = await create_repo(db_session), await create_repo(db_session) |
| 106 | running = await _job(db_session, repo_a.repo_id, _PREBUILD, "running", "a", claimed_min_ago=_STALE_CLAIM_MINUTES + 5 if how == "reclaimed" else 0) |
| 107 | pending_b = await _job(db_session, repo_b.repo_id, _PREBUILD, "pending", "b") |
| 108 | assert await _claim(db_session) is None |
| 109 | |
| 110 | if how == "done": |
| 111 | await complete_job(db_session, running, attempt=await current_attempt(db_session, running)) |
| 112 | elif how == "failed": |
| 113 | row = (await db_session.execute(select(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == running))).scalar_one() |
| 114 | row.attempt = musehub_jobs._MAX_ATTEMPTS |
| 115 | await db_session.flush() |
| 116 | await fail_job(db_session, running, "boom", attempt=musehub_jobs._MAX_ATTEMPTS) |
| 117 | elif how == "retry": |
| 118 | await fail_job(db_session, running, "boom", attempt=await current_attempt(db_session, running)) # back to pending (attempt < max); the older job now wins |
| 119 | else: |
| 120 | await reclaim_stale_jobs(db_session) |
| 121 | await db_session.commit() |
| 122 | |
| 123 | claimed = await _claim(db_session) |
| 124 | assert claimed is not None and claimed.job_type == _PREBUILD |
| 125 | assert claimed.job_id in {pending_b, running}, "after the running prebuild ends, a pending prebuild is claimable again" |
| 126 | assert (await _status(db_session, claimed.job_id)) == "running" |
| 127 | |
| 128 | |
| 129 | @pytest.mark.asyncio |
| 130 | async def test_pbs_04_racing_claims_never_both_take_a_prebuild( |
| 131 | db_session: AsyncSession, session_factory: async_sessionmaker[AsyncSession], |
| 132 | ) -> None: |
| 133 | """Two workers claim at the same moment: A's claim is not committed yet when B claims. |
| 134 | |
| 135 | Without admission serialisation B cannot see A's 'running' row, so both get a prebuild. |
| 136 | """ |
| 137 | repo_a, repo_b = await create_repo(db_session), await create_repo(db_session) |
| 138 | await _job(db_session, repo_a.repo_id, _PREBUILD, "pending", "a", age_s=30) |
| 139 | await _job(db_session, repo_b.repo_id, _PREBUILD, "pending", "b") |
| 140 | |
| 141 | async with session_factory() as sess_a, session_factory() as sess_b: |
| 142 | first = await claim_next_job(sess_a) # in flight: uncommitted |
| 143 | second = await claim_next_job(sess_b) |
| 144 | await sess_a.commit() |
| 145 | await sess_b.commit() |
| 146 | |
| 147 | claimed = [j for j in (first, second) if j is not None] |
| 148 | assert len(claimed) == 1 and first is not None, "exactly one prebuild is admitted" |
| 149 | # the loser is simply retried on its next poll, and then finds the barrier instead of the lock |
| 150 | async with session_factory() as sess_c: |
| 151 | assert await _claim(sess_c) is None |
| 152 | |
| 153 | |
| 154 | # ── the worker runs a prebuild in a child process ────────────────────────────────────────────────────────────── |
| 155 | |
| 156 | |
| 157 | @pytest.fixture |
| 158 | async def worker_db(db_session: AsyncSession, monkeypatch: pytest.MonkeyPatch) -> AsyncGenerator[AsyncSession, None]: |
| 159 | """The production engine (``init_db``) pointed at the test database, for ``worker._process_one``. |
| 160 | |
| 161 | The child process builds its own engine from the environment, so the test URL is exported too. |
| 162 | """ |
| 163 | from musehub.config import settings |
| 164 | from tests.conftest import _TEST_DATABASE_URL |
| 165 | |
| 166 | monkeypatch.setattr(settings, "database_url", _TEST_DATABASE_URL) |
| 167 | monkeypatch.setenv("DATABASE_URL", _TEST_DATABASE_URL) |
| 168 | await database.init_db(pool_size=1, max_overflow=1) |
| 169 | try: |
| 170 | yield db_session |
| 171 | finally: |
| 172 | await database.close_db() |
| 173 | |
| 174 | |
| 175 | @pytest.mark.asyncio |
| 176 | async def test_pbs_05_prebuild_runs_in_a_child_process_not_in_the_worker( |
| 177 | worker_db: AsyncSession, monkeypatch: pytest.MonkeyPatch, |
| 178 | ) -> None: |
| 179 | import musehub.worker as worker |
| 180 | from musehub.services import musehub_wire_fetch |
| 181 | |
| 182 | async def must_not_run_in_the_worker(*_a: object, **_k: object) -> object: |
| 183 | raise AssertionError("the prebuild ran inside the worker process") |
| 184 | |
| 185 | monkeypatch.setattr(musehub_wire_fetch, "process_fetch_mpack_prebuild_job", must_not_run_in_the_worker) |
| 186 | repo = await create_repo(worker_db) # no branches: the job body returns zeros after the real DB round trip |
| 187 | job_id = await _job(worker_db, repo.repo_id, _PREBUILD, "pending", "child") |
| 188 | |
| 189 | assert await worker._process_one() is True |
| 190 | assert await _status(worker_db, job_id) == "done" |
| 191 | |
| 192 | |
| 193 | @pytest.mark.asyncio |
| 194 | async def test_pbs_06_a_failing_child_fails_the_job_through_fail_job( |
| 195 | worker_db: AsyncSession, monkeypatch: pytest.MonkeyPatch, |
| 196 | ) -> None: |
| 197 | import musehub.worker as worker |
| 198 | |
| 199 | pids: list[int] = [] |
| 200 | real = asyncio.create_subprocess_exec |
| 201 | |
| 202 | async def spy(*args: object, **kwargs: object) -> asyncio.subprocess.Process: |
| 203 | proc = await real(*args, **kwargs) # type: ignore[arg-type] |
| 204 | pids.append(proc.pid) |
| 205 | return proc |
| 206 | |
| 207 | monkeypatch.setattr(asyncio, "create_subprocess_exec", spy) |
| 208 | # the child cannot find its job: process_fetch_mpack_prebuild_job raises ValueError inside the child |
| 209 | real_command = worker._child_command |
| 210 | monkeypatch.setattr(worker, "_child_command", lambda job_id: real_command("sha256:" + "0" * 64)) |
| 211 | repo = await create_repo(worker_db) |
| 212 | job_id = await _job(worker_db, repo.repo_id, _PREBUILD, "pending", "fail") |
| 213 | |
| 214 | assert await worker._process_one() is True |
| 215 | assert await _status(worker_db, job_id) == "pending", "fail_job's retry bookkeeping is unchanged (attempt 1 of 3)" |
| 216 | row = (await worker_db.execute(select(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == job_id))).scalar_one() |
| 217 | assert row.attempt == 1 and row.error and "ValueError" in row.error and "prebuild job not found" in row.error |
| 218 | assert len(pids) == 1 |
| 219 | with pytest.raises(ProcessLookupError): |
| 220 | os.kill(pids[0], 0) # reaped: no process left behind |
| 221 | |
| 222 | |
| 223 | def _script(tmp_path: Path, body: str) -> list[str]: |
| 224 | path = tmp_path / "child.py" |
| 225 | path.write_text(textwrap.dedent(body)) |
| 226 | return [sys.executable, str(path)] |
| 227 | |
| 228 | |
| 229 | async def _wait_for(path: Path, timeout: float = 10.0) -> str: |
| 230 | deadline = asyncio.get_running_loop().time() + timeout |
| 231 | while asyncio.get_running_loop().time() < deadline: |
| 232 | if path.exists() and path.read_text().strip(): |
| 233 | return path.read_text().strip() |
| 234 | await asyncio.sleep(0.02) |
| 235 | raise AssertionError(f"{path} never written") |
| 236 | |
| 237 | |
| 238 | def _alive(pid: int) -> bool: |
| 239 | try: |
| 240 | os.kill(pid, 0) |
| 241 | except ProcessLookupError: |
| 242 | return False |
| 243 | return True |
| 244 | |
| 245 | |
| 246 | @pytest.mark.asyncio |
| 247 | async def test_pbs_07_cancellation_and_the_lease_cap_kill_the_child(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: |
| 248 | import musehub.worker as worker |
| 249 | |
| 250 | pid_file = tmp_path / "pid" |
| 251 | sleeper = _script(tmp_path, f""" |
| 252 | import os, time |
| 253 | open({str(pid_file)!r}, "w").write(str(os.getpid())) |
| 254 | time.sleep(120) |
| 255 | """) |
| 256 | monkeypatch.setattr(worker, "_child_command", lambda job_id: sleeper) |
| 257 | |
| 258 | task = asyncio.create_task(worker._run_prebuild_child("sha256:job")) |
| 259 | pid = int(await _wait_for(pid_file)) |
| 260 | assert _alive(pid) |
| 261 | task.cancel() |
| 262 | with pytest.raises(asyncio.CancelledError): |
| 263 | await task |
| 264 | assert not _alive(pid), "cancelling the worker must not leave the child building" |
| 265 | |
| 266 | pid_file.unlink() |
| 267 | monkeypatch.setattr(worker, "_CLAIM_MAX_SECONDS", 0.5) |
| 268 | with pytest.raises(RuntimeError, match="lease cap"): |
| 269 | await worker._run_prebuild_child("sha256:job") |
| 270 | assert not _alive(int(pid_file.read_text())), "the child dies at the cap, before reclaim_stale_jobs could start a second build" |
| 271 | |
| 272 | |
| 273 | @pytest.mark.asyncio |
| 274 | async def test_pbs_08_a_child_killed_by_the_kernel_is_an_error(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: |
| 275 | import musehub.worker as worker |
| 276 | |
| 277 | oom = _script(tmp_path, """ |
| 278 | import os, signal |
| 279 | os.kill(os.getpid(), signal.SIGKILL) |
| 280 | """) |
| 281 | monkeypatch.setattr(worker, "_child_command", lambda job_id: oom) |
| 282 | with pytest.raises(RuntimeError, match="signal 9"): |
| 283 | await worker._run_prebuild_child("sha256:job") |
| 284 | |
| 285 | silent_exit = _script(tmp_path, "import sys; sys.exit(0)") # exit 0 but no result line: not a success |
| 286 | monkeypatch.setattr(worker, "_child_command", lambda job_id: silent_exit) |
| 287 | with pytest.raises(RuntimeError, match="no result"): |
| 288 | await worker._run_prebuild_child("sha256:job") |
| 289 | |
| 290 | |
| 291 | @pytest.mark.asyncio |
| 292 | async def test_pbs_09_a_child_whose_worker_is_gone_exits(monkeypatch: pytest.MonkeyPatch) -> None: |
| 293 | from musehub import prebuild_child |
| 294 | |
| 295 | died: list[int] = [] |
| 296 | monkeypatch.setattr(prebuild_child, "_die", lambda code: died.append(code)) |
| 297 | watcher = asyncio.create_task(prebuild_child._watch_parent(parent_pid=os.getpid() + 1, interval=0.01)) |
| 298 | await asyncio.wait_for(watcher, timeout=2) |
| 299 | assert died == [1] |
| 300 | |
| 301 | died.clear() |
| 302 | live = asyncio.create_task(prebuild_child._watch_parent(parent_pid=os.getppid(), interval=0.01)) |
| 303 | await asyncio.sleep(0.1) |
| 304 | live.cancel() |
| 305 | with pytest.raises(asyncio.CancelledError): |
| 306 | await live |
| 307 | assert died == [], "a child whose worker is alive keeps running" |
File History
2 commits
sha256:cb4b05e9d6caea5f351e3cb452ee6a1db8856169328ad96bfba9ee8e04b8afa6
fix(jobs): fence every execution of a job; an expired or su…
Sonnet 5.5
minor
⚠
4 days ago
sha256:217096701012b41f6362eec2e887220098537b3fc89eedd49d41eb11bb9e5e41
fix(worker): one prebuild host-wide, run in a short-lived c…
Sonnet 5.5
patch
4 days ago