gabriel / musehub public
test_intel_failure_hardening.py python
249 lines 13.2 KB
Raw
sha256:f9cda821d29cd59f4b95c1cf9cd667e1af3a8e40de55f505bcf50fdd0dc52b1d fix(intel): bounded failure recording, terminal resource-li… Sonnet 5.5 minor ⚠ breaking 3 days ago
1 """Failure persistence and retry classification — musehub#8 correctness hardening.
2
3 Localhost real-repository run: PostgreSQL was OOM-killed mid-ingest, the intel child exited with a traceback, and the worker's ``fail_job`` hit the same
4 recovering database. The error handler raised, the row stayed ``running``, and ``reclaim_stale_jobs`` would have re-run the same heavy job ~10 minutes later.
5
6 Run ONLY this file, against a private Postgres (never the live hub database)::
7
8 TEST_DATABASE_URL=postgresql+asyncpg://musehub:musehub@<private-host>:<port>/musehub_test python -P -m pytest tests/test_intel_failure_hardening.py
9
10 P1 ``fail_job`` rides out a short database outage (bounded retries) and records the failure.
11 P2 A database that never comes back: the worker gives up after a bounded number of tries, logs the job, and never marks it done.
12 P3 A hung ``fail_job`` is bounded by a per-try timeout.
13 P4 A persistence retry that lands after the job was reclaimed and re-claimed is fenced (changes nothing).
14 C1 Memory-limit, CPU-limit, wall-timeout and child-death failures of an intel job are terminal: no automatic repeat of the same workload.
15 C2 A child traceback of a database connection failure (what a PostgreSQL OOM looks like from the child) is terminal for an intel job.
16 C3 An ordinary job error stays retryable, bounded by the attempt count; non-intel jobs keep their semantics.
17 C4 After a terminal failure the job is never claimed again.
18 S1 A stale ``intel.code`` claim (worker died / could not record the failure) is failed, not re-run; other jobs are still reclaimed; attempts stay bounded.
19 """
20 from __future__ import annotations
21
22 import asyncio
23 import logging
24 from datetime import datetime, timedelta, timezone
25
26 import pytest
27 from sqlalchemy import select, update
28 from sqlalchemy.exc import OperationalError
29 from sqlalchemy.ext.asyncio import AsyncSession
30
31 from musehub import worker
32 from musehub.db import database
33 from musehub.db.musehub_jobs_models import MusehubBackgroundJob
34 from musehub.intel_supervisor import (
35 IntelChildCpuExceeded,
36 IntelChildDied,
37 IntelChildError,
38 IntelChildMemoryExceeded,
39 IntelChildTimeout,
40 )
41 from musehub.services import musehub_jobs as J
42 from tests.factories import create_repo
43 from tests.test_intel_job_state_machine import add_job, job, worker_db # noqa: F401 - worker_db: production engine on the test database
44
45
46 class RaisingChild:
47 def __init__(self, exc: BaseException) -> None:
48 self.exc = exc
49 self.calls = 0
50
51 async def run(self, *_a: object, **_k: object) -> tuple[str, dict]:
52 self.calls += 1
53 raise self.exc
54
55
56 def _outage() -> OperationalError:
57 return OperationalError("UPDATE musehub_background_jobs", {}, ConnectionRefusedError("the database system is in recovery mode"))
58
59
60 async def _run_intel(worker_db: AsyncSession, monkeypatch: pytest.MonkeyPatch, slug: str, exc: BaseException) -> tuple[str, RaisingChild]:
61 repo = await create_repo(worker_db, owner="gabriel", slug=slug)
62 job_id = await add_job(worker_db, repo.repo_id, "intel.code", "h")
63 child = RaisingChild(exc)
64 monkeypatch.setattr(worker, "_intel_child_for_worker", lambda: child)
65 monkeypatch.setenv("MUSEHUB_INTEL_WORKER_REPOS", repo.repo_id)
66 monkeypatch.setattr(worker, "_FAIL_PERSIST_DELAYS", (0.0, 0.0, 0.0))
67 assert await asyncio.wait_for(worker._process_one(), timeout=20) is True
68 return job_id, child
69
70
71 # ── C1 / C4: resource-limit failures are terminal ─────────────────────────────
72
73 @pytest.mark.asyncio
74 @pytest.mark.parametrize("exc", [
75 IntelChildMemoryExceeded("intel child used 900 MiB (ceiling 768 MiB)"),
76 IntelChildCpuExceeded("intel child used 901s of CPU (budget 900s)"),
77 IntelChildTimeout("intel job exceeded the wall-clock limit (1200s)"),
78 IntelChildDied("intel child ended without answering (signal 9)"),
79 ], ids=["memory", "cpu", "wall", "died"])
80 async def test_c1_c4_a_resource_limit_failure_is_terminal_and_never_re_run(worker_db, monkeypatch, exc) -> None: # noqa: F811
81 job_id, child = await _run_intel(worker_db, monkeypatch, f"term-{type(exc).__name__.lower()}", exc)
82 row = await job(worker_db, job_id)
83 assert (row.status, row.attempt) == ("failed", 1) and row.done_at is not None and str(exc)[:30] in (row.error or "")
84 assert await worker._process_one() is False # nothing left to claim: the same workload is not repeated
85 assert child.calls == 1
86
87
88 # ── C2: a database failure inside the child is terminal for intel jobs ────────
89
90 @pytest.mark.asyncio
91 @pytest.mark.parametrize("trace", [
92 "Traceback (most recent call last):\n ...\nasyncpg.exceptions.ConnectionDoesNotExistError: connection was closed in the middle of operation",
93 "sqlalchemy.exc.OperationalError: (asyncpg.exceptions.CrashShutdownError) terminating connection because of crash of another server process",
94 "ConnectionRefusedError: [Errno 111] Connect call failed ('172.18.0.2', 5432)",
95 "asyncpg.exceptions.CannotConnectNowError: the database system is in recovery mode",
96 ])
97 async def test_c2_a_database_connection_failure_in_the_child_is_terminal(worker_db, monkeypatch, trace: str) -> None: # noqa: F811
98 job_id, child = await _run_intel(worker_db, monkeypatch, "term-db", IntelChildError(trace))
99 row = await job(worker_db, job_id)
100 assert (row.status, row.attempt) == ("failed", 1)
101 assert await worker._process_one() is False and child.calls == 1
102
103
104 # ── C3: ordinary errors keep the bounded retry; other jobs do not change ──────
105
106 @pytest.mark.asyncio
107 async def test_c3_an_ordinary_child_error_is_still_retried_within_the_attempt_bound(worker_db, monkeypatch) -> None: # noqa: F811
108 job_id, child = await _run_intel(worker_db, monkeypatch, "retry-ordinary", IntelChildError("ValueError: bad provider input"))
109 assert (await job(worker_db, job_id)).status == "pending"
110 for _ in range(2):
111 assert await asyncio.wait_for(worker._process_one(), timeout=20) is True
112 row = await job(worker_db, job_id)
113 assert (row.status, row.attempt, child.calls) == ("failed", J._MAX_ATTEMPTS, J._MAX_ATTEMPTS)
114 assert await worker._process_one() is False
115
116
117 @pytest.mark.asyncio
118 async def test_c3_a_non_intel_job_failure_is_unchanged(worker_db, monkeypatch) -> None: # noqa: F811
119 repo = await create_repo(worker_db, owner="gabriel", slug="nonintel")
120 job_id = await add_job(worker_db, repo.repo_id, "mpack.index", "")
121
122 async def boom(*_a: object, **_k: object) -> dict:
123 raise ConnectionRefusedError("connection refused")
124
125 monkeypatch.setattr("musehub.services.musehub_wire.process_mpack_index_job", boom)
126 monkeypatch.setattr(worker, "_FAIL_PERSIST_DELAYS", (0.0,))
127 assert await asyncio.wait_for(worker._process_one(), timeout=20) is True
128 row = await job(worker_db, job_id)
129 assert (row.status, row.attempt) == ("pending", 1) # retried, exactly as before
130
131
132 @pytest.mark.asyncio
133 async def test_c3_the_lease_cap_of_an_intel_job_is_terminal(worker_db, monkeypatch) -> None: # noqa: F811
134 repo = await create_repo(worker_db, owner="gabriel", slug="cap-terminal")
135 job_id = await add_job(worker_db, repo.repo_id, "intel.code", "h")
136
137 class Hang:
138 async def run(self, *_a: object, **_k: object) -> tuple[str, dict]:
139 await asyncio.sleep(60)
140 raise AssertionError("unreachable")
141
142 monkeypatch.setattr(worker, "_intel_child_for_worker", lambda: Hang())
143 monkeypatch.setattr(worker, "_CLAIM_RENEW_SECONDS", 0.05)
144 monkeypatch.setattr(worker, "_CLAIM_MAX_SECONDS", 0.3)
145 monkeypatch.setenv("MUSEHUB_INTEL_WORKER_REPOS", repo.repo_id)
146 assert await asyncio.wait_for(worker._process_one(), timeout=10) is True
147 row = await job(worker_db, job_id)
148 assert row.status == "failed" and "lease cap" in (row.error or "")
149
150
151 # ── P1-P4: failure persistence under a database outage ────────────────────────
152
153 @pytest.mark.asyncio
154 async def test_p1_fail_job_rides_out_a_short_outage(worker_db, monkeypatch) -> None: # noqa: F811
155 real = J.fail_job
156 calls: list[int] = []
157
158 async def flaky(session: AsyncSession, job_id: str, error: str, **kw: object) -> bool:
159 calls.append(1)
160 if len(calls) <= 2:
161 raise _outage()
162 return await real(session, job_id, error, **kw) # type: ignore[arg-type]
163
164 monkeypatch.setattr(J, "fail_job", flaky)
165 job_id, _ = await _run_intel(worker_db, monkeypatch, "p1", IntelChildMemoryExceeded("over ceiling"))
166 row = await job(worker_db, job_id)
167 assert len(calls) == 3 and (row.status, row.attempt) == ("failed", 1) and "over ceiling" in (row.error or "")
168
169
170 @pytest.mark.asyncio
171 async def test_p2_a_database_that_never_recovers_is_bounded_logged_and_never_marks_the_job_done(worker_db, monkeypatch, caplog) -> None: # noqa: F811
172 calls: list[int] = []
173
174 async def down(*_a: object, **_k: object) -> bool:
175 calls.append(1)
176 raise _outage()
177
178 monkeypatch.setattr(J, "fail_job", down)
179 with caplog.at_level(logging.ERROR):
180 job_id, _ = await _run_intel(worker_db, monkeypatch, "p2", IntelChildMemoryExceeded("over ceiling"))
181 assert len(calls) == len(worker._FAIL_PERSIST_DELAYS) + 1 # bounded: the first try plus one per delay
182 row = await job(worker_db, job_id)
183 assert row.status == "running" and row.done_at is None # not complete, not silently failed: the lease sweep owns it
184 logged = "\n".join(r.getMessage() for r in caplog.records)
185 assert job_id[:16] in logged and "intel.code" in logged and "could not be recorded" in logged
186
187
188 @pytest.mark.asyncio
189 async def test_p3_a_hung_fail_job_is_bounded_by_a_per_try_timeout(worker_db, monkeypatch) -> None: # noqa: F811
190 async def hang(*_a: object, **_k: object) -> bool:
191 await asyncio.sleep(60)
192 return True
193
194 monkeypatch.setattr(J, "fail_job", hang)
195 monkeypatch.setattr(worker, "_FAIL_PERSIST_TRY_TIMEOUT", 0.05)
196 job_id, _ = await _run_intel(worker_db, monkeypatch, "p3", IntelChildMemoryExceeded("over ceiling")) # _run_intel itself fails after 20 s if unbounded
197 assert (await job(worker_db, job_id)).status == "running"
198
199
200 @pytest.mark.asyncio
201 async def test_p4_a_late_persistence_retry_cannot_touch_a_job_another_execution_now_owns(worker_db, monkeypatch) -> None: # noqa: F811
202 real = J.fail_job
203 state: dict[str, object] = {"n": 0}
204
205 async def outage_then_takeover(session: AsyncSession, job_id: str, error: str, *, attempt: int, retryable: bool) -> bool:
206 state["n"] = int(state["n"]) + 1 # type: ignore[call-overload]
207 if state["n"] == 1:
208 raise _outage()
209 if state["n"] == 2: # while the database was away the lease sweep reset the job and a second worker claimed it
210 async with database.AsyncSessionLocal() as s:
211 await s.execute(update(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == job_id).values(status="running", attempt=attempt + 1))
212 await s.commit()
213 return await real(session, job_id, error, attempt=attempt, retryable=retryable)
214
215 monkeypatch.setattr(J, "fail_job", outage_then_takeover)
216 job_id, _ = await _run_intel(worker_db, monkeypatch, "p4", IntelChildMemoryExceeded("over ceiling"))
217 row = await job(worker_db, job_id)
218 assert (row.status, row.attempt) == ("running", 2) # the new owner's execution is untouched
219
220
221 # ── S1: stale-claim reclamation ───────────────────────────────────────────────
222
223 async def _stale(s: AsyncSession, repo_id: str, job_type: str, attempt: int) -> str:
224 job_id = await add_job(s, repo_id, job_type, "h", status="running", attempt=attempt)
225 await s.execute(update(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == job_id)
226 .values(claimed_at=datetime.now(tz=timezone.utc) - timedelta(minutes=J._STALE_CLAIM_MINUTES + 1)))
227 await s.commit()
228 return job_id
229
230
231 @pytest.mark.asyncio
232 async def test_s1_a_stale_intel_code_claim_is_failed_not_re_run(db_session: AsyncSession) -> None:
233 repo = await create_repo(db_session, owner="gabriel", slug="stale-index")
234 heavy = await _stale(db_session, repo.repo_id, "intel.code", 1)
235 light = await _stale(db_session, repo.repo_id, "intel.code.dead", 1)
236 other = await _stale(db_session, repo.repo_id, "mpack.index", 1)
237 exhausted = await _stale(db_session, repo.repo_id, "mpack.index", J._MAX_ATTEMPTS)
238 assert await J.reclaim_stale_jobs(db_session) == 2 # only the two retryable rows are reset to pending
239 await db_session.commit()
240 rows = {}
241 for name, jid in (("heavy", heavy), ("light", light), ("other", other), ("exhausted", exhausted)):
242 r = await job(db_session, jid)
243 rows[name] = (r.status, r.error or "", r.done_at)
244 assert rows["heavy"][0] == "failed" and "stale" in rows["heavy"][1] and rows["heavy"][2] is not None
245 assert rows["light"][0] == "pending" and rows["other"][0] == "pending"
246 assert rows["exhausted"][0] == "failed"
247 assert await J.claim_next_job(db_session) is not None # the others are claimable again; the failed index job is not among them
248 claimed_types = {r.job_type for r in (await db_session.execute(select(MusehubBackgroundJob).where(MusehubBackgroundJob.status == "running"))).scalars()}
249 assert "intel.code" not in claimed_types
File History 1 commit
sha256:f9cda821d29cd59f4b95c1cf9cd667e1af3a8e40de55f505bcf50fdd0dc52b1d fix(intel): bounded failure recording, terminal resource-li… Sonnet 5.5 minor ⚠ 3 days ago