test_intel_failure_hardening.py
python
sha256:f9cda821d29cd59f4b95c1cf9cd667e1af3a8e40de55f505bcf50fdd0dc52b1d
fix(intel): bounded failure recording, terminal resource-li…
Sonnet 5.5
minor
⚠ breaking
3 days ago
| 1 | """Failure persistence and retry classification — musehub#8 correctness hardening. |
| 2 | |
| 3 | Localhost real-repository run: PostgreSQL was OOM-killed mid-ingest, the intel child exited with a traceback, and the worker's ``fail_job`` hit the same |
| 4 | recovering database. The error handler raised, the row stayed ``running``, and ``reclaim_stale_jobs`` would have re-run the same heavy job ~10 minutes later. |
| 5 | |
| 6 | Run ONLY this file, against a private Postgres (never the live hub database):: |
| 7 | |
| 8 | TEST_DATABASE_URL=postgresql+asyncpg://musehub:musehub@<private-host>:<port>/musehub_test python -P -m pytest tests/test_intel_failure_hardening.py |
| 9 | |
| 10 | P1 ``fail_job`` rides out a short database outage (bounded retries) and records the failure. |
| 11 | P2 A database that never comes back: the worker gives up after a bounded number of tries, logs the job, and never marks it done. |
| 12 | P3 A hung ``fail_job`` is bounded by a per-try timeout. |
| 13 | P4 A persistence retry that lands after the job was reclaimed and re-claimed is fenced (changes nothing). |
| 14 | C1 Memory-limit, CPU-limit, wall-timeout and child-death failures of an intel job are terminal: no automatic repeat of the same workload. |
| 15 | C2 A child traceback of a database connection failure (what a PostgreSQL OOM looks like from the child) is terminal for an intel job. |
| 16 | C3 An ordinary job error stays retryable, bounded by the attempt count; non-intel jobs keep their semantics. |
| 17 | C4 After a terminal failure the job is never claimed again. |
| 18 | S1 A stale ``intel.code`` claim (worker died / could not record the failure) is failed, not re-run; other jobs are still reclaimed; attempts stay bounded. |
| 19 | """ |
| 20 | from __future__ import annotations |
| 21 | |
| 22 | import asyncio |
| 23 | import logging |
| 24 | from datetime import datetime, timedelta, timezone |
| 25 | |
| 26 | import pytest |
| 27 | from sqlalchemy import select, update |
| 28 | from sqlalchemy.exc import OperationalError |
| 29 | from sqlalchemy.ext.asyncio import AsyncSession |
| 30 | |
| 31 | from musehub import worker |
| 32 | from musehub.db import database |
| 33 | from musehub.db.musehub_jobs_models import MusehubBackgroundJob |
| 34 | from musehub.intel_supervisor import ( |
| 35 | IntelChildCpuExceeded, |
| 36 | IntelChildDied, |
| 37 | IntelChildError, |
| 38 | IntelChildMemoryExceeded, |
| 39 | IntelChildTimeout, |
| 40 | ) |
| 41 | from musehub.services import musehub_jobs as J |
| 42 | from tests.factories import create_repo |
| 43 | from tests.test_intel_job_state_machine import add_job, job, worker_db # noqa: F401 - worker_db: production engine on the test database |
| 44 | |
| 45 | |
| 46 | class RaisingChild: |
| 47 | def __init__(self, exc: BaseException) -> None: |
| 48 | self.exc = exc |
| 49 | self.calls = 0 |
| 50 | |
| 51 | async def run(self, *_a: object, **_k: object) -> tuple[str, dict]: |
| 52 | self.calls += 1 |
| 53 | raise self.exc |
| 54 | |
| 55 | |
| 56 | def _outage() -> OperationalError: |
| 57 | return OperationalError("UPDATE musehub_background_jobs", {}, ConnectionRefusedError("the database system is in recovery mode")) |
| 58 | |
| 59 | |
| 60 | async def _run_intel(worker_db: AsyncSession, monkeypatch: pytest.MonkeyPatch, slug: str, exc: BaseException) -> tuple[str, RaisingChild]: |
| 61 | repo = await create_repo(worker_db, owner="gabriel", slug=slug) |
| 62 | job_id = await add_job(worker_db, repo.repo_id, "intel.code", "h") |
| 63 | child = RaisingChild(exc) |
| 64 | monkeypatch.setattr(worker, "_intel_child_for_worker", lambda: child) |
| 65 | monkeypatch.setenv("MUSEHUB_INTEL_WORKER_REPOS", repo.repo_id) |
| 66 | monkeypatch.setattr(worker, "_FAIL_PERSIST_DELAYS", (0.0, 0.0, 0.0)) |
| 67 | assert await asyncio.wait_for(worker._process_one(), timeout=20) is True |
| 68 | return job_id, child |
| 69 | |
| 70 | |
| 71 | # ── C1 / C4: resource-limit failures are terminal ───────────────────────────── |
| 72 | |
| 73 | @pytest.mark.asyncio |
| 74 | @pytest.mark.parametrize("exc", [ |
| 75 | IntelChildMemoryExceeded("intel child used 900 MiB (ceiling 768 MiB)"), |
| 76 | IntelChildCpuExceeded("intel child used 901s of CPU (budget 900s)"), |
| 77 | IntelChildTimeout("intel job exceeded the wall-clock limit (1200s)"), |
| 78 | IntelChildDied("intel child ended without answering (signal 9)"), |
| 79 | ], ids=["memory", "cpu", "wall", "died"]) |
| 80 | async def test_c1_c4_a_resource_limit_failure_is_terminal_and_never_re_run(worker_db, monkeypatch, exc) -> None: # noqa: F811 |
| 81 | job_id, child = await _run_intel(worker_db, monkeypatch, f"term-{type(exc).__name__.lower()}", exc) |
| 82 | row = await job(worker_db, job_id) |
| 83 | assert (row.status, row.attempt) == ("failed", 1) and row.done_at is not None and str(exc)[:30] in (row.error or "") |
| 84 | assert await worker._process_one() is False # nothing left to claim: the same workload is not repeated |
| 85 | assert child.calls == 1 |
| 86 | |
| 87 | |
| 88 | # ── C2: a database failure inside the child is terminal for intel jobs ──────── |
| 89 | |
| 90 | @pytest.mark.asyncio |
| 91 | @pytest.mark.parametrize("trace", [ |
| 92 | "Traceback (most recent call last):\n ...\nasyncpg.exceptions.ConnectionDoesNotExistError: connection was closed in the middle of operation", |
| 93 | "sqlalchemy.exc.OperationalError: (asyncpg.exceptions.CrashShutdownError) terminating connection because of crash of another server process", |
| 94 | "ConnectionRefusedError: [Errno 111] Connect call failed ('172.18.0.2', 5432)", |
| 95 | "asyncpg.exceptions.CannotConnectNowError: the database system is in recovery mode", |
| 96 | ]) |
| 97 | async def test_c2_a_database_connection_failure_in_the_child_is_terminal(worker_db, monkeypatch, trace: str) -> None: # noqa: F811 |
| 98 | job_id, child = await _run_intel(worker_db, monkeypatch, "term-db", IntelChildError(trace)) |
| 99 | row = await job(worker_db, job_id) |
| 100 | assert (row.status, row.attempt) == ("failed", 1) |
| 101 | assert await worker._process_one() is False and child.calls == 1 |
| 102 | |
| 103 | |
| 104 | # ── C3: ordinary errors keep the bounded retry; other jobs do not change ────── |
| 105 | |
| 106 | @pytest.mark.asyncio |
| 107 | async def test_c3_an_ordinary_child_error_is_still_retried_within_the_attempt_bound(worker_db, monkeypatch) -> None: # noqa: F811 |
| 108 | job_id, child = await _run_intel(worker_db, monkeypatch, "retry-ordinary", IntelChildError("ValueError: bad provider input")) |
| 109 | assert (await job(worker_db, job_id)).status == "pending" |
| 110 | for _ in range(2): |
| 111 | assert await asyncio.wait_for(worker._process_one(), timeout=20) is True |
| 112 | row = await job(worker_db, job_id) |
| 113 | assert (row.status, row.attempt, child.calls) == ("failed", J._MAX_ATTEMPTS, J._MAX_ATTEMPTS) |
| 114 | assert await worker._process_one() is False |
| 115 | |
| 116 | |
| 117 | @pytest.mark.asyncio |
| 118 | async def test_c3_a_non_intel_job_failure_is_unchanged(worker_db, monkeypatch) -> None: # noqa: F811 |
| 119 | repo = await create_repo(worker_db, owner="gabriel", slug="nonintel") |
| 120 | job_id = await add_job(worker_db, repo.repo_id, "mpack.index", "") |
| 121 | |
| 122 | async def boom(*_a: object, **_k: object) -> dict: |
| 123 | raise ConnectionRefusedError("connection refused") |
| 124 | |
| 125 | monkeypatch.setattr("musehub.services.musehub_wire.process_mpack_index_job", boom) |
| 126 | monkeypatch.setattr(worker, "_FAIL_PERSIST_DELAYS", (0.0,)) |
| 127 | assert await asyncio.wait_for(worker._process_one(), timeout=20) is True |
| 128 | row = await job(worker_db, job_id) |
| 129 | assert (row.status, row.attempt) == ("pending", 1) # retried, exactly as before |
| 130 | |
| 131 | |
| 132 | @pytest.mark.asyncio |
| 133 | async def test_c3_the_lease_cap_of_an_intel_job_is_terminal(worker_db, monkeypatch) -> None: # noqa: F811 |
| 134 | repo = await create_repo(worker_db, owner="gabriel", slug="cap-terminal") |
| 135 | job_id = await add_job(worker_db, repo.repo_id, "intel.code", "h") |
| 136 | |
| 137 | class Hang: |
| 138 | async def run(self, *_a: object, **_k: object) -> tuple[str, dict]: |
| 139 | await asyncio.sleep(60) |
| 140 | raise AssertionError("unreachable") |
| 141 | |
| 142 | monkeypatch.setattr(worker, "_intel_child_for_worker", lambda: Hang()) |
| 143 | monkeypatch.setattr(worker, "_CLAIM_RENEW_SECONDS", 0.05) |
| 144 | monkeypatch.setattr(worker, "_CLAIM_MAX_SECONDS", 0.3) |
| 145 | monkeypatch.setenv("MUSEHUB_INTEL_WORKER_REPOS", repo.repo_id) |
| 146 | assert await asyncio.wait_for(worker._process_one(), timeout=10) is True |
| 147 | row = await job(worker_db, job_id) |
| 148 | assert row.status == "failed" and "lease cap" in (row.error or "") |
| 149 | |
| 150 | |
| 151 | # ── P1-P4: failure persistence under a database outage ──────────────────────── |
| 152 | |
| 153 | @pytest.mark.asyncio |
| 154 | async def test_p1_fail_job_rides_out_a_short_outage(worker_db, monkeypatch) -> None: # noqa: F811 |
| 155 | real = J.fail_job |
| 156 | calls: list[int] = [] |
| 157 | |
| 158 | async def flaky(session: AsyncSession, job_id: str, error: str, **kw: object) -> bool: |
| 159 | calls.append(1) |
| 160 | if len(calls) <= 2: |
| 161 | raise _outage() |
| 162 | return await real(session, job_id, error, **kw) # type: ignore[arg-type] |
| 163 | |
| 164 | monkeypatch.setattr(J, "fail_job", flaky) |
| 165 | job_id, _ = await _run_intel(worker_db, monkeypatch, "p1", IntelChildMemoryExceeded("over ceiling")) |
| 166 | row = await job(worker_db, job_id) |
| 167 | assert len(calls) == 3 and (row.status, row.attempt) == ("failed", 1) and "over ceiling" in (row.error or "") |
| 168 | |
| 169 | |
| 170 | @pytest.mark.asyncio |
| 171 | async def test_p2_a_database_that_never_recovers_is_bounded_logged_and_never_marks_the_job_done(worker_db, monkeypatch, caplog) -> None: # noqa: F811 |
| 172 | calls: list[int] = [] |
| 173 | |
| 174 | async def down(*_a: object, **_k: object) -> bool: |
| 175 | calls.append(1) |
| 176 | raise _outage() |
| 177 | |
| 178 | monkeypatch.setattr(J, "fail_job", down) |
| 179 | with caplog.at_level(logging.ERROR): |
| 180 | job_id, _ = await _run_intel(worker_db, monkeypatch, "p2", IntelChildMemoryExceeded("over ceiling")) |
| 181 | assert len(calls) == len(worker._FAIL_PERSIST_DELAYS) + 1 # bounded: the first try plus one per delay |
| 182 | row = await job(worker_db, job_id) |
| 183 | assert row.status == "running" and row.done_at is None # not complete, not silently failed: the lease sweep owns it |
| 184 | logged = "\n".join(r.getMessage() for r in caplog.records) |
| 185 | assert job_id[:16] in logged and "intel.code" in logged and "could not be recorded" in logged |
| 186 | |
| 187 | |
| 188 | @pytest.mark.asyncio |
| 189 | async def test_p3_a_hung_fail_job_is_bounded_by_a_per_try_timeout(worker_db, monkeypatch) -> None: # noqa: F811 |
| 190 | async def hang(*_a: object, **_k: object) -> bool: |
| 191 | await asyncio.sleep(60) |
| 192 | return True |
| 193 | |
| 194 | monkeypatch.setattr(J, "fail_job", hang) |
| 195 | monkeypatch.setattr(worker, "_FAIL_PERSIST_TRY_TIMEOUT", 0.05) |
| 196 | job_id, _ = await _run_intel(worker_db, monkeypatch, "p3", IntelChildMemoryExceeded("over ceiling")) # _run_intel itself fails after 20 s if unbounded |
| 197 | assert (await job(worker_db, job_id)).status == "running" |
| 198 | |
| 199 | |
| 200 | @pytest.mark.asyncio |
| 201 | async def test_p4_a_late_persistence_retry_cannot_touch_a_job_another_execution_now_owns(worker_db, monkeypatch) -> None: # noqa: F811 |
| 202 | real = J.fail_job |
| 203 | state: dict[str, object] = {"n": 0} |
| 204 | |
| 205 | async def outage_then_takeover(session: AsyncSession, job_id: str, error: str, *, attempt: int, retryable: bool) -> bool: |
| 206 | state["n"] = int(state["n"]) + 1 # type: ignore[call-overload] |
| 207 | if state["n"] == 1: |
| 208 | raise _outage() |
| 209 | if state["n"] == 2: # while the database was away the lease sweep reset the job and a second worker claimed it |
| 210 | async with database.AsyncSessionLocal() as s: |
| 211 | await s.execute(update(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == job_id).values(status="running", attempt=attempt + 1)) |
| 212 | await s.commit() |
| 213 | return await real(session, job_id, error, attempt=attempt, retryable=retryable) |
| 214 | |
| 215 | monkeypatch.setattr(J, "fail_job", outage_then_takeover) |
| 216 | job_id, _ = await _run_intel(worker_db, monkeypatch, "p4", IntelChildMemoryExceeded("over ceiling")) |
| 217 | row = await job(worker_db, job_id) |
| 218 | assert (row.status, row.attempt) == ("running", 2) # the new owner's execution is untouched |
| 219 | |
| 220 | |
| 221 | # ── S1: stale-claim reclamation ─────────────────────────────────────────────── |
| 222 | |
| 223 | async def _stale(s: AsyncSession, repo_id: str, job_type: str, attempt: int) -> str: |
| 224 | job_id = await add_job(s, repo_id, job_type, "h", status="running", attempt=attempt) |
| 225 | await s.execute(update(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == job_id) |
| 226 | .values(claimed_at=datetime.now(tz=timezone.utc) - timedelta(minutes=J._STALE_CLAIM_MINUTES + 1))) |
| 227 | await s.commit() |
| 228 | return job_id |
| 229 | |
| 230 | |
| 231 | @pytest.mark.asyncio |
| 232 | async def test_s1_a_stale_intel_code_claim_is_failed_not_re_run(db_session: AsyncSession) -> None: |
| 233 | repo = await create_repo(db_session, owner="gabriel", slug="stale-index") |
| 234 | heavy = await _stale(db_session, repo.repo_id, "intel.code", 1) |
| 235 | light = await _stale(db_session, repo.repo_id, "intel.code.dead", 1) |
| 236 | other = await _stale(db_session, repo.repo_id, "mpack.index", 1) |
| 237 | exhausted = await _stale(db_session, repo.repo_id, "mpack.index", J._MAX_ATTEMPTS) |
| 238 | assert await J.reclaim_stale_jobs(db_session) == 2 # only the two retryable rows are reset to pending |
| 239 | await db_session.commit() |
| 240 | rows = {} |
| 241 | for name, jid in (("heavy", heavy), ("light", light), ("other", other), ("exhausted", exhausted)): |
| 242 | r = await job(db_session, jid) |
| 243 | rows[name] = (r.status, r.error or "", r.done_at) |
| 244 | assert rows["heavy"][0] == "failed" and "stale" in rows["heavy"][1] and rows["heavy"][2] is not None |
| 245 | assert rows["light"][0] == "pending" and rows["other"][0] == "pending" |
| 246 | assert rows["exhausted"][0] == "failed" |
| 247 | assert await J.claim_next_job(db_session) is not None # the others are claimable again; the failed index job is not among them |
| 248 | claimed_types = {r.job_type for r in (await db_session.execute(select(MusehubBackgroundJob).where(MusehubBackgroundJob.status == "running"))).scalars()} |
| 249 | assert "intel.code" not in claimed_types |
File History
1 commit
sha256:f9cda821d29cd59f4b95c1cf9cd667e1af3a8e40de55f505bcf50fdd0dc52b1d
fix(intel): bounded failure recording, terminal resource-li…
Sonnet 5.5
minor
⚠
3 days ago