test_intel_lease_fencing.py
python
sha256:f9cda821d29cd59f4b95c1cf9cd667e1af3a8e40de55f505bcf50fdd0dc52b1d
fix(intel): bounded failure recording, terminal resource-li…
Sonnet 5.5
minor
⚠ breaking
3 days ago
| 1 | """Execution fencing after lease reclaim — musehub#8 release gate (terminal-state updates, first publication). |
| 2 | |
| 3 | A running job could outlive its lease: ``reclaim_stale_jobs`` reset it to ``pending``, a second copy started, and the first copy could still overwrite the |
| 4 | outcome (``complete_job`` / ``skip_job`` / ``block_job`` / ``fail_job`` had no ``status='running'`` or claim fence) and publish. ``attempt`` increments at |
| 5 | every claim, so (job_id, status='running', attempt) identifies exactly one execution — a fencing generation that needs no migration. |
| 6 | |
| 7 | Run ONLY this file, against a private Postgres (never the live hub database):: |
| 8 | |
| 9 | TEST_DATABASE_URL=postgresql+asyncpg://musehub:musehub@<private-host>:<port>/musehub_test python -P -m pytest tests/test_intel_lease_fencing.py |
| 10 | |
| 11 | F1 A terminal update from a superseded execution changes nothing and says so (all four outcomes); the current execution's update applies. |
| 12 | F2 A stale failure cannot push a newer running execution back to ``pending``. |
| 13 | F3 A terminal update on a job that was reclaimed (pending again) changes nothing. |
| 14 | F4 A lease renewal from a superseded execution fails. |
| 15 | F5 A superseded execution cannot publish: its transaction is rolled back at the commit fence, so a reclaimed job's first publication writes nothing. |
| 16 | F6 Losing the lease cancels the running execution and writes no terminal state; the job stays with whoever owns it now. |
| 17 | F7 Reaching the lease cap cancels the running execution and records a terminal failure (a time limit is not retried: musehub#8) instead of leaving the job running forever. |
| 18 | """ |
| 19 | from __future__ import annotations |
| 20 | |
| 21 | import asyncio |
| 22 | |
| 23 | import pytest |
| 24 | from sqlalchemy import func, select, update |
| 25 | from sqlalchemy.ext.asyncio import AsyncSession |
| 26 | |
| 27 | from musehub import worker |
| 28 | from musehub.db import database |
| 29 | from musehub.db.musehub_intel_models import MusehubIntelLanguages |
| 30 | from musehub.db.musehub_jobs_models import MusehubBackgroundJob |
| 31 | from musehub.services import musehub_jobs as J |
| 32 | from tests.factories import create_repo |
| 33 | from tests.test_intel_job_state_machine import add_job, job, worker_db # noqa: F401 - worker_db: production engine on the test database |
| 34 | from tests.test_intel_snapshot_providers import world_with_side_branch |
| 35 | |
| 36 | |
| 37 | async def claim(s: AsyncSession) -> MusehubBackgroundJob: |
| 38 | claimed = await J.claim_next_job(s) |
| 39 | assert claimed is not None |
| 40 | await s.commit() |
| 41 | return claimed |
| 42 | |
| 43 | |
| 44 | async def reclaim_and_claim_again(s: AsyncSession, job_id: str) -> int: |
| 45 | """What a stale-lease sweep followed by a second worker's claim does: returns the new execution's attempt.""" |
| 46 | await s.execute(update(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == job_id).values(status="pending")) |
| 47 | await s.commit() |
| 48 | return (await claim(s)).attempt |
| 49 | |
| 50 | |
| 51 | @pytest.mark.asyncio |
| 52 | @pytest.mark.parametrize("outcome", ["complete", "skip", "block", "fail"]) |
| 53 | async def test_f1_a_superseded_execution_cannot_write_a_terminal_state(db_session: AsyncSession, outcome: str) -> None: |
| 54 | repo = await create_repo(db_session, owner="gabriel", slug=f"fence-{outcome}") |
| 55 | job_id = await add_job(db_session, repo.repo_id, "intel.code") |
| 56 | old = (await claim(db_session)).attempt |
| 57 | new = await reclaim_and_claim_again(db_session, job_id) |
| 58 | assert new == old + 1 |
| 59 | stale: dict[str, object] = { |
| 60 | "complete": lambda: J.complete_job(db_session, job_id, result={"x": 1}, attempt=old), |
| 61 | "skip": lambda: J.skip_job(db_session, job_id, "stale", attempt=old), |
| 62 | "block": lambda: J.block_job(db_session, job_id, "stale", attempt=old), |
| 63 | "fail": lambda: J.fail_job(db_session, job_id, "stale", attempt=old), |
| 64 | } |
| 65 | assert await stale[outcome]() is False # type: ignore[operator] |
| 66 | await db_session.commit() |
| 67 | row = await job(db_session, job_id) |
| 68 | assert (row.status, row.attempt, row.result, row.error) == ("running", new, None, None) |
| 69 | assert await J.complete_job(db_session, job_id, result={"ok": True}, attempt=new) is True |
| 70 | await db_session.commit() |
| 71 | assert (await job(db_session, job_id)).status == "done" |
| 72 | |
| 73 | |
| 74 | @pytest.mark.asyncio |
| 75 | async def test_f2_a_stale_failure_cannot_reset_a_newer_running_execution(db_session: AsyncSession) -> None: |
| 76 | repo = await create_repo(db_session, owner="gabriel", slug="fence-f2") |
| 77 | job_id = await add_job(db_session, repo.repo_id, "intel.code") |
| 78 | old = (await claim(db_session)).attempt |
| 79 | new = await reclaim_and_claim_again(db_session, job_id) |
| 80 | await J.fail_job(db_session, job_id, "late failure of the first execution", attempt=old) |
| 81 | await db_session.commit() |
| 82 | row = await job(db_session, job_id) |
| 83 | assert (row.status, row.attempt) == ("running", new) # not 'pending': that would let a THIRD copy start beside the second |
| 84 | |
| 85 | |
| 86 | @pytest.mark.asyncio |
| 87 | async def test_f3_a_terminal_update_on_a_reclaimed_pending_job_changes_nothing(db_session: AsyncSession) -> None: |
| 88 | repo = await create_repo(db_session, owner="gabriel", slug="fence-f3") |
| 89 | job_id = await add_job(db_session, repo.repo_id, "intel.code") |
| 90 | attempt = (await claim(db_session)).attempt |
| 91 | await db_session.execute(update(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == job_id).values(status="pending")) |
| 92 | await db_session.commit() |
| 93 | assert await J.complete_job(db_session, job_id, result={}, attempt=attempt) is False |
| 94 | await db_session.commit() |
| 95 | assert (await job(db_session, job_id)).status == "pending" |
| 96 | |
| 97 | |
| 98 | @pytest.mark.asyncio |
| 99 | async def test_f4_a_superseded_execution_cannot_renew_the_lease(db_session: AsyncSession) -> None: |
| 100 | repo = await create_repo(db_session, owner="gabriel", slug="fence-f4") |
| 101 | job_id = await add_job(db_session, repo.repo_id, "intel.code") |
| 102 | old = (await claim(db_session)).attempt |
| 103 | assert await J.renew_job_claim(db_session, job_id, attempt=old) is True |
| 104 | new = await reclaim_and_claim_again(db_session, job_id) |
| 105 | assert await J.renew_job_claim(db_session, job_id, attempt=old) is False |
| 106 | assert await J.renew_job_claim(db_session, job_id, attempt=new) is True |
| 107 | |
| 108 | |
| 109 | @pytest.mark.asyncio |
| 110 | async def test_f5_a_superseded_execution_publishes_nothing(worker_db) -> None: # noqa: F811 |
| 111 | w, c2, _side = await world_with_side_branch(worker_db, "fence-f5") |
| 112 | job_id = await add_job(worker_db, w.repo_id, "intel.code.languages", c2) |
| 113 | old = (await claim(worker_db)).attempt |
| 114 | await reclaim_and_claim_again(worker_db, job_id) # a second execution owns the job now |
| 115 | status, result = await worker._run_intel_job("intel.code.languages", w.repo_id, {"head": c2, "branch": "main"}, |
| 116 | fence={"job_id": job_id, "attempt": old}) |
| 117 | assert (status, result["executed"]) == ("skipped", False) and "fenced" in str(result["reason"]) |
| 118 | async with database.AsyncSessionLocal() as s: |
| 119 | assert (await s.execute(select(func.count()).select_from(MusehubIntelLanguages).where(MusehubIntelLanguages.repo_id == w.repo_id))).scalar_one() == 0 |
| 120 | # the current execution is not fenced |
| 121 | current = await job(worker_db, job_id) |
| 122 | status, _ = await worker._run_intel_job("intel.code.languages", w.repo_id, {"head": c2, "branch": "main"}, |
| 123 | fence={"job_id": job_id, "attempt": current.attempt}) |
| 124 | assert status == "done" |
| 125 | |
| 126 | |
| 127 | class SlowChild: |
| 128 | """A child whose job never finishes on its own; records whether it was cancelled.""" |
| 129 | |
| 130 | def __init__(self) -> None: |
| 131 | self.cancelled = False |
| 132 | self.started = asyncio.Event() |
| 133 | |
| 134 | async def run(self, job_type, repo_id, payload, fence=None): |
| 135 | self.started.set() |
| 136 | try: |
| 137 | await asyncio.sleep(30) |
| 138 | except asyncio.CancelledError: |
| 139 | self.cancelled = True |
| 140 | raise |
| 141 | return "done", {} |
| 142 | |
| 143 | async def close(self) -> None: |
| 144 | return None |
| 145 | |
| 146 | |
| 147 | @pytest.mark.asyncio |
| 148 | async def test_f6_losing_the_lease_cancels_the_execution_and_writes_no_terminal_state(worker_db, monkeypatch) -> None: # noqa: F811 |
| 149 | repo = await create_repo(worker_db, owner="gabriel", slug="fence-f6") |
| 150 | job_id = await add_job(worker_db, repo.repo_id, "intel.code", "h") |
| 151 | child = SlowChild() |
| 152 | monkeypatch.setattr(worker, "_intel_child_for_worker", lambda: child) |
| 153 | monkeypatch.setattr(worker, "_CLAIM_RENEW_SECONDS", 0.05) |
| 154 | monkeypatch.setenv("MUSEHUB_INTEL_WORKER_REPOS", repo.repo_id) |
| 155 | |
| 156 | async def steal() -> None: |
| 157 | await child.started.wait() |
| 158 | async with database.AsyncSessionLocal() as s: # the stale-lease sweep resets the job while the first execution still runs |
| 159 | await s.execute(update(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == job_id).values(status="pending")) |
| 160 | await s.commit() |
| 161 | |
| 162 | stealer = asyncio.create_task(steal()) |
| 163 | assert await asyncio.wait_for(worker._process_one(), timeout=10) is True |
| 164 | await stealer |
| 165 | assert child.cancelled is True |
| 166 | row = await job(worker_db, job_id) |
| 167 | assert row.status == "pending" and row.error is None # untouched by the superseded execution |
| 168 | |
| 169 | |
| 170 | @pytest.mark.asyncio |
| 171 | async def test_f7_the_lease_cap_cancels_the_execution_and_records_a_terminal_failure(worker_db, monkeypatch) -> None: # noqa: F811 |
| 172 | repo = await create_repo(worker_db, owner="gabriel", slug="fence-f7") |
| 173 | job_id = await add_job(worker_db, repo.repo_id, "intel.code", "h") |
| 174 | child = SlowChild() |
| 175 | monkeypatch.setattr(worker, "_intel_child_for_worker", lambda: child) |
| 176 | monkeypatch.setattr(worker, "_CLAIM_RENEW_SECONDS", 0.05) |
| 177 | monkeypatch.setattr(worker, "_CLAIM_MAX_SECONDS", 0.3) |
| 178 | monkeypatch.setenv("MUSEHUB_INTEL_WORKER_REPOS", repo.repo_id) |
| 179 | assert await asyncio.wait_for(worker._process_one(), timeout=10) is True |
| 180 | assert child.cancelled is True |
| 181 | row = await job(worker_db, job_id) |
| 182 | assert row.status == "failed" and "lease cap" in (row.error or "") # attempt 1 of 3, but a time limit would only repeat: not retried |
File History
3 commits
sha256:f9cda821d29cd59f4b95c1cf9cd667e1af3a8e40de55f505bcf50fdd0dc52b1d
fix(intel): bounded failure recording, terminal resource-li…
Sonnet 5.5
minor
⚠
3 days ago
sha256:3733f942311ecd1beecb0d6f112f3ebd87b2801a5a410bf35b2415a84a238f1b
fix(intel): one-shot child, reap confirmed before the refer…
Sonnet 5.5
minor
⚠
3 days ago
sha256:cb4b05e9d6caea5f351e3cb452ee6a1db8856169328ad96bfba9ee8e04b8afa6
fix(jobs): fence every execution of a job; an expired or su…
Sonnet 5.5
minor
⚠
4 days ago