gabriel / musehub public
test_intel_lease_fencing.py python
182 lines 9.6 KB
Raw
sha256:f9cda821d29cd59f4b95c1cf9cd667e1af3a8e40de55f505bcf50fdd0dc52b1d fix(intel): bounded failure recording, terminal resource-li… Sonnet 5.5 minor ⚠ breaking 3 days ago
1 """Execution fencing after lease reclaim — musehub#8 release gate (terminal-state updates, first publication).
2
3 A running job could outlive its lease: ``reclaim_stale_jobs`` reset it to ``pending``, a second copy started, and the first copy could still overwrite the
4 outcome (``complete_job`` / ``skip_job`` / ``block_job`` / ``fail_job`` had no ``status='running'`` or claim fence) and publish. ``attempt`` increments at
5 every claim, so (job_id, status='running', attempt) identifies exactly one execution — a fencing generation that needs no migration.
6
7 Run ONLY this file, against a private Postgres (never the live hub database)::
8
9 TEST_DATABASE_URL=postgresql+asyncpg://musehub:musehub@<private-host>:<port>/musehub_test python -P -m pytest tests/test_intel_lease_fencing.py
10
11 F1 A terminal update from a superseded execution changes nothing and says so (all four outcomes); the current execution's update applies.
12 F2 A stale failure cannot push a newer running execution back to ``pending``.
13 F3 A terminal update on a job that was reclaimed (pending again) changes nothing.
14 F4 A lease renewal from a superseded execution fails.
15 F5 A superseded execution cannot publish: its transaction is rolled back at the commit fence, so a reclaimed job's first publication writes nothing.
16 F6 Losing the lease cancels the running execution and writes no terminal state; the job stays with whoever owns it now.
17 F7 Reaching the lease cap cancels the running execution and records a terminal failure (a time limit is not retried: musehub#8) instead of leaving the job running forever.
18 """
19 from __future__ import annotations
20
21 import asyncio
22
23 import pytest
24 from sqlalchemy import func, select, update
25 from sqlalchemy.ext.asyncio import AsyncSession
26
27 from musehub import worker
28 from musehub.db import database
29 from musehub.db.musehub_intel_models import MusehubIntelLanguages
30 from musehub.db.musehub_jobs_models import MusehubBackgroundJob
31 from musehub.services import musehub_jobs as J
32 from tests.factories import create_repo
33 from tests.test_intel_job_state_machine import add_job, job, worker_db # noqa: F401 - worker_db: production engine on the test database
34 from tests.test_intel_snapshot_providers import world_with_side_branch
35
36
37 async def claim(s: AsyncSession) -> MusehubBackgroundJob:
38 claimed = await J.claim_next_job(s)
39 assert claimed is not None
40 await s.commit()
41 return claimed
42
43
44 async def reclaim_and_claim_again(s: AsyncSession, job_id: str) -> int:
45 """What a stale-lease sweep followed by a second worker's claim does: returns the new execution's attempt."""
46 await s.execute(update(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == job_id).values(status="pending"))
47 await s.commit()
48 return (await claim(s)).attempt
49
50
51 @pytest.mark.asyncio
52 @pytest.mark.parametrize("outcome", ["complete", "skip", "block", "fail"])
53 async def test_f1_a_superseded_execution_cannot_write_a_terminal_state(db_session: AsyncSession, outcome: str) -> None:
54 repo = await create_repo(db_session, owner="gabriel", slug=f"fence-{outcome}")
55 job_id = await add_job(db_session, repo.repo_id, "intel.code")
56 old = (await claim(db_session)).attempt
57 new = await reclaim_and_claim_again(db_session, job_id)
58 assert new == old + 1
59 stale: dict[str, object] = {
60 "complete": lambda: J.complete_job(db_session, job_id, result={"x": 1}, attempt=old),
61 "skip": lambda: J.skip_job(db_session, job_id, "stale", attempt=old),
62 "block": lambda: J.block_job(db_session, job_id, "stale", attempt=old),
63 "fail": lambda: J.fail_job(db_session, job_id, "stale", attempt=old),
64 }
65 assert await stale[outcome]() is False # type: ignore[operator]
66 await db_session.commit()
67 row = await job(db_session, job_id)
68 assert (row.status, row.attempt, row.result, row.error) == ("running", new, None, None)
69 assert await J.complete_job(db_session, job_id, result={"ok": True}, attempt=new) is True
70 await db_session.commit()
71 assert (await job(db_session, job_id)).status == "done"
72
73
74 @pytest.mark.asyncio
75 async def test_f2_a_stale_failure_cannot_reset_a_newer_running_execution(db_session: AsyncSession) -> None:
76 repo = await create_repo(db_session, owner="gabriel", slug="fence-f2")
77 job_id = await add_job(db_session, repo.repo_id, "intel.code")
78 old = (await claim(db_session)).attempt
79 new = await reclaim_and_claim_again(db_session, job_id)
80 await J.fail_job(db_session, job_id, "late failure of the first execution", attempt=old)
81 await db_session.commit()
82 row = await job(db_session, job_id)
83 assert (row.status, row.attempt) == ("running", new) # not 'pending': that would let a THIRD copy start beside the second
84
85
86 @pytest.mark.asyncio
87 async def test_f3_a_terminal_update_on_a_reclaimed_pending_job_changes_nothing(db_session: AsyncSession) -> None:
88 repo = await create_repo(db_session, owner="gabriel", slug="fence-f3")
89 job_id = await add_job(db_session, repo.repo_id, "intel.code")
90 attempt = (await claim(db_session)).attempt
91 await db_session.execute(update(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == job_id).values(status="pending"))
92 await db_session.commit()
93 assert await J.complete_job(db_session, job_id, result={}, attempt=attempt) is False
94 await db_session.commit()
95 assert (await job(db_session, job_id)).status == "pending"
96
97
98 @pytest.mark.asyncio
99 async def test_f4_a_superseded_execution_cannot_renew_the_lease(db_session: AsyncSession) -> None:
100 repo = await create_repo(db_session, owner="gabriel", slug="fence-f4")
101 job_id = await add_job(db_session, repo.repo_id, "intel.code")
102 old = (await claim(db_session)).attempt
103 assert await J.renew_job_claim(db_session, job_id, attempt=old) is True
104 new = await reclaim_and_claim_again(db_session, job_id)
105 assert await J.renew_job_claim(db_session, job_id, attempt=old) is False
106 assert await J.renew_job_claim(db_session, job_id, attempt=new) is True
107
108
109 @pytest.mark.asyncio
110 async def test_f5_a_superseded_execution_publishes_nothing(worker_db) -> None: # noqa: F811
111 w, c2, _side = await world_with_side_branch(worker_db, "fence-f5")
112 job_id = await add_job(worker_db, w.repo_id, "intel.code.languages", c2)
113 old = (await claim(worker_db)).attempt
114 await reclaim_and_claim_again(worker_db, job_id) # a second execution owns the job now
115 status, result = await worker._run_intel_job("intel.code.languages", w.repo_id, {"head": c2, "branch": "main"},
116 fence={"job_id": job_id, "attempt": old})
117 assert (status, result["executed"]) == ("skipped", False) and "fenced" in str(result["reason"])
118 async with database.AsyncSessionLocal() as s:
119 assert (await s.execute(select(func.count()).select_from(MusehubIntelLanguages).where(MusehubIntelLanguages.repo_id == w.repo_id))).scalar_one() == 0
120 # the current execution is not fenced
121 current = await job(worker_db, job_id)
122 status, _ = await worker._run_intel_job("intel.code.languages", w.repo_id, {"head": c2, "branch": "main"},
123 fence={"job_id": job_id, "attempt": current.attempt})
124 assert status == "done"
125
126
127 class SlowChild:
128 """A child whose job never finishes on its own; records whether it was cancelled."""
129
130 def __init__(self) -> None:
131 self.cancelled = False
132 self.started = asyncio.Event()
133
134 async def run(self, job_type, repo_id, payload, fence=None):
135 self.started.set()
136 try:
137 await asyncio.sleep(30)
138 except asyncio.CancelledError:
139 self.cancelled = True
140 raise
141 return "done", {}
142
143 async def close(self) -> None:
144 return None
145
146
147 @pytest.mark.asyncio
148 async def test_f6_losing_the_lease_cancels_the_execution_and_writes_no_terminal_state(worker_db, monkeypatch) -> None: # noqa: F811
149 repo = await create_repo(worker_db, owner="gabriel", slug="fence-f6")
150 job_id = await add_job(worker_db, repo.repo_id, "intel.code", "h")
151 child = SlowChild()
152 monkeypatch.setattr(worker, "_intel_child_for_worker", lambda: child)
153 monkeypatch.setattr(worker, "_CLAIM_RENEW_SECONDS", 0.05)
154 monkeypatch.setenv("MUSEHUB_INTEL_WORKER_REPOS", repo.repo_id)
155
156 async def steal() -> None:
157 await child.started.wait()
158 async with database.AsyncSessionLocal() as s: # the stale-lease sweep resets the job while the first execution still runs
159 await s.execute(update(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == job_id).values(status="pending"))
160 await s.commit()
161
162 stealer = asyncio.create_task(steal())
163 assert await asyncio.wait_for(worker._process_one(), timeout=10) is True
164 await stealer
165 assert child.cancelled is True
166 row = await job(worker_db, job_id)
167 assert row.status == "pending" and row.error is None # untouched by the superseded execution
168
169
170 @pytest.mark.asyncio
171 async def test_f7_the_lease_cap_cancels_the_execution_and_records_a_terminal_failure(worker_db, monkeypatch) -> None: # noqa: F811
172 repo = await create_repo(worker_db, owner="gabriel", slug="fence-f7")
173 job_id = await add_job(worker_db, repo.repo_id, "intel.code", "h")
174 child = SlowChild()
175 monkeypatch.setattr(worker, "_intel_child_for_worker", lambda: child)
176 monkeypatch.setattr(worker, "_CLAIM_RENEW_SECONDS", 0.05)
177 monkeypatch.setattr(worker, "_CLAIM_MAX_SECONDS", 0.3)
178 monkeypatch.setenv("MUSEHUB_INTEL_WORKER_REPOS", repo.repo_id)
179 assert await asyncio.wait_for(worker._process_one(), timeout=10) is True
180 assert child.cancelled is True
181 row = await job(worker_db, job_id)
182 assert row.status == "failed" and "lease cap" in (row.error or "") # attempt 1 of 3, but a time limit would only repeat: not retried
File History 3 commits
sha256:f9cda821d29cd59f4b95c1cf9cd667e1af3a8e40de55f505bcf50fdd0dc52b1d fix(intel): bounded failure recording, terminal resource-li… Sonnet 5.5 minor ⚠ 3 days ago
sha256:3733f942311ecd1beecb0d6f112f3ebd87b2801a5a410bf35b2415a84a238f1b fix(intel): one-shot child, reap confirmed before the refer… Sonnet 5.5 minor ⚠ 3 days ago
sha256:cb4b05e9d6caea5f351e3cb452ee6a1db8856169328ad96bfba9ee8e04b8afa6 fix(jobs): fence every execution of a job; an expired or su… Sonnet 5.5 minor ⚠ 4 days ago