gabriel / musehub public
test_intel_tier_a_fixes.py python
296 lines 15.0 KB
Raw
sha256:f9cda821d29cd59f4b95c1cf9cd667e1af3a8e40de55f505bcf50fdd0dc52b1d fix(intel): bounded failure recording, terminal resource-li… Sonnet 5.5 minor ⚠ breaking 3 days ago
1 """Fixes for the independent Tier A review of musehub#8 (B1 lease cancellation escaping the worker loop, E1 child/prebuild memory overlap, H1 non-intel behaviour).
2
3 Run ONLY this file, against a private Postgres (never the live hub database)::
4
5 TEST_DATABASE_URL=postgresql+asyncpg://musehub:musehub@<private-host>:<port>/musehub_test python -P -m pytest tests/test_intel_tier_a_fixes.py
6
7 B1 The lease never cancels the worker's own task. A lease that ends (cap / lost) while the terminal state is being recorded - on the success path or in the
8 error handler - cannot raise ``CancelledError`` out of ``_process_one`` (which would end the worker loop), and the recorded outcome stays the truthful one.
9 E1 A non-intel job that the worker executes starts only after an idle intel child has been shut down and reaped (the child holds hundreds of MiB of caches).
10 H1 ``push.unpack`` / ``mpack.index`` keep their pre-hardening lease semantics: renewed, never cancelled by the lease, and a superseded outcome is discarded by the
11 attempt fence. Only supervised child jobs (``intel.*``, the prebuild child) are cancelled when the lease ends.
12 """
13 from __future__ import annotations
14
15 import asyncio
16
17 import pytest
18 from sqlalchemy import update
19
20 from musehub import worker
21 from musehub.db import database
22 from musehub.db.musehub_jobs_models import MusehubBackgroundJob
23 from musehub.services import musehub_jobs as J
24 from tests.factories import create_repo
25 from tests.test_intel_job_state_machine import add_job, job, worker_db # noqa: F401 - worker_db: production engine on the test database
26
27
28 class QuickChild:
29 """An intel child whose job answers at once (or raises), and that records closing."""
30
31 def __init__(self, *, fail: bool = False) -> None:
32 self.fail = fail
33 self.closed = False
34
35 async def run(self, job_type, repo_id, payload, fence=None):
36 if self.fail:
37 raise RuntimeError("provider exploded")
38 return "done", {"executed": True, "outcome": "computed"}
39
40 async def close(self) -> None:
41 self.closed = True
42
43
44 def slow(real, delay: float):
45 """Wrap a job-state writer so the write takes ``delay`` seconds: the lease can end while the terminal state is being recorded."""
46 async def wrapper(*args, **kwargs):
47 await asyncio.sleep(delay)
48 return await real(*args, **kwargs)
49 return wrapper
50
51
52 def quick_lease(monkeypatch, *, renew: float = 0.05, cap: float = 0.1) -> None:
53 monkeypatch.setattr(worker, "_CLAIM_RENEW_SECONDS", renew)
54 monkeypatch.setattr(worker, "_CLAIM_MAX_SECONDS", cap)
55
56
57 # --------------------------------------------------------------------------------------------------------------------------------------------- B1
58
59
60 @pytest.mark.asyncio
61 async def test_b1_a_lease_that_ends_in_the_error_handler_cannot_kill_the_worker_loop(worker_db, monkeypatch) -> None: # noqa: F811
62 repo = await create_repo(worker_db, owner="gabriel", slug="tier-a-b1-error")
63 job_id = await add_job(worker_db, repo.repo_id, "intel.code", "h")
64 monkeypatch.setattr(worker, "_intel_child_for_worker", lambda: QuickChild(fail=True))
65 monkeypatch.setenv("MUSEHUB_INTEL_WORKER_REPOS", repo.repo_id)
66 monkeypatch.setenv("MUSEHUB_DEPLOYMENT", "local")
67 quick_lease(monkeypatch) # the cap is reached while fail_job is still writing
68 monkeypatch.setattr(J, "fail_job", slow(J.fail_job, 0.3))
69 assert await asyncio.wait_for(worker._process_one(), timeout=10) is True # used to raise CancelledError: the loop's `except Exception` does not catch it
70 row = await job(worker_db, job_id)
71 assert row.status == "pending" and "provider exploded" in (row.error or "") # the real failure, not a lease artefact; retried (attempt 1 of 3)
72
73
74 @pytest.mark.asyncio
75 async def test_b1_a_lease_that_ends_while_completing_does_not_turn_success_into_failure(worker_db, monkeypatch) -> None: # noqa: F811
76 repo = await create_repo(worker_db, owner="gabriel", slug="tier-a-b1-done")
77 job_id = await add_job(worker_db, repo.repo_id, "intel.code", "h")
78 monkeypatch.setattr(worker, "_intel_child_for_worker", lambda: QuickChild())
79 monkeypatch.setenv("MUSEHUB_INTEL_WORKER_REPOS", repo.repo_id)
80 monkeypatch.setenv("MUSEHUB_DEPLOYMENT", "local")
81 quick_lease(monkeypatch)
82 monkeypatch.setattr(J, "complete_job", slow(J.complete_job, 0.3))
83 assert await asyncio.wait_for(worker._process_one(), timeout=10) is True
84 row = await job(worker_db, job_id)
85 assert (row.status, row.error) == ("done", None) # the work finished: that is what is recorded, not "lease cap"
86
87
88 @pytest.mark.asyncio
89 async def test_b1_the_loop_survives_a_lease_ending_during_terminal_recording_and_takes_the_next_job(worker_db, monkeypatch) -> None: # noqa: F811
90 repo = await create_repo(worker_db, owner="gabriel", slug="tier-a-b1-loop")
91 first = await add_job(worker_db, repo.repo_id, "intel.code", "h", age=10)
92 second = await add_job(worker_db, repo.repo_id, "intel.code.languages", "h", age=5)
93 monkeypatch.setattr(worker, "_intel_child_for_worker", lambda: QuickChild())
94 monkeypatch.setenv("MUSEHUB_INTEL_WORKER_REPOS", repo.repo_id)
95 monkeypatch.setenv("MUSEHUB_DEPLOYMENT", "local")
96 quick_lease(monkeypatch)
97 monkeypatch.setattr(J, "complete_job", slow(J.complete_job, 0.25))
98 for _ in range(2):
99 assert await asyncio.wait_for(worker._process_one(), timeout=10) is True
100 assert (await job(worker_db, first)).status == "done" and (await job(worker_db, second)).status == "done"
101
102
103 @pytest.mark.asyncio
104 async def test_b1_an_execution_cancelled_by_the_lease_and_a_slow_failure_write_still_returns(worker_db, monkeypatch) -> None: # noqa: F811
105 """Cap reached mid-execution: the execution is cancelled, and the failure (terminal for an intel job: a time limit is not retried, musehub#8) is recorded even though that write is slow."""
106 repo = await create_repo(worker_db, owner="gabriel", slug="tier-a-b1-cap")
107 job_id = await add_job(worker_db, repo.repo_id, "intel.code", "h")
108
109 class Hung(QuickChild):
110 async def run(self, *a, **k):
111 await asyncio.sleep(30)
112
113 monkeypatch.setattr(worker, "_intel_child_for_worker", lambda: Hung())
114 monkeypatch.setenv("MUSEHUB_INTEL_WORKER_REPOS", repo.repo_id)
115 monkeypatch.setenv("MUSEHUB_DEPLOYMENT", "local")
116 quick_lease(monkeypatch, cap=0.15)
117 monkeypatch.setattr(J, "fail_job", slow(J.fail_job, 0.2))
118 assert await asyncio.wait_for(worker._process_one(), timeout=10) is True
119 row = await job(worker_db, job_id)
120 assert row.status == "failed" and "lease cap" in (row.error or "")
121
122
123 @pytest.mark.asyncio
124 async def test_b1_cancelling_the_worker_itself_still_propagates(worker_db, monkeypatch) -> None: # noqa: F811
125 repo = await create_repo(worker_db, owner="gabriel", slug="tier-a-b1-stop")
126 await add_job(worker_db, repo.repo_id, "intel.code", "h")
127 started = asyncio.Event()
128 seen: list[str] = []
129
130 class Hung(QuickChild):
131 async def run(self, *a, **k):
132 started.set()
133 try:
134 await asyncio.sleep(30)
135 except asyncio.CancelledError:
136 seen.append("child cancelled")
137 raise
138
139 monkeypatch.setattr(worker, "_intel_child_for_worker", lambda: Hung())
140 monkeypatch.setenv("MUSEHUB_INTEL_WORKER_REPOS", repo.repo_id)
141 monkeypatch.setenv("MUSEHUB_DEPLOYMENT", "local")
142 task = asyncio.create_task(worker._process_one())
143 await asyncio.wait_for(started.wait(), timeout=10)
144 task.cancel()
145 with pytest.raises(asyncio.CancelledError):
146 await task # a shutdown is not swallowed by the lease machinery
147 assert seen == ["child cancelled"]
148
149
150 # --------------------------------------------------------------------------------------------------------------------------------------------- E1
151
152
153 @pytest.mark.asyncio
154 @pytest.mark.parametrize("job_type", ["mpack.index", "push.unpack", "fetch.mpack.prebuild"])
155 async def test_e1_the_idle_intel_child_is_gone_before_a_memory_heavy_job_starts(worker_db, monkeypatch, job_type) -> None: # noqa: F811
156 repo = await create_repo(worker_db, owner="gabriel", slug=f"tier-a-e1-{job_type.replace('.', '-')}")
157 await add_job(worker_db, repo.repo_id, job_type)
158 child = QuickChild()
159 monkeypatch.setattr(worker, "_intel_child", child)
160 closed_when_started: list[bool] = []
161
162 async def fake_prebuild(job_id: str):
163 closed_when_started.append(child.closed)
164 return {"tips_requested": 0, "tips_built": 0, "tips_skipped": 0, "elapsed_ms": 0}
165
166 async def fake_index(session, job_id):
167 closed_when_started.append(child.closed)
168 return {"mpack_index_written": 0, "byte_ranges_computed": 0, "elapsed_ms": 0}
169
170 async def fake_unpack(session, job_id):
171 closed_when_started.append(child.closed)
172 return {"branch": "main", "head": "h", "blobs_written": 0, "commits_written": 0}
173
174 import musehub.services.musehub_wire as wire
175 import musehub.services.musehub_wire_push as wire_push
176 monkeypatch.setattr(worker, "_run_prebuild_child", fake_prebuild)
177 monkeypatch.setattr(wire, "process_mpack_index_job", fake_index)
178 monkeypatch.setattr(wire_push, "process_push_unpack_job", fake_unpack)
179 assert await asyncio.wait_for(worker._process_one(), timeout=10) is True
180 assert closed_when_started == [True], "the intel child must be shut down and reaped BEFORE the job's work begins"
181 assert worker._intel_child is None
182
183
184 @pytest.mark.asyncio
185 async def test_e1_an_intel_job_keeps_the_child_and_a_skipped_job_does_not_cost_its_caches(worker_db, monkeypatch) -> None: # noqa: F811
186 repo = await create_repo(worker_db, owner="gabriel", slug="tier-a-e1-keep")
187 await add_job(worker_db, repo.repo_id, "gc", age=10) # not executed by this worker: ends 'skipped', allocates nothing
188 await add_job(worker_db, repo.repo_id, "intel.code", "h", age=5)
189 child = QuickChild()
190 monkeypatch.setattr(worker, "_intel_child", child)
191 monkeypatch.setattr(worker, "_intel_child_for_worker", lambda: child)
192 monkeypatch.setenv("MUSEHUB_INTEL_WORKER_REPOS", repo.repo_id)
193 monkeypatch.setenv("MUSEHUB_DEPLOYMENT", "local")
194 for _ in range(2):
195 assert await asyncio.wait_for(worker._process_one(), timeout=10) is True
196 assert child.closed is False and worker._intel_child is child
197
198
199 @pytest.mark.asyncio
200 async def test_e1_a_child_that_cannot_be_reaped_stops_the_job_from_starting(worker_db, monkeypatch) -> None: # noqa: F811
201 repo = await create_repo(worker_db, owner="gabriel", slug="tier-a-e1-fail")
202 job_id = await add_job(worker_db, repo.repo_id, "mpack.index")
203
204 class Stuck(QuickChild):
205 stuck = True
206
207 async def close(self) -> None:
208 if self.stuck:
209 raise OSError("cannot reap")
210 await super().close()
211
212 started: list[str] = []
213
214 async def fake_index(session, job_id):
215 started.append(job_id)
216 return {}
217
218 import musehub.services.musehub_wire as wire
219 child = Stuck()
220 monkeypatch.setattr(worker, "_intel_child", child)
221 monkeypatch.setattr(wire, "process_mpack_index_job", fake_index)
222 assert await asyncio.wait_for(worker._process_one(), timeout=10) is True
223 assert started == [] # fail closed: no memory-heavy work beside a child that may still be alive
224 row = await job(worker_db, job_id)
225 assert row.status == "pending" and "cannot reap" in (row.error or "")
226 assert worker._intel_child is child # still remembered: a child that may be alive is never forgotten
227 child.stuck = False # let the suite's teardown close it
228
229
230 # --------------------------------------------------------------------------------------------------------------------------------------------- H1
231
232
233 @pytest.mark.asyncio
234 @pytest.mark.parametrize("job_type", ["mpack.index", "push.unpack"])
235 async def test_h1_in_process_jobs_are_not_cancelled_when_the_lease_cap_is_reached(worker_db, monkeypatch, job_type) -> None: # noqa: F811
236 repo = await create_repo(worker_db, owner="gabriel", slug=f"tier-a-h1-cap-{job_type.replace('.', '-')}")
237 job_id = await add_job(worker_db, repo.repo_id, job_type)
238 quick_lease(monkeypatch, renew=0.03, cap=0.08)
239 finished: list[str] = []
240
241 async def work(session, jid):
242 await asyncio.sleep(0.3) # well past the cap: before the hardening this simply ran on and completed
243 finished.append(jid)
244 return {"branch": "main", "head": "h", "blobs_written": 0, "commits_written": 0, "mpack_index_written": 0, "byte_ranges_computed": 0, "elapsed_ms": 0}
245
246 import musehub.services.musehub_wire as wire
247 import musehub.services.musehub_wire_push as wire_push
248 monkeypatch.setattr(wire, "process_mpack_index_job", work)
249 monkeypatch.setattr(wire_push, "process_push_unpack_job", work)
250 assert await asyncio.wait_for(worker._process_one(), timeout=10) is True
251 assert finished == [job_id] # it ran to the end
252 row = await job(worker_db, job_id)
253 assert (row.status, row.error) == ("done", None)
254
255
256 @pytest.mark.asyncio
257 async def test_h1_an_in_process_job_whose_lease_was_lost_runs_on_and_its_outcome_is_discarded_by_the_fence(worker_db, monkeypatch) -> None: # noqa: F811
258 repo = await create_repo(worker_db, owner="gabriel", slug="tier-a-h1-lost")
259 job_id = await add_job(worker_db, repo.repo_id, "mpack.index")
260 monkeypatch.setattr(worker, "_CLAIM_RENEW_SECONDS", 0.03)
261
262 async def work(session, jid):
263 async with database.AsyncSessionLocal() as s: # the stale sweep reclaims the job under the running execution
264 await s.execute(update(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == jid).values(status="pending"))
265 await s.commit()
266 await asyncio.sleep(0.2)
267 return {"mpack_index_written": 0, "byte_ranges_computed": 0, "elapsed_ms": 0}
268
269 import musehub.services.musehub_wire as wire
270 monkeypatch.setattr(wire, "process_mpack_index_job", work)
271 assert await asyncio.wait_for(worker._process_one(), timeout=10) is True # no CancelledError, no crash
272 row = await job(worker_db, job_id)
273 assert row.status == "pending" and row.result is None # the superseded outcome was fenced off
274
275
276 @pytest.mark.asyncio
277 async def test_h1_a_prebuild_child_is_still_cancelled_when_its_lease_is_lost(worker_db, monkeypatch) -> None: # noqa: F811
278 repo = await create_repo(worker_db, owner="gabriel", slug="tier-a-h1-prebuild")
279 job_id = await add_job(worker_db, repo.repo_id, "fetch.mpack.prebuild")
280 monkeypatch.setattr(worker, "_CLAIM_RENEW_SECONDS", 0.03)
281 cancelled: list[bool] = []
282
283 async def fake_prebuild(jid: str):
284 async with database.AsyncSessionLocal() as s:
285 await s.execute(update(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == jid).values(status="pending"))
286 await s.commit()
287 try:
288 await asyncio.sleep(30)
289 except asyncio.CancelledError:
290 cancelled.append(True) # the supervised child is killed here, so no second build can overlap the replacement
291 raise
292
293 monkeypatch.setattr(worker, "_run_prebuild_child", fake_prebuild)
294 assert await asyncio.wait_for(worker._process_one(), timeout=10) is True
295 assert cancelled == [True]
296 assert (await job(worker_db, job_id)).status == "pending"
File History 3 commits
sha256:f9cda821d29cd59f4b95c1cf9cd667e1af3a8e40de55f505bcf50fdd0dc52b1d fix(intel): bounded failure recording, terminal resource-li… Sonnet 5.5 minor ⚠ 3 days ago
sha256:3733f942311ecd1beecb0d6f112f3ebd87b2801a5a410bf35b2415a84a238f1b fix(intel): one-shot child, reap confirmed before the refer… Sonnet 5.5 minor ⚠ 3 days ago
sha256:1c4df3b9e9cf3f6fafd636a97366487274883c16a3d578e971e3ddcee10da0f3 fix(worker): the lease cancels only the execution's own tas… Sonnet 5.5 minor ⚠ 3 days ago