gabriel / musehub public
test_job_claim_lease.py python
132 lines 6.0 KB
Raw
sha256:1c4df3b9e9cf3f6fafd636a97366487274883c16a3d578e971e3ddcee10da0f3 fix(worker): the lease cancels only the execution's own tas… Sonnet 5.5 minor ⚠ breaking 3 days ago
1 """A running job's claim is a lease the worker keeps renewing (found live on staging, 2026-11-12).
2
3 Every app slot runs an embedded job worker next to the dedicated worker container (by design, see
4 test_db_connection_budget). ``reclaim_stale_jobs`` resets any 'running' job whose ``claimed_at`` is
5 older than ``_STALE_CLAIM_MINUTES`` (10) — meant for crashed workers — but nothing renewed the claim
6 while a job was still running. On staging a gabriel/musehub ``fetch.mpack.prebuild`` (rc.2, ~20 min)
7 was claimed by blue's embedded worker at 16:12:57Z; at 16:22:57Z the dedicated worker "reclaimed" the
8 live job and ran a second full build concurrently (both finished: 16:32:41 and 16:42:39). Contract:
9
10 * JCL_01 a renewed claim is not reclaimed, even when the job was first claimed long ago;
11 * JCL_02 a crashed worker's claim (never renewed) is still reclaimed;
12 * JCL_03 the worker renews the claim periodically while a job runs, and stops when it ends.
13 """
14 from __future__ import annotations
15
16 import asyncio
17 from datetime import datetime, timedelta, timezone
18
19 import pytest
20 from sqlalchemy import select
21 from sqlalchemy.ext.asyncio import AsyncSession
22
23 from musehub.core.genesis import compute_job_id
24 from musehub.db.musehub_jobs_models import MusehubBackgroundJob
25 from musehub.services import musehub_jobs
26 from musehub.services.musehub_jobs import _STALE_CLAIM_MINUTES, reclaim_stale_jobs
27 from tests.factories import create_repo
28
29
30 async def _running_job(session: AsyncSession, repo_id: str, claimed_minutes_ago: int, seed: str) -> str:
31 when = datetime.now(tz=timezone.utc) - timedelta(minutes=claimed_minutes_ago)
32 job_id = compute_job_id(repo_id, "fetch.mpack.prebuild", when.isoformat() + seed)
33 session.add(MusehubBackgroundJob(job_id=job_id, repo_id=repo_id, job_type="fetch.mpack.prebuild",
34 payload={}, status="running", created_at=when, claimed_at=when, attempt=1))
35 await session.commit()
36 return job_id
37
38
39 async def _status(session: AsyncSession, job_id: str) -> str:
40 session.expire_all()
41 return (await session.execute(select(MusehubBackgroundJob.status).where(MusehubBackgroundJob.job_id == job_id))).scalar_one()
42
43
44 @pytest.mark.asyncio
45 async def test_jcl_01_renewed_claim_is_not_reclaimed(db_session: AsyncSession) -> None:
46 repo = await create_repo(db_session)
47 job_id = await _running_job(db_session, repo.repo_id, _STALE_CLAIM_MINUTES + 10, "live")
48 assert await musehub_jobs.renew_job_claim(db_session, job_id, attempt=1) is True
49 await db_session.commit()
50 await reclaim_stale_jobs(db_session)
51 await db_session.commit()
52 assert await _status(db_session, job_id) == "running"
53
54
55 @pytest.mark.asyncio
56 async def test_jcl_02_unrenewed_claim_is_still_reclaimed(db_session: AsyncSession) -> None:
57 repo = await create_repo(db_session)
58 job_id = await _running_job(db_session, repo.repo_id, _STALE_CLAIM_MINUTES + 10, "crashed")
59 await reclaim_stale_jobs(db_session)
60 await db_session.commit()
61 assert await _status(db_session, job_id) == "pending"
62
63
64 @pytest.mark.asyncio
65 async def test_jcl_03_worker_renews_while_the_job_runs(monkeypatch: pytest.MonkeyPatch) -> None:
66 import musehub.worker as worker
67
68 assert 0 < worker._CLAIM_RENEW_SECONDS * 3 <= 60 * _STALE_CLAIM_MINUTES, "renew well inside the stale cutoff"
69 calls: list[str] = []
70
71 async def fake_renew(job_id: str, attempt: int) -> bool:
72 calls.append(job_id)
73 return True
74
75 monkeypatch.setattr(worker, "_CLAIM_RENEW_SECONDS", 0.02)
76 monkeypatch.setattr(worker, "_renew_claim_once", fake_renew)
77 await worker._run_leased("sha256:job", 1, asyncio.sleep(0.15), cancel_on_end=True)
78 n = len(calls)
79 assert n >= 3 and set(calls) == {"sha256:job"}, calls
80 await asyncio.sleep(0.08)
81 assert len(calls) == n, "the lease must stop renewing once the job has ended"
82
83
84 @pytest.mark.asyncio
85 @pytest.mark.parametrize("final_status", ["done", "failed"])
86 async def test_jcl_04_renewal_never_resurrects_a_finished_job(db_session: AsyncSession, final_status: str) -> None:
87 repo = await create_repo(db_session)
88 job_id = await _running_job(db_session, repo.repo_id, 1, final_status)
89 row = (await db_session.execute(select(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == job_id))).scalar_one()
90 row.status = final_status
91 await db_session.commit()
92 assert await musehub_jobs.renew_job_claim(db_session, job_id, attempt=row.attempt) is False
93 await db_session.commit()
94 assert await _status(db_session, job_id) == final_status
95
96
97 @pytest.mark.asyncio
98 async def test_jcl_05_a_failing_renewal_is_swallowed(monkeypatch: pytest.MonkeyPatch) -> None:
99 import musehub.worker as worker
100
101 async def boom(session: object, job_id: str, *, attempt: int) -> bool:
102 raise RuntimeError("db unavailable")
103
104 monkeypatch.setattr(musehub_jobs, "renew_job_claim", boom)
105 assert await worker._renew_claim_once("sha256:job", 1) is True # must not raise, and must not claim the lease is lost
106
107
108 @pytest.mark.asyncio
109 async def test_jcl_06_the_cap_cancels_a_hung_execution_so_it_cannot_overlap_its_replacement(monkeypatch: pytest.MonkeyPatch) -> None:
110 import musehub.worker as worker
111
112 calls: list[str] = []
113
114 async def fake_renew(job_id: str, attempt: int) -> bool:
115 calls.append(job_id)
116 return True
117
118 monkeypatch.setattr(worker, "_CLAIM_RENEW_SECONDS", 0.02)
119 monkeypatch.setattr(worker, "_CLAIM_MAX_SECONDS", 0.07)
120 monkeypatch.setattr(worker, "_renew_claim_once", fake_renew)
121 ended, result = await worker._run_leased("sha256:hung", 1, asyncio.sleep(0.3), cancel_on_end=True)
122 assert result is None and ended.reason == "cap" # the cap cancels the hung supervised execution itself (it used to leave it running)
123 assert 1 <= len(calls) <= 3, calls
124 assert worker._CLAIM_MAX_SECONDS < 0.1
125
126
127 def test_jcl_07_cap_is_well_above_the_stale_cutoff_and_renewal_well_below_it() -> None:
128 import importlib
129 import musehub.worker as worker
130
131 fresh = importlib.reload(worker)
132 assert fresh._CLAIM_RENEW_SECONDS * 3 <= 60 * _STALE_CLAIM_MINUTES < fresh._CLAIM_MAX_SECONDS
File History 5 commits
sha256:1c4df3b9e9cf3f6fafd636a97366487274883c16a3d578e971e3ddcee10da0f3 fix(worker): the lease cancels only the execution's own tas… Sonnet 5.5 minor ⚠ 3 days ago
sha256:cb4b05e9d6caea5f351e3cb452ee6a1db8856169328ad96bfba9ee8e04b8afa6 fix(jobs): fence every execution of a job; an expired or su… Sonnet 5.5 minor ⚠ 4 days ago
sha256:57cb3fc1975a4f3b093a5bd577f3ee8dd312716221155cee07f77960d420bf0e Merge task/job-claim-lease: job claim is a renewed, capped … Opus 5.5 patch 4 days ago
sha256:7006c99f3d033385ec7a94f8ffeba55ce8246b8f540a9eb3102364ac7ac0c4ef fix(jobs): cap claim-lease renewal so a hung job stays recl… Opus 5.5 4 days ago
sha256:af2897d03be5eeb5121ab5faf50ef629399c62eb340498d2e29c9e46d39ea9b3 fix(jobs): a running job's claim is a lease the worker rene… Opus 5.5 patch 4 days ago