test_job_claim_lease.py
python
sha256:1c4df3b9e9cf3f6fafd636a97366487274883c16a3d578e971e3ddcee10da0f3
fix(worker): the lease cancels only the execution's own tas…
Sonnet 5.5
minor
⚠ breaking
4 days ago
| 1 | """A running job's claim is a lease the worker keeps renewing (found live on staging, 2026-11-12). |
| 2 | |
| 3 | Every app slot runs an embedded job worker next to the dedicated worker container (by design, see |
| 4 | test_db_connection_budget). ``reclaim_stale_jobs`` resets any 'running' job whose ``claimed_at`` is |
| 5 | older than ``_STALE_CLAIM_MINUTES`` (10) — meant for crashed workers — but nothing renewed the claim |
| 6 | while a job was still running. On staging a gabriel/musehub ``fetch.mpack.prebuild`` (rc.2, ~20 min) |
| 7 | was claimed by blue's embedded worker at 16:12:57Z; at 16:22:57Z the dedicated worker "reclaimed" the |
| 8 | live job and ran a second full build concurrently (both finished: 16:32:41 and 16:42:39). Contract: |
| 9 | |
| 10 | * JCL_01 a renewed claim is not reclaimed, even when the job was first claimed long ago; |
| 11 | * JCL_02 a crashed worker's claim (never renewed) is still reclaimed; |
| 12 | * JCL_03 the worker renews the claim periodically while a job runs, and stops when it ends. |
| 13 | """ |
| 14 | from __future__ import annotations |
| 15 | |
| 16 | import asyncio |
| 17 | from datetime import datetime, timedelta, timezone |
| 18 | |
| 19 | import pytest |
| 20 | from sqlalchemy import select |
| 21 | from sqlalchemy.ext.asyncio import AsyncSession |
| 22 | |
| 23 | from musehub.core.genesis import compute_job_id |
| 24 | from musehub.db.musehub_jobs_models import MusehubBackgroundJob |
| 25 | from musehub.services import musehub_jobs |
| 26 | from musehub.services.musehub_jobs import _STALE_CLAIM_MINUTES, reclaim_stale_jobs |
| 27 | from tests.factories import create_repo |
| 28 | |
| 29 | |
| 30 | async def _running_job(session: AsyncSession, repo_id: str, claimed_minutes_ago: int, seed: str) -> str: |
| 31 | when = datetime.now(tz=timezone.utc) - timedelta(minutes=claimed_minutes_ago) |
| 32 | job_id = compute_job_id(repo_id, "fetch.mpack.prebuild", when.isoformat() + seed) |
| 33 | session.add(MusehubBackgroundJob(job_id=job_id, repo_id=repo_id, job_type="fetch.mpack.prebuild", |
| 34 | payload={}, status="running", created_at=when, claimed_at=when, attempt=1)) |
| 35 | await session.commit() |
| 36 | return job_id |
| 37 | |
| 38 | |
| 39 | async def _status(session: AsyncSession, job_id: str) -> str: |
| 40 | session.expire_all() |
| 41 | return (await session.execute(select(MusehubBackgroundJob.status).where(MusehubBackgroundJob.job_id == job_id))).scalar_one() |
| 42 | |
| 43 | |
| 44 | @pytest.mark.asyncio |
| 45 | async def test_jcl_01_renewed_claim_is_not_reclaimed(db_session: AsyncSession) -> None: |
| 46 | repo = await create_repo(db_session) |
| 47 | job_id = await _running_job(db_session, repo.repo_id, _STALE_CLAIM_MINUTES + 10, "live") |
| 48 | assert await musehub_jobs.renew_job_claim(db_session, job_id, attempt=1) is True |
| 49 | await db_session.commit() |
| 50 | await reclaim_stale_jobs(db_session) |
| 51 | await db_session.commit() |
| 52 | assert await _status(db_session, job_id) == "running" |
| 53 | |
| 54 | |
| 55 | @pytest.mark.asyncio |
| 56 | async def test_jcl_02_unrenewed_claim_is_still_reclaimed(db_session: AsyncSession) -> None: |
| 57 | repo = await create_repo(db_session) |
| 58 | job_id = await _running_job(db_session, repo.repo_id, _STALE_CLAIM_MINUTES + 10, "crashed") |
| 59 | await reclaim_stale_jobs(db_session) |
| 60 | await db_session.commit() |
| 61 | assert await _status(db_session, job_id) == "pending" |
| 62 | |
| 63 | |
| 64 | @pytest.mark.asyncio |
| 65 | async def test_jcl_03_worker_renews_while_the_job_runs(monkeypatch: pytest.MonkeyPatch) -> None: |
| 66 | import musehub.worker as worker |
| 67 | |
| 68 | assert 0 < worker._CLAIM_RENEW_SECONDS * 3 <= 60 * _STALE_CLAIM_MINUTES, "renew well inside the stale cutoff" |
| 69 | calls: list[str] = [] |
| 70 | |
| 71 | async def fake_renew(job_id: str, attempt: int) -> bool: |
| 72 | calls.append(job_id) |
| 73 | return True |
| 74 | |
| 75 | monkeypatch.setattr(worker, "_CLAIM_RENEW_SECONDS", 0.02) |
| 76 | monkeypatch.setattr(worker, "_renew_claim_once", fake_renew) |
| 77 | await worker._run_leased("sha256:job", 1, asyncio.sleep(0.15), cancel_on_end=True) |
| 78 | n = len(calls) |
| 79 | assert n >= 3 and set(calls) == {"sha256:job"}, calls |
| 80 | await asyncio.sleep(0.08) |
| 81 | assert len(calls) == n, "the lease must stop renewing once the job has ended" |
| 82 | |
| 83 | |
| 84 | @pytest.mark.asyncio |
| 85 | @pytest.mark.parametrize("final_status", ["done", "failed"]) |
| 86 | async def test_jcl_04_renewal_never_resurrects_a_finished_job(db_session: AsyncSession, final_status: str) -> None: |
| 87 | repo = await create_repo(db_session) |
| 88 | job_id = await _running_job(db_session, repo.repo_id, 1, final_status) |
| 89 | row = (await db_session.execute(select(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == job_id))).scalar_one() |
| 90 | row.status = final_status |
| 91 | await db_session.commit() |
| 92 | assert await musehub_jobs.renew_job_claim(db_session, job_id, attempt=row.attempt) is False |
| 93 | await db_session.commit() |
| 94 | assert await _status(db_session, job_id) == final_status |
| 95 | |
| 96 | |
| 97 | @pytest.mark.asyncio |
| 98 | async def test_jcl_05_a_failing_renewal_is_swallowed(monkeypatch: pytest.MonkeyPatch) -> None: |
| 99 | import musehub.worker as worker |
| 100 | |
| 101 | async def boom(session: object, job_id: str, *, attempt: int) -> bool: |
| 102 | raise RuntimeError("db unavailable") |
| 103 | |
| 104 | monkeypatch.setattr(musehub_jobs, "renew_job_claim", boom) |
| 105 | assert await worker._renew_claim_once("sha256:job", 1) is True # must not raise, and must not claim the lease is lost |
| 106 | |
| 107 | |
| 108 | @pytest.mark.asyncio |
| 109 | async def test_jcl_06_the_cap_cancels_a_hung_execution_so_it_cannot_overlap_its_replacement(monkeypatch: pytest.MonkeyPatch) -> None: |
| 110 | import musehub.worker as worker |
| 111 | |
| 112 | calls: list[str] = [] |
| 113 | |
| 114 | async def fake_renew(job_id: str, attempt: int) -> bool: |
| 115 | calls.append(job_id) |
| 116 | return True |
| 117 | |
| 118 | monkeypatch.setattr(worker, "_CLAIM_RENEW_SECONDS", 0.02) |
| 119 | monkeypatch.setattr(worker, "_CLAIM_MAX_SECONDS", 0.07) |
| 120 | monkeypatch.setattr(worker, "_renew_claim_once", fake_renew) |
| 121 | ended, result = await worker._run_leased("sha256:hung", 1, asyncio.sleep(0.3), cancel_on_end=True) |
| 122 | assert result is None and ended.reason == "cap" # the cap cancels the hung supervised execution itself (it used to leave it running) |
| 123 | assert 1 <= len(calls) <= 3, calls |
| 124 | assert worker._CLAIM_MAX_SECONDS < 0.1 |
| 125 | |
| 126 | |
| 127 | def test_jcl_07_cap_is_well_above_the_stale_cutoff_and_renewal_well_below_it() -> None: |
| 128 | import importlib |
| 129 | import musehub.worker as worker |
| 130 | |
| 131 | fresh = importlib.reload(worker) |
| 132 | assert fresh._CLAIM_RENEW_SECONDS * 3 <= 60 * _STALE_CLAIM_MINUTES < fresh._CLAIM_MAX_SECONDS |
File History
5 commits
sha256:1c4df3b9e9cf3f6fafd636a97366487274883c16a3d578e971e3ddcee10da0f3
fix(worker): the lease cancels only the execution's own tas…
Sonnet 5.5
minor
⚠
4 days ago
sha256:cb4b05e9d6caea5f351e3cb452ee6a1db8856169328ad96bfba9ee8e04b8afa6
fix(jobs): fence every execution of a job; an expired or su…
Sonnet 5.5
minor
⚠
4 days ago
sha256:57cb3fc1975a4f3b093a5bd577f3ee8dd312716221155cee07f77960d420bf0e
Merge task/job-claim-lease: job claim is a renewed, capped …
Opus 5.5
patch
4 days ago
sha256:7006c99f3d033385ec7a94f8ffeba55ce8246b8f540a9eb3102364ac7ac0c4ef
fix(jobs): cap claim-lease renewal so a hung job stays recl…
Opus 5.5
4 days ago
sha256:af2897d03be5eeb5121ab5faf50ef629399c62eb340498d2e29c9e46d39ea9b3
fix(jobs): a running job's claim is a lease the worker rene…
Opus 5.5
patch
4 days ago