test_intel_concurrency.py
python
sha256:8ab5dc9648872e524e32a32f68a3cada8b3e45c3d16931aa57e9a8a8178dc022
test+fix(intel): real two-session concurrency tests; serial…
Sonnet 5.5
4 days ago
| 1 | """Real two-session concurrency for the job queue and intel publication — musehub#8 release gate. |
| 2 | |
| 3 | The barrier tests in test_intel_job_state_machine.py call ``claim_next_job`` one session at a time. These tests run two genuinely concurrent database |
| 4 | sessions (separate connections, ``asyncio.gather``) so the properties they check hold under real interleaving, not by construction. |
| 5 | |
| 6 | Run ONLY this file, against a private Postgres (never the live hub database):: |
| 7 | |
| 8 | TEST_DATABASE_URL=postgresql+asyncpg://musehub:musehub@<private-host>:<port>/musehub_test python -P -m pytest tests/test_intel_concurrency.py |
| 9 | |
| 10 | K1 Simultaneous claimers: one pending job is claimed by exactly one of them, every round. |
| 11 | K2 Two pending jobs, two claimers: never the same row twice; both get claimed. |
| 12 | K3 Advisory-lock contention: a claimer that cannot take the claim lock returns at once (it never blocks, never claims) and succeeds after the holder commits. |
| 13 | K4 A claim racing the enqueue-time refresh: the newest push is always covered, a running job is never retargeted, no stale pending job survives. |
| 14 | K5 Reversed completion order under real concurrency: the older revision finishes last and cannot overwrite the newer one's published result. |
| 15 | K6 Failure and retry across two workers: bounded by ``_MAX_ATTEMPTS``, never claimed concurrently, then terminal. |
| 16 | """ |
| 17 | from __future__ import annotations |
| 18 | |
| 19 | import asyncio |
| 20 | from collections.abc import AsyncGenerator |
| 21 | from datetime import datetime, timezone |
| 22 | |
| 23 | import pytest |
| 24 | import pytest_asyncio |
| 25 | from sqlalchemy import func, select, text |
| 26 | from sqlalchemy.ext.asyncio import AsyncSession |
| 27 | |
| 28 | from musehub import worker |
| 29 | from musehub.db import database |
| 30 | from musehub.db.musehub_intel_models import MusehubIntelLanguages, MusehubIntelResult |
| 31 | from musehub.db.musehub_jobs_models import MusehubBackgroundJob |
| 32 | from musehub.services import musehub_intel_providers as P |
| 33 | from musehub.services import musehub_jobs as J |
| 34 | from tests.factories import create_repo |
| 35 | from tests.job_execution import current_attempt |
| 36 | from tests.test_intel_snapshot_providers import A_V1, B_SRC, World |
| 37 | |
| 38 | ROUNDS = 25 |
| 39 | |
| 40 | |
| 41 | @pytest_asyncio.fixture |
| 42 | async def pool(db_session: AsyncSession, monkeypatch: pytest.MonkeyPatch) -> AsyncGenerator[AsyncSession, None]: |
| 43 | """The production engine on the test database with room for several concurrent sessions.""" |
| 44 | from musehub.config import settings |
| 45 | from tests.conftest import _TEST_DATABASE_URL |
| 46 | |
| 47 | monkeypatch.setattr(settings, "database_url", _TEST_DATABASE_URL) |
| 48 | monkeypatch.setenv("DATABASE_URL", _TEST_DATABASE_URL) |
| 49 | await database.init_db(pool_size=6, max_overflow=2) |
| 50 | try: |
| 51 | yield db_session |
| 52 | finally: |
| 53 | await database.close_db() |
| 54 | |
| 55 | |
| 56 | async def put_job(repo_id: str, job_type: str, head: str = "h1", *, status: str = "pending", age: int = 0) -> str: |
| 57 | async with database.AsyncSessionLocal() as s: |
| 58 | job_id = J.compute_job_id(repo_id, job_type, f"{head}:{status}:{age}:{datetime.now(tz=timezone.utc).isoformat()}") |
| 59 | s.add(MusehubBackgroundJob(job_id=job_id, repo_id=repo_id, job_type=job_type, payload={"head": head, "branch": "main"}, status=status, |
| 60 | created_at=datetime.now(tz=timezone.utc), claimed_at=datetime.now(tz=timezone.utc) if status == "running" else None)) |
| 61 | await s.commit() |
| 62 | return job_id |
| 63 | |
| 64 | |
| 65 | async def try_claim() -> MusehubBackgroundJob | None: |
| 66 | async with database.AsyncSessionLocal() as s: |
| 67 | claimed = await J.claim_next_job(s) |
| 68 | await s.commit() |
| 69 | return claimed |
| 70 | |
| 71 | |
| 72 | async def jobs_of(repo_id: str, prefix: str) -> list[MusehubBackgroundJob]: |
| 73 | async with database.AsyncSessionLocal() as s: |
| 74 | return list((await s.execute(select(MusehubBackgroundJob).where(MusehubBackgroundJob.repo_id == repo_id, |
| 75 | MusehubBackgroundJob.job_type.like(f"{prefix}%")))).scalars()) |
| 76 | |
| 77 | |
| 78 | @pytest.mark.asyncio |
| 79 | async def test_k1_simultaneous_claimers_never_double_claim(pool) -> None: |
| 80 | repo = await create_repo(pool, owner="gabriel", slug="k1") |
| 81 | await pool.commit() |
| 82 | for i in range(ROUNDS): |
| 83 | job_id = await put_job(repo.repo_id, "gc", f"round{i}") |
| 84 | got = [c for c in await asyncio.gather(try_claim(), try_claim()) if c is not None] |
| 85 | assert [c.job_id for c in got] == [job_id], f"round {i}: {[c.job_id[:12] for c in got]}" |
| 86 | assert got[0].attempt == 1 |
| 87 | |
| 88 | |
| 89 | @pytest.mark.asyncio |
| 90 | async def test_k2_two_jobs_two_claimers_get_distinct_rows(pool) -> None: |
| 91 | a, b = await create_repo(pool, owner="gabriel", slug="k2a"), await create_repo(pool, owner="gabriel", slug="k2b") |
| 92 | await pool.commit() |
| 93 | for i in range(ROUNDS // 2): |
| 94 | wanted = {await put_job(a.repo_id, "gc", f"a{i}"), await put_job(b.repo_id, "gc", f"b{i}")} |
| 95 | got: dict[str, int] = {} |
| 96 | for _ in range(10): # the claim lock is try-lock: a loser returns None and polls again, exactly like the worker loop |
| 97 | for c in await asyncio.gather(try_claim(), try_claim()): |
| 98 | if c is not None: |
| 99 | assert c.job_id not in got, "the same row was claimed twice" |
| 100 | got[c.job_id] = c.attempt |
| 101 | if set(got) >= wanted: |
| 102 | break |
| 103 | assert set(got) >= wanted and all(att == 1 for jid, att in got.items() if jid in wanted) |
| 104 | async with database.AsyncSessionLocal() as s: # finish them so the next round starts clean |
| 105 | await s.execute(text("UPDATE musehub_background_jobs SET status='done' WHERE status='running'")) |
| 106 | await s.commit() |
| 107 | |
| 108 | |
| 109 | @pytest.mark.asyncio |
| 110 | async def test_k3_a_claimer_that_cannot_take_the_claim_lock_returns_at_once_and_succeeds_later(pool) -> None: |
| 111 | repo = await create_repo(pool, owner="gabriel", slug="k3") |
| 112 | await pool.commit() |
| 113 | job_id = await put_job(repo.repo_id, "gc") |
| 114 | async with database.AsyncSessionLocal() as holder: |
| 115 | await holder.execute(text(f"SELECT pg_advisory_xact_lock({J._CLAIM_LOCK_KEY})")) |
| 116 | blocked = await asyncio.wait_for(try_claim(), timeout=3) # a blocking claimer would hang here until the timeout |
| 117 | assert blocked is None |
| 118 | await holder.commit() |
| 119 | claimed = await try_claim() |
| 120 | assert claimed is not None and claimed.job_id == job_id |
| 121 | |
| 122 | |
| 123 | @pytest.mark.asyncio |
| 124 | async def test_k4_a_claim_racing_the_enqueue_refresh_never_loses_the_newest_push(pool) -> None: |
| 125 | for i in range(ROUNDS): |
| 126 | repo = await create_repo(pool, owner="gabriel", slug=f"k4-{i}") |
| 127 | await pool.commit() |
| 128 | await put_job(repo.repo_id, "intel.code", "h1") |
| 129 | |
| 130 | async def enqueue() -> None: |
| 131 | async with database.AsyncSessionLocal() as s: |
| 132 | await J.enqueue_push_intel(s, repo.repo_id, "h2", branch="main") |
| 133 | await s.commit() |
| 134 | |
| 135 | await asyncio.gather(try_claim(), enqueue()) |
| 136 | rows = await jobs_of(repo.repo_id, "intel.code") |
| 137 | code = [r for r in rows if r.job_type == "intel.code"] |
| 138 | shape = sorted((r.status, r.payload["head"]) for r in code) |
| 139 | allowed = [[("pending", "h2"), ("running", "h1")], [("running", "h2")], [("pending", "h2")]] |
| 140 | assert shape in allowed, f"round {i}: {shape}" # never [running h1] alone (h2 lost) and never a stale pending h1 |
| 141 | for r in rows: # a RUNNING job is never retargeted: its head is the one it was claimed with |
| 142 | if r.status == "running": |
| 143 | assert r.attempt == 1 |
| 144 | |
| 145 | |
| 146 | @pytest.mark.asyncio |
| 147 | async def test_k5_reversed_completion_under_real_concurrency_cannot_overwrite_the_newer_result(pool, monkeypatch) -> None: |
| 148 | """The newer revision is mid-transaction (its provider rows written, not yet committed) when the OLDER revision's job starts: the older job blocks |
| 149 | behind the newer one's row locks, then proceeds after the commit and must be rejected at its stale-write check — and its writes rolled back.""" |
| 150 | repo = await create_repo(pool, owner="gabriel", slug="k5") |
| 151 | w = World(pool, repo.repo_id) |
| 152 | c1 = await w.commit("c1", {"pkg/a.py": A_V1}, minute=0) |
| 153 | c2 = await w.commit("c2", {"pkg/a.py": A_V1, "pkg/b.py": B_SRC}, ["c1"], minute=10) |
| 154 | await pool.commit() |
| 155 | newer_writing = asyncio.Event() |
| 156 | real_compute = P.LanguagesProvider.compute |
| 157 | |
| 158 | async def compute(self, session, repo_id, ref, payload): |
| 159 | out = await real_compute(self, session, repo_id, ref, payload) |
| 160 | if ref == c2: |
| 161 | newer_writing.set() |
| 162 | await asyncio.sleep(1.5) # the older job starts now, writes, and blocks on this transaction's row locks |
| 163 | return out |
| 164 | monkeypatch.setattr(P.LanguagesProvider, "compute", compute) |
| 165 | |
| 166 | async def older() -> tuple[str, dict]: |
| 167 | await newer_writing.wait() |
| 168 | return await worker._run_intel_job("intel.code.languages", repo.repo_id, {"head": c1, "branch": "main"}) |
| 169 | |
| 170 | older_out, newer_out = await asyncio.gather( |
| 171 | older(), worker._run_intel_job("intel.code.languages", repo.repo_id, {"head": c2, "branch": "main"})) |
| 172 | assert newer_out[0] == "done" |
| 173 | assert older_out[0] == "skipped" and "superseded" in str(older_out[1]["reason"]), older_out |
| 174 | async with database.AsyncSessionLocal() as s: |
| 175 | results = (await s.execute(select(MusehubIntelResult.ref).where(MusehubIntelResult.repo_id == repo.repo_id, |
| 176 | MusehubIntelResult.intel_type == "intel.code.languages"))).scalars().all() |
| 177 | refs = (await s.execute(select(MusehubIntelLanguages.ref).where(MusehubIntelLanguages.repo_id == repo.repo_id))).scalars().all() |
| 178 | assert results == [c2] and refs and set(refs) == {c2} # result row AND provider table agree on the newer revision |
| 179 | |
| 180 | |
| 181 | @pytest.mark.asyncio |
| 182 | async def test_k6_failure_and_retry_across_two_workers_is_bounded_and_never_concurrent(pool) -> None: |
| 183 | repo = await create_repo(pool, owner="gabriel", slug="k6") |
| 184 | await pool.commit() |
| 185 | job_id = await put_job(repo.repo_id, "gc") |
| 186 | for expected in range(1, J._MAX_ATTEMPTS + 1): |
| 187 | first = await try_claim() |
| 188 | assert first is not None and first.job_id == job_id and first.attempt == expected |
| 189 | assert await try_claim() is None # the second worker cannot take a running job |
| 190 | async with database.AsyncSessionLocal() as s: |
| 191 | assert await J.fail_job(s, job_id, f"boom {expected}", attempt=expected) is True |
| 192 | await s.commit() |
| 193 | assert await try_claim() is None # exhausted: terminal, not retried a fourth time |
| 194 | async with database.AsyncSessionLocal() as s: |
| 195 | row = (await s.execute(select(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == job_id))).scalar_one() |
| 196 | assert (row.status, row.attempt) == ("failed", J._MAX_ATTEMPTS) |
| 197 | assert await current_attempt(pool, job_id) == J._MAX_ATTEMPTS |
File History
1 commit
sha256:8ab5dc9648872e524e32a32f68a3cada8b3e45c3d16931aa57e9a8a8178dc022
test+fix(intel): real two-session concurrency tests; serial…
Sonnet 5.5
4 days ago