gabriel / musehub public
test_intel_concurrency.py python
197 lines 10.5 KB
Raw
sha256:8ab5dc9648872e524e32a32f68a3cada8b3e45c3d16931aa57e9a8a8178dc022 test+fix(intel): real two-session concurrency tests; serial… Sonnet 5.5 4 days ago
1 """Real two-session concurrency for the job queue and intel publication — musehub#8 release gate.
2
3 The barrier tests in test_intel_job_state_machine.py call ``claim_next_job`` one session at a time. These tests run two genuinely concurrent database
4 sessions (separate connections, ``asyncio.gather``) so the properties they check hold under real interleaving, not by construction.
5
6 Run ONLY this file, against a private Postgres (never the live hub database)::
7
8 TEST_DATABASE_URL=postgresql+asyncpg://musehub:musehub@<private-host>:<port>/musehub_test python -P -m pytest tests/test_intel_concurrency.py
9
10 K1 Simultaneous claimers: one pending job is claimed by exactly one of them, every round.
11 K2 Two pending jobs, two claimers: never the same row twice; both get claimed.
12 K3 Advisory-lock contention: a claimer that cannot take the claim lock returns at once (it never blocks, never claims) and succeeds after the holder commits.
13 K4 A claim racing the enqueue-time refresh: the newest push is always covered, a running job is never retargeted, no stale pending job survives.
14 K5 Reversed completion order under real concurrency: the older revision finishes last and cannot overwrite the newer one's published result.
15 K6 Failure and retry across two workers: bounded by ``_MAX_ATTEMPTS``, never claimed concurrently, then terminal.
16 """
17 from __future__ import annotations
18
19 import asyncio
20 from collections.abc import AsyncGenerator
21 from datetime import datetime, timezone
22
23 import pytest
24 import pytest_asyncio
25 from sqlalchemy import func, select, text
26 from sqlalchemy.ext.asyncio import AsyncSession
27
28 from musehub import worker
29 from musehub.db import database
30 from musehub.db.musehub_intel_models import MusehubIntelLanguages, MusehubIntelResult
31 from musehub.db.musehub_jobs_models import MusehubBackgroundJob
32 from musehub.services import musehub_intel_providers as P
33 from musehub.services import musehub_jobs as J
34 from tests.factories import create_repo
35 from tests.job_execution import current_attempt
36 from tests.test_intel_snapshot_providers import A_V1, B_SRC, World
37
38 ROUNDS = 25
39
40
41 @pytest_asyncio.fixture
42 async def pool(db_session: AsyncSession, monkeypatch: pytest.MonkeyPatch) -> AsyncGenerator[AsyncSession, None]:
43 """The production engine on the test database with room for several concurrent sessions."""
44 from musehub.config import settings
45 from tests.conftest import _TEST_DATABASE_URL
46
47 monkeypatch.setattr(settings, "database_url", _TEST_DATABASE_URL)
48 monkeypatch.setenv("DATABASE_URL", _TEST_DATABASE_URL)
49 await database.init_db(pool_size=6, max_overflow=2)
50 try:
51 yield db_session
52 finally:
53 await database.close_db()
54
55
56 async def put_job(repo_id: str, job_type: str, head: str = "h1", *, status: str = "pending", age: int = 0) -> str:
57 async with database.AsyncSessionLocal() as s:
58 job_id = J.compute_job_id(repo_id, job_type, f"{head}:{status}:{age}:{datetime.now(tz=timezone.utc).isoformat()}")
59 s.add(MusehubBackgroundJob(job_id=job_id, repo_id=repo_id, job_type=job_type, payload={"head": head, "branch": "main"}, status=status,
60 created_at=datetime.now(tz=timezone.utc), claimed_at=datetime.now(tz=timezone.utc) if status == "running" else None))
61 await s.commit()
62 return job_id
63
64
65 async def try_claim() -> MusehubBackgroundJob | None:
66 async with database.AsyncSessionLocal() as s:
67 claimed = await J.claim_next_job(s)
68 await s.commit()
69 return claimed
70
71
72 async def jobs_of(repo_id: str, prefix: str) -> list[MusehubBackgroundJob]:
73 async with database.AsyncSessionLocal() as s:
74 return list((await s.execute(select(MusehubBackgroundJob).where(MusehubBackgroundJob.repo_id == repo_id,
75 MusehubBackgroundJob.job_type.like(f"{prefix}%")))).scalars())
76
77
78 @pytest.mark.asyncio
79 async def test_k1_simultaneous_claimers_never_double_claim(pool) -> None:
80 repo = await create_repo(pool, owner="gabriel", slug="k1")
81 await pool.commit()
82 for i in range(ROUNDS):
83 job_id = await put_job(repo.repo_id, "gc", f"round{i}")
84 got = [c for c in await asyncio.gather(try_claim(), try_claim()) if c is not None]
85 assert [c.job_id for c in got] == [job_id], f"round {i}: {[c.job_id[:12] for c in got]}"
86 assert got[0].attempt == 1
87
88
89 @pytest.mark.asyncio
90 async def test_k2_two_jobs_two_claimers_get_distinct_rows(pool) -> None:
91 a, b = await create_repo(pool, owner="gabriel", slug="k2a"), await create_repo(pool, owner="gabriel", slug="k2b")
92 await pool.commit()
93 for i in range(ROUNDS // 2):
94 wanted = {await put_job(a.repo_id, "gc", f"a{i}"), await put_job(b.repo_id, "gc", f"b{i}")}
95 got: dict[str, int] = {}
96 for _ in range(10): # the claim lock is try-lock: a loser returns None and polls again, exactly like the worker loop
97 for c in await asyncio.gather(try_claim(), try_claim()):
98 if c is not None:
99 assert c.job_id not in got, "the same row was claimed twice"
100 got[c.job_id] = c.attempt
101 if set(got) >= wanted:
102 break
103 assert set(got) >= wanted and all(att == 1 for jid, att in got.items() if jid in wanted)
104 async with database.AsyncSessionLocal() as s: # finish them so the next round starts clean
105 await s.execute(text("UPDATE musehub_background_jobs SET status='done' WHERE status='running'"))
106 await s.commit()
107
108
109 @pytest.mark.asyncio
110 async def test_k3_a_claimer_that_cannot_take_the_claim_lock_returns_at_once_and_succeeds_later(pool) -> None:
111 repo = await create_repo(pool, owner="gabriel", slug="k3")
112 await pool.commit()
113 job_id = await put_job(repo.repo_id, "gc")
114 async with database.AsyncSessionLocal() as holder:
115 await holder.execute(text(f"SELECT pg_advisory_xact_lock({J._CLAIM_LOCK_KEY})"))
116 blocked = await asyncio.wait_for(try_claim(), timeout=3) # a blocking claimer would hang here until the timeout
117 assert blocked is None
118 await holder.commit()
119 claimed = await try_claim()
120 assert claimed is not None and claimed.job_id == job_id
121
122
123 @pytest.mark.asyncio
124 async def test_k4_a_claim_racing_the_enqueue_refresh_never_loses_the_newest_push(pool) -> None:
125 for i in range(ROUNDS):
126 repo = await create_repo(pool, owner="gabriel", slug=f"k4-{i}")
127 await pool.commit()
128 await put_job(repo.repo_id, "intel.code", "h1")
129
130 async def enqueue() -> None:
131 async with database.AsyncSessionLocal() as s:
132 await J.enqueue_push_intel(s, repo.repo_id, "h2", branch="main")
133 await s.commit()
134
135 await asyncio.gather(try_claim(), enqueue())
136 rows = await jobs_of(repo.repo_id, "intel.code")
137 code = [r for r in rows if r.job_type == "intel.code"]
138 shape = sorted((r.status, r.payload["head"]) for r in code)
139 allowed = [[("pending", "h2"), ("running", "h1")], [("running", "h2")], [("pending", "h2")]]
140 assert shape in allowed, f"round {i}: {shape}" # never [running h1] alone (h2 lost) and never a stale pending h1
141 for r in rows: # a RUNNING job is never retargeted: its head is the one it was claimed with
142 if r.status == "running":
143 assert r.attempt == 1
144
145
146 @pytest.mark.asyncio
147 async def test_k5_reversed_completion_under_real_concurrency_cannot_overwrite_the_newer_result(pool, monkeypatch) -> None:
148 """The newer revision is mid-transaction (its provider rows written, not yet committed) when the OLDER revision's job starts: the older job blocks
149 behind the newer one's row locks, then proceeds after the commit and must be rejected at its stale-write check — and its writes rolled back."""
150 repo = await create_repo(pool, owner="gabriel", slug="k5")
151 w = World(pool, repo.repo_id)
152 c1 = await w.commit("c1", {"pkg/a.py": A_V1}, minute=0)
153 c2 = await w.commit("c2", {"pkg/a.py": A_V1, "pkg/b.py": B_SRC}, ["c1"], minute=10)
154 await pool.commit()
155 newer_writing = asyncio.Event()
156 real_compute = P.LanguagesProvider.compute
157
158 async def compute(self, session, repo_id, ref, payload):
159 out = await real_compute(self, session, repo_id, ref, payload)
160 if ref == c2:
161 newer_writing.set()
162 await asyncio.sleep(1.5) # the older job starts now, writes, and blocks on this transaction's row locks
163 return out
164 monkeypatch.setattr(P.LanguagesProvider, "compute", compute)
165
166 async def older() -> tuple[str, dict]:
167 await newer_writing.wait()
168 return await worker._run_intel_job("intel.code.languages", repo.repo_id, {"head": c1, "branch": "main"})
169
170 older_out, newer_out = await asyncio.gather(
171 older(), worker._run_intel_job("intel.code.languages", repo.repo_id, {"head": c2, "branch": "main"}))
172 assert newer_out[0] == "done"
173 assert older_out[0] == "skipped" and "superseded" in str(older_out[1]["reason"]), older_out
174 async with database.AsyncSessionLocal() as s:
175 results = (await s.execute(select(MusehubIntelResult.ref).where(MusehubIntelResult.repo_id == repo.repo_id,
176 MusehubIntelResult.intel_type == "intel.code.languages"))).scalars().all()
177 refs = (await s.execute(select(MusehubIntelLanguages.ref).where(MusehubIntelLanguages.repo_id == repo.repo_id))).scalars().all()
178 assert results == [c2] and refs and set(refs) == {c2} # result row AND provider table agree on the newer revision
179
180
181 @pytest.mark.asyncio
182 async def test_k6_failure_and_retry_across_two_workers_is_bounded_and_never_concurrent(pool) -> None:
183 repo = await create_repo(pool, owner="gabriel", slug="k6")
184 await pool.commit()
185 job_id = await put_job(repo.repo_id, "gc")
186 for expected in range(1, J._MAX_ATTEMPTS + 1):
187 first = await try_claim()
188 assert first is not None and first.job_id == job_id and first.attempt == expected
189 assert await try_claim() is None # the second worker cannot take a running job
190 async with database.AsyncSessionLocal() as s:
191 assert await J.fail_job(s, job_id, f"boom {expected}", attempt=expected) is True
192 await s.commit()
193 assert await try_claim() is None # exhausted: terminal, not retried a fourth time
194 async with database.AsyncSessionLocal() as s:
195 row = (await s.execute(select(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == job_id))).scalar_one()
196 assert (row.status, row.attempt) == ("failed", J._MAX_ATTEMPTS)
197 assert await current_attempt(pool, job_id) == J._MAX_ATTEMPTS
File History 1 commit
sha256:8ab5dc9648872e524e32a32f68a3cada8b3e45c3d16931aa57e9a8a8178dc022 test+fix(intel): real two-session concurrency tests; serial… Sonnet 5.5 4 days ago