gabriel / musehub public
test_intel_job_state_machine.py python
341 lines 19.2 KB
Raw
sha256:cb660189f57c2e986c5f5cf076547a8e496b63e420c8fcce5e3beef78822783b fix(intel): correct stale production-gate statements; cover… Sonnet 5.5 3 days ago
1 """Truthful job states, dependency barriers, coalescing and stale-write protection for the intel jobs — musehub#8 campaign, Phase B.
2
3 Run ONLY this file, against a private Postgres (never the live hub database)::
4
5 TEST_DATABASE_URL=postgresql+asyncpg://musehub:musehub@<private-host>:<port>/musehub_test \
6 python -P -m pytest tests/test_intel_job_state_machine.py
7
8 Rules under test (finding numbers are the campaign's cross-cutting findings):
9
10 C1 A job is ``done`` only if its work ran. A job type the worker does not execute ends ``skipped`` with a reason; a computed,
11 empty or already-current run says so in ``result``; a missing prerequisite ends ``blocked``; an exception retries and then ``failed``.
12 C2 Index-dependent providers run only after ``intel.code`` finished *for the same revision*; a failed index blocks them.
13 C3 No two jobs of one (repository, intel type) run at once; a push that arrives while one runs is queued, not dropped; a pending job
14 is refreshed to the newest push instead of keeping a stale head.
15 C4 A completion for an older revision can never overwrite a newer result, whatever order jobs finish in.
16
17 Intel execution is OFF unless the repository is listed in ``MUSEHUB_INTEL_WORKER_REPOS`` *and* the deployment identity is known (a local database is required on ``local`` only).
18 """
19 from __future__ import annotations
20
21 from collections.abc import AsyncGenerator
22 from datetime import datetime, timedelta, timezone
23
24 import pytest
25 import pytest_asyncio
26 from sqlalchemy import select
27 from sqlalchemy.ext.asyncio import AsyncSession
28
29 from musehub.db import database
30 from musehub.db.musehub_intel_models import MusehubIntelResult, MusehubSymbolIntel
31 from musehub.db.musehub_jobs_models import MusehubBackgroundJob
32 from musehub.services.musehub_jobs import claim_next_job, enqueue_push_intel
33 from tests.factories import create_repo
34 from tests.test_intel_code_correctness import Dag, cid, ins, standard_dag
35
36 ALLOW_ENV = "MUSEHUB_INTEL_WORKER_REPOS"
37
38
39 @pytest_asyncio.fixture
40 async def worker_db(db_session: AsyncSession, monkeypatch: pytest.MonkeyPatch) -> AsyncGenerator[AsyncSession, None]:
41 """The production engine pointed at the test database, for ``worker._process_one`` (same recipe as the prebuild tests)."""
42 from musehub.config import settings
43 from tests.conftest import _TEST_DATABASE_URL
44
45 monkeypatch.delenv(ALLOW_ENV, raising=False)
46 monkeypatch.setattr(settings, "database_url", _TEST_DATABASE_URL)
47 monkeypatch.setenv("DATABASE_URL", _TEST_DATABASE_URL)
48 await database.init_db(pool_size=1, max_overflow=1)
49 try:
50 yield db_session
51 finally:
52 await database.close_db()
53
54
55 _seq = 0
56
57
58 async def add_job(s: AsyncSession, repo_id: str, job_type: str, head: str = "", *, status: str = "pending", attempt: int = 0,
59 age: int = 0) -> str:
60 global _seq
61 _seq += 1
62 job_id = cid(f"job:{_seq}:{job_type}:{repo_id}")
63 s.add(MusehubBackgroundJob(
64 job_id=job_id, repo_id=repo_id, job_type=job_type, payload={"head": head, "branch": "main"}, status=status,
65 created_at=datetime.now(tz=timezone.utc) - timedelta(seconds=age), attempt=attempt,
66 claimed_at=datetime.now(tz=timezone.utc) if status == "running" else None,
67 ))
68 await s.commit()
69 return job_id
70
71
72 async def job(s: AsyncSession, job_id: str) -> MusehubBackgroundJob:
73 await s.rollback()
74 return (await s.execute(select(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == job_id))).scalar_one()
75
76
77 async def repo_with_history(s: AsyncSession, slug: str) -> tuple[str, Dag]:
78 dag = standard_dag()
79 repo = await create_repo(s, owner="gabriel", slug=slug)
80 await dag.install(s, repo.repo_id, ["c0", "c1", "c2", "m", "c3"])
81 await s.commit()
82 return repo.repo_id, dag
83
84
85 # ---------------------------------------------------------------------------
86 # C1 — truthful states
87 # ---------------------------------------------------------------------------
88
89 @pytest.mark.asyncio
90 @pytest.mark.parametrize("job_type", ["intel.code", "intel.code.dead", "intel.structural", "gc", "profile.snapshot", "push.file_last_commits"])
91 async def test_a_job_type_the_worker_does_not_execute_ends_skipped_never_done(worker_db, job_type) -> None:
92 import musehub.worker as worker
93 repo = await create_repo(worker_db, owner="gabriel", slug=f"skip-{job_type.replace('.', '-')}")
94 jid = await add_job(worker_db, repo.repo_id, job_type, cid("h"))
95 assert await worker._process_one() is True
96 row = await job(worker_db, jid)
97 assert row.status == "skipped"
98 assert row.result is not None and row.result["executed"] is False and row.result["reason"]
99 assert row.done_at is not None
100
101
102 @pytest.mark.asyncio
103 async def test_an_allowlisted_intel_code_job_runs_and_records_what_it_computed(worker_db, monkeypatch) -> None:
104 import musehub.worker as worker
105 repo_id, dag = await repo_with_history(worker_db, "run-code")
106 monkeypatch.setenv(ALLOW_ENV, repo_id)
107 jid = await add_job(worker_db, repo_id, "intel.code", dag.commits_id("c3"))
108 assert await worker._process_one() is True
109 row = await job(worker_db, jid)
110 assert row.status == "done"
111 assert row.result["executed"] is True and row.result["outcome"] == "computed"
112 assert sorted(row.result["intel_types"]) == ["code.intel_snapshot", "code.intel_summary"]
113 n = (await worker_db.execute(select(MusehubSymbolIntel).where(MusehubSymbolIntel.repo_id == repo_id))).scalars().all()
114 assert len(n) == 4
115
116
117 @pytest.mark.asyncio
118 async def test_production_runs_only_the_listed_repository_and_skips_every_other(worker_db, monkeypatch) -> None:
119 """Production authorization boundary through the real job path: the listed repository computes; an unlisted one, with identical history, is skipped."""
120 import musehub.worker as worker
121 listed, dag = await repo_with_history(worker_db, "prod-listed")
122 other, other_dag = await repo_with_history(worker_db, "prod-other")
123 monkeypatch.setenv("MUSEHUB_DEPLOYMENT", "production")
124 monkeypatch.setenv("MUSE_ENV", "production")
125 monkeypatch.setenv(ALLOW_ENV, listed)
126 jid_other = await add_job(worker_db, other, "intel.code", other_dag.commits_id("c3"))
127 jid_listed = await add_job(worker_db, listed, "intel.code", dag.commits_id("c3"))
128 for _ in range(2):
129 assert await worker._process_one() is True
130 row = await job(worker_db, jid_listed)
131 assert row.status == "done" and row.result["executed"] is True and row.result["outcome"] == "computed"
132 skipped = await job(worker_db, jid_other)
133 assert skipped.status == "skipped" and skipped.result["executed"] is False
134 assert (await worker_db.execute(select(MusehubSymbolIntel).where(MusehubSymbolIntel.repo_id == other))).first() is None
135
136
137 @pytest.mark.asyncio
138 @pytest.mark.parametrize("deployment,muse_env,allow", [("production", "production", ""), ("production", "staging", "LISTED"), ("production", "test", "LISTED"), ("staging", "production", "LISTED")])
139 async def test_production_without_an_exact_listing_or_matching_identity_skips(worker_db, monkeypatch, deployment, muse_env, allow) -> None:
140 import musehub.worker as worker
141 repo_id, dag = await repo_with_history(worker_db, f"prod-refuse-{deployment}-{muse_env}-{bool(allow)}")
142 monkeypatch.setenv("MUSEHUB_DEPLOYMENT", deployment)
143 monkeypatch.setenv("MUSE_ENV", muse_env)
144 monkeypatch.setenv(ALLOW_ENV, repo_id if allow == "LISTED" else allow)
145 jid = await add_job(worker_db, repo_id, "intel.code", dag.commits_id("c3"))
146 assert await worker._process_one() is True
147 row = await job(worker_db, jid)
148 assert row.status == "skipped" and row.result["executed"] is False
149
150
151 @pytest.mark.asyncio
152 async def test_empty_input_is_reported_as_empty_not_as_a_success_with_nothing(worker_db, monkeypatch) -> None:
153 import musehub.worker as worker
154 dag = Dag()
155 dag.add("d0", [], None, 0)
156 repo = await create_repo(worker_db, owner="gabriel", slug="empty-input")
157 await dag.install(worker_db, repo.repo_id, ["d0"])
158 await worker_db.commit()
159 monkeypatch.setenv(ALLOW_ENV, repo.repo_id)
160 jid = await add_job(worker_db, repo.repo_id, "intel.code", dag.commits_id("d0"))
161 assert await worker._process_one() is True
162 row = await job(worker_db, jid)
163 assert row.status == "done" and row.result == {"executed": True, "outcome": "empty", "intel_types": [], "ref": dag.commits_id("d0")}
164 assert (await worker_db.execute(select(MusehubIntelResult).where(MusehubIntelResult.repo_id == repo.repo_id))).first() is None
165
166
167 @pytest.mark.asyncio
168 async def test_the_allowlist_never_applies_to_a_non_local_database(worker_db, monkeypatch) -> None:
169 from musehub.services.musehub_intel_gate import intel_execution_allowed
170 from musehub.config import settings
171 monkeypatch.setenv(ALLOW_ENV, "sha256:abc")
172 assert intel_execution_allowed("sha256:abc") is True
173 assert intel_execution_allowed("sha256:other") is False
174 monkeypatch.setattr(settings, "database_url", "postgresql+asyncpg://u:[email protected]:5432/musehub")
175 assert intel_execution_allowed("sha256:abc") is False
176
177
178 @pytest.mark.asyncio
179 async def test_a_failing_job_is_retried_a_bounded_number_of_times_then_fails_leaving_nothing(worker_db, monkeypatch) -> None:
180 import musehub.worker as worker
181 import musehub.services.musehub_intel as intel_mod
182 repo_id, dag = await repo_with_history(worker_db, "retry")
183 monkeypatch.setenv(ALLOW_ENV, repo_id)
184
185 def boom(*a, **k):
186 raise RuntimeError("aggregate exploded")
187 monkeypatch.setattr(intel_mod, "compute_intel", boom)
188 jid = await add_job(worker_db, repo_id, "intel.code", dag.commits_id("c3"))
189 for expected in ("pending", "pending", "failed"):
190 assert await worker._process_one() is True
191 assert (await job(worker_db, jid)).status == expected
192 assert await worker._process_one() is False # nothing left to claim: no fourth attempt
193 row = await job(worker_db, jid)
194 assert row.attempt == 3 and "aggregate exploded" in (row.error or "")
195 assert (await worker_db.execute(select(MusehubIntelResult).where(MusehubIntelResult.repo_id == repo_id))).first() is None
196 assert (await worker_db.execute(select(MusehubSymbolIntel).where(MusehubSymbolIntel.repo_id == repo_id))).first() is None
197
198
199 # ---------------------------------------------------------------------------
200 # C2 — dependency barriers
201 # ---------------------------------------------------------------------------
202
203 @pytest.mark.asyncio
204 async def test_a_dependent_provider_without_a_current_index_is_blocked_not_computed(worker_db, monkeypatch) -> None:
205 import musehub.worker as worker
206 repo_id, dag = await repo_with_history(worker_db, "blocked")
207 monkeypatch.setenv(ALLOW_ENV, repo_id)
208 jid = await add_job(worker_db, repo_id, "intel.code.dead", dag.commits_id("c3")) # no intel.code ever ran or is queued
209 assert await worker._process_one() is True
210 row = await job(worker_db, jid)
211 assert row.status == "blocked" and "intel.code" in (row.error or "")
212 assert row.result["executed"] is False
213 assert (await worker_db.execute(select(MusehubIntelResult).where(MusehubIntelResult.repo_id == repo_id))).first() is None
214
215
216 @pytest.mark.asyncio
217 async def test_dependents_wait_for_a_pending_index_even_when_they_are_older(worker_db) -> None:
218 repo = await create_repo(worker_db, owner="gabriel", slug="barrier")
219 await add_job(worker_db, repo.repo_id, "intel.code.dead", "h", age=30) # older than the index
220 await add_job(worker_db, repo.repo_id, "intel.code.gravity", "h", age=20)
221 code = await add_job(worker_db, repo.repo_id, "intel.code", "h", age=10)
222 first = await claim_next_job(worker_db)
223 assert first is not None and first.job_id == code
224 await worker_db.commit()
225 assert await claim_next_job(worker_db) is None # still running: dependents stay blocked
226 await worker_db.rollback()
227
228
229 @pytest.mark.asyncio
230 async def test_the_index_never_starts_while_a_dependent_is_reading_it(worker_db) -> None:
231 repo = await create_repo(worker_db, owner="gabriel", slug="torn-read")
232 await add_job(worker_db, repo.repo_id, "intel.code.dead", "h1", status="running")
233 await add_job(worker_db, repo.repo_id, "intel.code", "h2")
234 assert await claim_next_job(worker_db) is None
235 await worker_db.rollback()
236
237
238 @pytest.mark.asyncio
239 async def test_snapshot_providers_do_not_wait_for_the_index(worker_db) -> None:
240 """Not index-dependent: a pending index job (queued later) does not hold a snapshot provider back. (They never overlap in time:
241 intel jobs share the host-wide memory barrier, musehub#8 R1 — see test_intel_serial_barrier.py.)"""
242 repo = await create_repo(worker_db, owner="gabriel", slug="independent")
243 lang = await add_job(worker_db, repo.repo_id, "intel.code.languages", "h", age=60)
244 await add_job(worker_db, repo.repo_id, "intel.code", "h")
245 claimed = await claim_next_job(worker_db)
246 assert claimed is not None and claimed.job_id == lang
247 await worker_db.rollback()
248
249
250 # ---------------------------------------------------------------------------
251 # C3 — no overlap, nothing lost
252 # ---------------------------------------------------------------------------
253
254 @pytest.mark.asyncio
255 async def test_one_running_intel_job_on_the_host_and_the_queue_resumes_oldest_first(worker_db) -> None:
256 """Never two runs of one analysis for one repository — and, since musehub#8 R1, never two intel jobs at all: another repository waits too."""
257 a = await create_repo(worker_db, owner="gabriel", slug="ra")
258 b = await create_repo(worker_db, owner="gabriel", slug="rb")
259 running = await add_job(worker_db, a.repo_id, "intel.code.languages", "h1", status="running")
260 queued_same = await add_job(worker_db, a.repo_id, "intel.code.languages", "h2", age=30)
261 await add_job(worker_db, b.repo_id, "intel.code.languages", "h3")
262 assert await claim_next_job(worker_db) is None
263 await worker_db.rollback()
264 (await job(worker_db, running)).status = "done"
265 await worker_db.commit()
266 claimed = await claim_next_job(worker_db)
267 assert claimed is not None and claimed.job_id == queued_same
268 await worker_db.rollback()
269
270
271 @pytest.mark.asyncio
272 async def test_a_push_during_a_running_job_is_queued_not_dropped_and_a_pending_job_is_refreshed(worker_db) -> None:
273 repo = await create_repo(worker_db, owner="gabriel", slug="coalesce")
274 await worker_db.commit()
275 running = await add_job(worker_db, repo.repo_id, "intel.code", "h1", status="running")
276
277 await enqueue_push_intel(worker_db, repo.repo_id, "h2", branch="main") # arrives while h1 is running
278 await worker_db.commit()
279 rows = (await worker_db.execute(select(MusehubBackgroundJob).where(
280 MusehubBackgroundJob.repo_id == repo.repo_id, MusehubBackgroundJob.job_type == "intel.code"))).scalars().all()
281 assert sorted((r.status, r.payload["head"]) for r in rows) == [("pending", "h2"), ("running", "h1")]
282
283 await enqueue_push_intel(worker_db, repo.repo_id, "h3", branch="main") # arrives while h2 is still pending
284 await worker_db.commit()
285 rows = (await worker_db.execute(select(MusehubBackgroundJob).where(
286 MusehubBackgroundJob.repo_id == repo.repo_id, MusehubBackgroundJob.job_type.like("intel.%")))).scalars().all()
287 pending = [r for r in rows if r.status == "pending"]
288 assert len(pending) == len({r.job_type for r in pending}), "one pending job per type"
289 assert all(r.payload["head"] == "h3" for r in pending), "every pending intel job now targets the newest push"
290 assert (await job(worker_db, running)).payload["head"] == "h1" # the running job is untouched
291
292
293 # ---------------------------------------------------------------------------
294 # C4 — stale results never overwrite newer ones
295 # ---------------------------------------------------------------------------
296
297 @pytest.mark.asyncio
298 async def test_completions_in_reversed_order_leave_the_newest_revision_published(worker_db) -> None:
299 from musehub.services.musehub_intel_providers import persist_intel_results
300 repo_id, dag = await repo_with_history(worker_db, "reversed")
301 new, old = dag.commits_id("c3"), dag.commits_id("c1")
302 assert await persist_intel_results(worker_db, repo_id, new, [("intel.code.dead", {"v": "new"})]) == ["intel.code.dead"]
303 assert await persist_intel_results(worker_db, repo_id, old, [("intel.code.dead", {"v": "old"})]) == [] # stale: rejected
304 await worker_db.commit()
305 row = (await worker_db.execute(select(MusehubIntelResult).where(MusehubIntelResult.repo_id == repo_id))).scalar_one()
306 assert row.ref == new and '"new"' in row.data_json
307
308
309 @pytest.mark.asyncio
310 async def test_a_newer_revision_replaces_an_older_one_and_a_divergent_one_is_kept_out(worker_db) -> None:
311 """Branch-head policy (musehub#8): between divergent revisions there is no order, so the published result is kept (the default branch's history
312 outranks a revision not on it: see test_intel_branch_policy.py); a descendant of every stored revision still replaces it."""
313 from musehub.services.musehub_intel_providers import persist_intel_results
314 repo_id, dag = await repo_with_history(worker_db, "forward")
315 c1, c2, c3 = dag.commits_id("c1"), dag.commits_id("c2"), dag.commits_id("c3") # c1 and c2 are siblings; c3 descends from both
316 assert await persist_intel_results(worker_db, repo_id, c1, [("intel.code.dead", {"v": 1})]) == ["intel.code.dead"]
317 assert await persist_intel_results(worker_db, repo_id, c2, [("intel.code.dead", {"v": 2})]) == [] # divergent: the published c1 is kept
318 assert await persist_intel_results(worker_db, repo_id, c3, [("intel.code.dead", {"v": 3})]) == ["intel.code.dead"] # a descendant of both: newer
319 assert await persist_intel_results(worker_db, repo_id, c2, [("intel.code.dead", {"v": 4})]) == [] # now an ancestor of the stored ref
320 await worker_db.commit()
321 row = (await worker_db.execute(select(MusehubIntelResult).where(MusehubIntelResult.repo_id == repo_id))).scalar_one()
322 assert row.ref == c3 and '"v": 3' in row.data_json
323
324
325 @pytest.mark.asyncio
326 async def test_a_dependent_job_for_a_superseded_revision_is_skipped_as_stale(worker_db, monkeypatch) -> None:
327 import musehub.worker as worker
328 repo_id, dag = await repo_with_history(worker_db, "superseded")
329 monkeypatch.setenv(ALLOW_ENV, repo_id)
330 code = await add_job(worker_db, repo_id, "intel.code", dag.commits_id("c3"))
331 assert await worker._process_one() is True and (await job(worker_db, code)).status == "done"
332 older = await add_job(worker_db, repo_id, "intel.code.dead", dag.commits_id("c1")) # a retried job for an older push
333 assert await worker._process_one() is True
334 row = await job(worker_db, older)
335 assert row.status == "skipped" and "superseded" in row.result["reason"]
336 # the exact type the dead provider publishes ("intel.code.dead"): the earlier like("code.dead%") matched no real type, so it could
337 # never fail (see tests/test_intel_published_types.py, which proves the difference)
338 assert (await worker_db.execute(select(MusehubIntelResult).where(
339 MusehubIntelResult.repo_id == repo_id, MusehubIntelResult.intel_type == "intel.code.dead"))).first() is None
340 published = set((await worker_db.execute(select(MusehubIntelResult.intel_type).where(MusehubIntelResult.repo_id == repo_id))).scalars())
341 assert published == {"code.intel_summary", "code.intel_snapshot"}, "only the index published; the superseded dependent added nothing"
File History 6 commits
sha256:cb660189f57c2e986c5f5cf076547a8e496b63e420c8fcce5e3beef78822783b fix(intel): correct stale production-gate statements; cover… Sonnet 5.5 3 days ago
sha256:4a18e5cebf34af33a6c511683aec240fe33af4f9254a7e1b8251d0bfb296aca0 feat(intel): allow production intel execution for explicitl… Sonnet 5.5 patch 3 days ago
sha256:07dade914422a4077032f3ead242320995c8fdedd1a3bb7f4af27708754ce7b4 fix(intel): branch-head policy - analyse the revision the p… Sonnet 5.5 minor ⚠ 4 days ago
sha256:29f70c9cc76cb0f77c2573efaf7b8bbc64de5a995d17aab676e774332be8f16b fix(jobs): intel jobs join the host-wide serial memory barr… Sonnet 5.5 patch 4 days ago
sha256:0611c8ad66a0e8f8e0806057e72df8725adadb1682e7b446e1bc346702ddff51 test(intel): exact result types; a wrong provider's publica… Sonnet 5.5 4 days ago
sha256:1d6f4203c8aefabfff4a51b02e93f64e7b44858566bbcbfc6ce9d5c791691174 fix(worker): truthful job states, intel dependency barriers… Sonnet 5.5 minor ⚠ 4 days ago