gabriel / musehub public
test_mwp4_prebuild_ordering.py python
1,007 lines 39.5 KB Cold
Raw
sha256:cb4b05e9d6caea5f351e3cb452ee6a1db8856169328ad96bfba9ee8e04b8afa6 fix(jobs): fence every execution of a job; an expired or su… Sonnet 5.5 minor ⚠ breaking 4 days ago
1 """MWP-4 — Gate fetch.mpack.prebuild behind mpack.index (fixes RC-4).
2
3 Sub-ticket: musehub#109 — https://staging.musehub.ai/gabriel/musehub/issues/109
4 Master: muse#58 — https://staging.musehub.ai/gabriel/muse/issues/58
5 Predecessors:
6 musehub#106 (MWP-1, generation authority) — closed.
7 musehub#107 (MWP-2, walk fallback) — closed.
8 musehub#108 (MWP-3, job-enqueue idempotency) — closed.
9
10 Root cause (RC-4): ``claim_next_job`` orders only by ``created_at`` with no
11 tiebreaker. When ``mpack.index`` and ``fetch.mpack.prebuild`` share the same
12 ``created_at`` (the normal case from ``enqueue_push_intel``), or the prebuild
13 lands with an earlier timestamp due to cross-push interleaving, PostgreSQL may
14 return the prebuild first. The prebuild then calls ``wire_fetch_mpack`` with
15 ``force_build=True``, hits the index-coverage check at
16 ``musehub_wire_fetch.py:681–694``, and raises ``FetchNotIndexedError`` because
17 ``MusehubMPackIndex`` is still empty. The broad ``except Exception`` at
18 ``musehub_wire_fetch.py:1164–1167`` swallows the error silently: ``tips_built``
19 stays ``0``, no ``MusehubFetchMPackCache`` row is written, and the next clone
20 returns 503.
21
22 Phase 0 — RED reproduction (no production code changes):
23 MWP4_01 RED — ``claim_next_job`` returns the prebuild before the index
24 when the prebuild has an earlier ``created_at``. Asserts
25 the desired correct behavior → FAILS before Phase 2 fix.
26 MWP4_02 — handler-level proof: prebuild run before index swallows
27 FetchNotIndexedError and writes no cache row. Asserts the
28 broken outcome → PASSES before fix (proves the bug).
29 Must be updated in Phase 3 when the swallow is narrowed.
30 """
31
32 from __future__ import annotations
33
34 import hashlib
35 from datetime import datetime, timedelta, timezone
36
37 import pytest
38 from sqlalchemy import delete, func, select
39 from sqlalchemy.ext.asyncio import AsyncSession
40
41 from muse.core.ids import hash_commit, hash_snapshot
42 from muse.core.mpack import build_wire_mpack
43 from muse.core.types import blob_id
44 from musehub.core.genesis import compute_job_id
45 from musehub.db.musehub_jobs_models import MusehubBackgroundJob
46 from musehub.db.musehub_repo_models import MusehubFetchMPackCache, MusehubMPackIndex
47 from musehub.services.musehub_jobs import claim_next_job, complete_job, enqueue_push_intel
48 from musehub.services.musehub_wire_fetch import process_fetch_mpack_prebuild_job
49 from musehub.services.musehub_wire_push import process_mpack_index_job, wire_push_unpack_mpack
50 from musehub.services.musehub_wire_shared import FetchNotIndexedError
51 from tests.factories import create_repo
52
53
54 # ---------------------------------------------------------------------------
55 # Helpers
56 # ---------------------------------------------------------------------------
57
58
59 def _fake_commit_id(seed: str) -> str:
60 return "sha256:" + hashlib.sha256(seed.encode()).hexdigest()
61
62
63 def _now() -> datetime:
64 return datetime.now(tz=timezone.utc)
65
66
67 def _job_row(
68 repo_id: str,
69 job_type: str,
70 payload: dict,
71 created_at: datetime,
72 status: str = "pending",
73 ) -> MusehubBackgroundJob:
74 """Build a MusehubBackgroundJob row ready to be added to a session."""
75 job_id = compute_job_id(repo_id, job_type, created_at.isoformat())
76 return MusehubBackgroundJob(
77 job_id=job_id,
78 repo_id=repo_id,
79 job_type=job_type,
80 payload=payload,
81 status=status,
82 created_at=created_at,
83 attempt=0,
84 )
85
86
87 _E2E_OWNER = "gabriel"
88
89
90 # id -> (seed, committed_at) for commits built by _e2e_raw_commit and pushed;
91 # push ingest verifies that each pushed id reproduces from its fields.
92 _PUSHED: dict[str, tuple[str, str]] = {}
93
94
95 def _pushed_cid(seed: str, parent: str | None, snap_id: str) -> str:
96 """The id of the commit _e2e_raw_commit builds for *seed*."""
97 at = _now().isoformat()
98 cid = hash_commit(parent_ids=[parent] if parent else [], snapshot_id=snap_id, message=f"commit {seed}",
99 committed_at_iso=at, author=_E2E_OWNER, signer_public_key="")
100 _PUSHED[cid] = (seed, at)
101 return cid
102
103
104 def _e2e_raw_commit(
105 cid: str,
106 parent: str | None,
107 snap_id: str,
108 branch: str = "main",
109 ) -> dict:
110 seed, at = _PUSHED.get(cid, (cid[:12], _now().isoformat()))
111 return {
112 "commit_id": cid,
113 "branch": branch,
114 "message": f"commit {seed}",
115 "author": _E2E_OWNER,
116 "committed_at": at,
117 "parent_commit_id": parent,
118 "parent2_commit_id": None,
119 "snapshot_id": snap_id,
120 "agent_id": "",
121 "model_id": "",
122 "toolchain_id": "",
123 "sem_ver_bump": "none",
124 "breaking_changes": [],
125 "signature": "",
126 "signer_key_id": "",
127 "signer_public_key": "",
128 "prompt_hash": "",
129 }
130
131
132 async def _e2e_push(
133 session: AsyncSession,
134 repo_id: str,
135 cid: str,
136 parent: str | None,
137 snap_id: str,
138 fpath: str,
139 oid: str,
140 data: bytes,
141 branch: str = "main",
142 ) -> str:
143 """Build a single-commit mpack, store it in the conftest MemoryBackend, push it.
144
145 The conftest autouse fixture patches ``musehub.storage.backends.get_backend``
146 to a ``MemoryBackend`` instance shared across all modules. Pre-seeding
147 ``backend._mpacks[key]`` is equivalent to a MinIO PUT — both
148 ``wire_push_unpack_mpack`` and ``process_mpack_index_job`` call
149 ``get_backend()`` and see the same bytes.
150
151 Returns the ``mpack_key`` so callers can locate the enqueued index job.
152 """
153 import musehub.storage.backends as _b_mod
154 backend = _b_mod.get_backend()
155
156 mpack_bytes = build_wire_mpack({
157 "commits": [_e2e_raw_commit(cid, parent, snap_id, branch=branch)],
158 "snapshots": [{
159 "snapshot_id": snap_id,
160 "parent_snapshot_id": None,
161 "delta_upsert": {fpath: oid},
162 "delta_remove": [],
163 "directories": [],
164 }],
165 "blobs": [{"object_id": oid, "content": data}],
166 "tags": [],
167 })
168 mpack_key = blob_id(mpack_bytes)
169 backend._mpacks[mpack_key] = mpack_bytes
170
171 await wire_push_unpack_mpack(
172 session,
173 repo_id,
174 mpack_key,
175 pusher_id=_E2E_OWNER,
176 branch=branch,
177 head_commit_id=cid,
178 commits_count=1,
179 blobs_count=1,
180 force=True,
181 )
182 await session.commit()
183 return mpack_key
184
185
186 # ---------------------------------------------------------------------------
187 # MWP4_01 — claim_next_job returns prebuild before a pending index (RED)
188 # ---------------------------------------------------------------------------
189
190
191 @pytest.mark.asyncio
192 async def test_mwp4_01_claim_returns_prebuild_before_index(
193 db_session: AsyncSession,
194 ) -> None:
195 """MWP4_01 RED: claim_next_job must NOT return fetch.mpack.prebuild while
196 an mpack.index for the same repo is pending.
197
198 Setup: insert a fetch.mpack.prebuild with created_at = now-5s (earlier
199 than the mpack.index at now). This simulates the cross-push race where
200 push A's prebuild is enqueued before push B's index job arrives.
201
202 Pre-fix (expected to FAIL):
203 claim_next_job orders solely by created_at; the prebuild (earlier
204 timestamp) is returned ahead of the index. The assertion
205 ``claimed.job_type == "mpack.index"`` therefore fails.
206
207 Post-fix (Phase 2 — correlated NOT EXISTS barrier):
208 The barrier prevents fetch.mpack.prebuild from being claimed while any
209 mpack.index for the same repo_id is pending or running. The index is
210 returned instead, and the prebuild waits until the index drains.
211 """
212 repo = await create_repo(db_session, owner="gabriel")
213 repo_id = str(repo.repo_id)
214
215 now = _now()
216 earlier = now - timedelta(seconds=5)
217
218 # Insert prebuild FIRST with an earlier timestamp to force it ahead in the
219 # ORDER BY created_at ordering — this is the race we're fixing.
220 prebuild = _job_row(
221 repo_id,
222 "fetch.mpack.prebuild",
223 {"tip_commit_ids": [_fake_commit_id("mwp4-01-tip")]},
224 created_at=earlier,
225 )
226 index = _job_row(
227 repo_id,
228 "mpack.index",
229 {
230 "mpack_key": "sha256:" + hashlib.sha256(b"mwp4-01-mpack").hexdigest(),
231 "head": _fake_commit_id("mwp4-01-head"),
232 "branch": "main",
233 },
234 created_at=now,
235 )
236 db_session.add(prebuild)
237 db_session.add(index)
238 await db_session.commit()
239
240 # Claim the next job.
241 # Pre-fix: returns the prebuild (earliest created_at wins) → assertion fails.
242 # Post-fix: barrier skips the prebuild; returns the index instead.
243 claimed = await claim_next_job(db_session)
244 await db_session.commit()
245
246 assert claimed is not None, "Expected a job to be claimed — queue has two pending rows."
247 assert claimed.job_type == "mpack.index", (
248 f"claim_next_job must skip fetch.mpack.prebuild while mpack.index is "
249 f"pending for the same repo_id, but returned '{claimed.job_type}'. "
250 f"The prebuild must be gated behind all pending index jobs (RC-4 barrier)."
251 )
252
253
254 # ---------------------------------------------------------------------------
255 # MWP4_02 — prebuild run before index swallows FetchNotIndexedError (bug proof)
256 # ---------------------------------------------------------------------------
257
258
259 @pytest.mark.asyncio
260 async def test_mwp4_02_prebuild_swallows_fetch_not_indexed_error(
261 db_session: AsyncSession,
262 ) -> None:
263 """MWP4_02 (updated for Phase 3): process_fetch_mpack_prebuild_job raises
264 FetchNotIndexedError when MusehubMPackIndex is empty — no longer swallowed.
265
266 Setup: push one commit so MusehubObject, MusehubCommit, and MusehubBranch
267 rows exist. wire_push_unpack_mpack writes MusehubMPackIndex rows at step 7d
268 synchronously, so we explicitly delete them after the push to simulate the
269 race state: DB has commits/snapshots but MusehubMPackIndex is empty (as if
270 the mpack.index job hasn't run yet and has also terminally failed, bypassing
271 the Phase 2 barrier).
272
273 Phase 0 behavior (pre-Phase 3): FetchNotIndexedError was silently swallowed
274 by the broad ``except Exception``; tips_built==0, no cache row.
275
276 Phase 3 behavior: FetchNotIndexedError is caught specifically and re-raised
277 so the worker's fail_job records it as a failed job (retryable). The broad
278 ``except Exception`` still handles genuinely unexpected errors.
279
280 With the Phase 2 barrier in place this path is only reachable when an
281 mpack.index has terminally failed — exactly the case that should surface as
282 a failed job, not a silent empty cache.
283 """
284 repo = await create_repo(db_session, owner="gabriel")
285 repo_id = str(repo.repo_id)
286
287 data = b"mwp4-02-blob-content"
288 oid = blob_id(data)
289 snap_id = hash_snapshot({"file.txt": oid})
290 cid = _pushed_cid("mwp4-02-c1", None, snap_id)
291
292 # Push one commit — creates MusehubObject, MusehubCommit, MusehubBranch,
293 # enqueues mpack.index + fetch.mpack.prebuild, AND writes MusehubMPackIndex
294 # rows at step 7d of wire_push_unpack_mpack (synchronous inline indexing).
295 mpack_key = await _e2e_push(db_session, repo_id, cid, None, snap_id, "file.txt", oid, data)
296
297 # Simulate the RC-4 race: delete the inline MusehubMPackIndex rows that the
298 # push wrote. This replicates the state where the prebuild job is claimed
299 # before the mpack.index job has had a chance to run — MusehubCommit,
300 # MusehubSnapshot, and MusehubBranch are populated, but MusehubMPackIndex
301 # is empty for this mpack_key.
302 await db_session.execute(
303 delete(MusehubMPackIndex).where(MusehubMPackIndex.mpack_id == mpack_key)
304 )
305 await db_session.commit()
306
307 # Locate the prebuild job enqueued by wire_push_unpack_mpack.
308 prebuild_job = (await db_session.execute(
309 select(MusehubBackgroundJob).where(
310 MusehubBackgroundJob.repo_id == repo_id,
311 MusehubBackgroundJob.job_type == "fetch.mpack.prebuild",
312 MusehubBackgroundJob.status == "pending",
313 ).limit(1)
314 )).scalar_one_or_none()
315 assert prebuild_job is not None, (
316 "wire_push_unpack_mpack must enqueue a fetch.mpack.prebuild job after a push."
317 )
318
319 # Phase 3: FetchNotIndexedError must propagate (no longer swallowed).
320 # The worker's fail_job path will record this as a retryable failure.
321 with pytest.raises(FetchNotIndexedError):
322 await process_fetch_mpack_prebuild_job(db_session, prebuild_job.job_id)
323 await db_session.rollback()
324
325
326 # ---------------------------------------------------------------------------
327 # MWP4_03 — fetch.mpack.prebuild created_at is strictly after mpack.index
328 # ---------------------------------------------------------------------------
329
330
331 @pytest.mark.asyncio
332 async def test_mwp4_03_prebuild_created_at_after_index(
333 db_session: AsyncSession,
334 ) -> None:
335 """MWP4_03: enqueue_push_intel must assign fetch.mpack.prebuild a created_at
336 strictly greater than mpack.index and strictly less than intel.code.* subtypes.
337
338 This is the Phase 1 defense-in-depth stagger:
339
340 t+0s mpack.index, intel.code, intel.structural, gc, ... (foundation)
341 t+1s fetch.mpack.prebuild (NEW middle tier)
342 t+2s intel.code.* subtypes (subtype tier)
343
344 Pre-fix: FAILS — fetch.mpack.prebuild and mpack.index share the same
345 created_at (both are in _FOUNDATION_TYPES at t+0s).
346
347 Post-fix (Phase 1): PASSES — fetch.mpack.prebuild gets created_at_prebuild
348 (t+1s) while mpack.index stays at the foundation tier (t+0s).
349 """
350 repo = await create_repo(db_session, owner="gabriel")
351 repo_id = str(repo.repo_id)
352 fake_head = _fake_commit_id("mwp4-03-head")
353 fake_mpack_key = "sha256:" + hashlib.sha256(b"mwp4-03-mpack").hexdigest()
354
355 await enqueue_push_intel(
356 db_session,
357 repo_id=repo_id,
358 head=fake_head,
359 domain_id=None, # triggers code-domain subtypes (intel.code.*)
360 branch="main",
361 owner=None,
362 mpack_key=fake_mpack_key,
363 )
364 await db_session.commit()
365
366 # Fetch created_at for mpack.index
367 index_row = (await db_session.execute(
368 select(MusehubBackgroundJob.created_at).where(
369 MusehubBackgroundJob.repo_id == repo_id,
370 MusehubBackgroundJob.job_type == "mpack.index",
371 ).limit(1)
372 )).scalar_one_or_none()
373 assert index_row is not None, "mpack.index job must be enqueued by enqueue_push_intel"
374
375 # Fetch created_at for fetch.mpack.prebuild
376 prebuild_row = (await db_session.execute(
377 select(MusehubBackgroundJob.created_at).where(
378 MusehubBackgroundJob.repo_id == repo_id,
379 MusehubBackgroundJob.job_type == "fetch.mpack.prebuild",
380 ).limit(1)
381 )).scalar_one_or_none()
382 assert prebuild_row is not None, "fetch.mpack.prebuild job must be enqueued by enqueue_push_intel"
383
384 # Fetch created_at for any intel.code.* subtype (the t+2s tier)
385 subtype_row = (await db_session.execute(
386 select(MusehubBackgroundJob.created_at).where(
387 MusehubBackgroundJob.repo_id == repo_id,
388 MusehubBackgroundJob.job_type == "intel.code.coupling",
389 ).limit(1)
390 )).scalar_one_or_none()
391 assert subtype_row is not None, (
392 "intel.code.coupling job must be enqueued when domain_id=None — "
393 "needed to verify the three-tier ordering"
394 )
395
396 index_at = index_row
397 prebuild_at = prebuild_row
398 subtype_at = subtype_row
399
400 # Tier 1 → Tier 2: prebuild strictly after index
401 assert prebuild_at > index_at, (
402 f"fetch.mpack.prebuild created_at ({prebuild_at.isoformat()}) must be "
403 f"strictly greater than mpack.index created_at ({index_at.isoformat()}). "
404 f"Pre-fix: they share the same timestamp (both in _FOUNDATION_TYPES at t+0s). "
405 f"Phase 1 fix: assign fetch.mpack.prebuild created_at_prebuild = t+1s."
406 )
407
408 # Tier 2 → Tier 3: prebuild strictly before intel.code.* subtypes
409 assert prebuild_at < subtype_at, (
410 f"fetch.mpack.prebuild created_at ({prebuild_at.isoformat()}) must be "
411 f"strictly less than intel.code.* subtype created_at ({subtype_at.isoformat()}). "
412 f"The three-tier ordering must be: foundation < prebuild < subtypes."
413 )
414
415
416 # ---------------------------------------------------------------------------
417 # Phase 2 — claim-layer dependency barrier (MWP4_04 – MWP4_09)
418 # ---------------------------------------------------------------------------
419
420
421 @pytest.mark.asyncio
422 async def test_mwp4_04_prebuild_not_claimed_while_index_pending(
423 db_session: AsyncSession,
424 ) -> None:
425 """MWP4_04: claim_next_job must NOT return fetch.mpack.prebuild while any
426 mpack.index for the same repo is pending — even when the prebuild has an
427 earlier created_at (the exact cross-push race from MWP4_01).
428
429 Pre-fix: FAILS — claim returns the prebuild (earlier timestamp wins).
430 Post-fix (Phase 2 barrier): PASSES — the correlated NOT EXISTS filter
431 blocks the prebuild; claim returns the mpack.index instead.
432 """
433 repo = await create_repo(db_session, owner="gabriel")
434 repo_id = str(repo.repo_id)
435
436 now = _now()
437 earlier = now - timedelta(seconds=5)
438
439 prebuild = _job_row(
440 repo_id, "fetch.mpack.prebuild",
441 {"tip_commit_ids": [_fake_commit_id("mwp4-04-tip")]},
442 created_at=earlier,
443 )
444 index = _job_row(
445 repo_id, "mpack.index",
446 {"mpack_key": "sha256:" + hashlib.sha256(b"mwp4-04").hexdigest(), "head": _fake_commit_id("h"), "branch": "main"},
447 created_at=now,
448 )
449 db_session.add(prebuild)
450 db_session.add(index)
451 await db_session.commit()
452
453 claimed = await claim_next_job(db_session)
454 await db_session.commit()
455
456 assert claimed is not None, "A claimable job must exist (mpack.index is pending)"
457 assert claimed.job_type == "mpack.index", (
458 f"claim_next_job must skip fetch.mpack.prebuild while mpack.index is "
459 f"pending for the same repo — the Phase 2 barrier. Got: {claimed.job_type}"
460 )
461
462
463 @pytest.mark.asyncio
464 async def test_mwp4_05_prebuild_claimable_after_index_done(
465 db_session: AsyncSession,
466 ) -> None:
467 """MWP4_05: once mpack.index is in terminal state 'done', the prebuild
468 must be claimable on the next claim_next_job call.
469
470 Pre-fix: PASSES (no barrier means prebuild is always claimable by timestamp).
471 Post-fix: must still PASS — 'done' is not a blocking state.
472 """
473 repo = await create_repo(db_session, owner="gabriel")
474 repo_id = str(repo.repo_id)
475
476 now = _now()
477 index_done = _job_row(
478 repo_id, "mpack.index",
479 {"mpack_key": "sha256:" + hashlib.sha256(b"mwp4-05").hexdigest(), "head": _fake_commit_id("h"), "branch": "main"},
480 created_at=now - timedelta(seconds=10),
481 status="done",
482 )
483 prebuild = _job_row(
484 repo_id, "fetch.mpack.prebuild",
485 {"tip_commit_ids": [_fake_commit_id("mwp4-05-tip")]},
486 created_at=now,
487 )
488 db_session.add(index_done)
489 db_session.add(prebuild)
490 await db_session.commit()
491
492 claimed = await claim_next_job(db_session)
493 await db_session.commit()
494
495 assert claimed is not None, "fetch.mpack.prebuild must be claimable when mpack.index is done"
496 assert claimed.job_type == "fetch.mpack.prebuild", (
497 f"mpack.index in 'done' state must not block the prebuild. Got: {claimed.job_type}"
498 )
499
500
501 @pytest.mark.asyncio
502 async def test_mwp4_06_prebuild_blocked_while_index_running(
503 db_session: AsyncSession,
504 ) -> None:
505 """MWP4_06: an mpack.index in 'running' state (already claimed by another
506 worker) must also block the prebuild — the barrier covers both pending and
507 running.
508
509 Setup: mpack.index is running (claimed), fetch.mpack.prebuild is pending.
510 Expected: claim_next_job returns None (the only pending job is blocked).
511
512 Pre-fix: FAILS — prebuild is returned (no barrier).
513 Post-fix: PASSES — running mpack.index blocks the prebuild.
514 """
515 repo = await create_repo(db_session, owner="gabriel")
516 repo_id = str(repo.repo_id)
517
518 now = _now()
519 index_running = _job_row(
520 repo_id, "mpack.index",
521 {"mpack_key": "sha256:" + hashlib.sha256(b"mwp4-06").hexdigest(), "head": _fake_commit_id("h"), "branch": "main"},
522 created_at=now - timedelta(seconds=10),
523 status="running",
524 )
525 prebuild = _job_row(
526 repo_id, "fetch.mpack.prebuild",
527 {"tip_commit_ids": [_fake_commit_id("mwp4-06-tip")]},
528 created_at=now,
529 )
530 db_session.add(index_running)
531 db_session.add(prebuild)
532 await db_session.commit()
533
534 claimed = await claim_next_job(db_session)
535 await db_session.commit()
536
537 assert claimed is None, (
538 f"claim_next_job must return None when the only pending job is a "
539 f"fetch.mpack.prebuild blocked by a running mpack.index. Got: "
540 f"{claimed.job_type if claimed else None}"
541 )
542
543
544 @pytest.mark.asyncio
545 async def test_mwp4_07_terminal_index_does_not_block_prebuild(
546 db_session: AsyncSession,
547 ) -> None:
548 """MWP4_07: mpack.index rows in terminal states (failed, quarantined) must
549 NOT block the prebuild. A permanently-failed index should surface via
550 fail_job, not silently hold up the prebuild forever.
551
552 Pre-fix: PASSES (no barrier means prebuild is always claimable).
553 Post-fix: must still PASS — only pending/running are blocking states.
554 """
555 repo = await create_repo(db_session, owner="gabriel")
556 repo_id = str(repo.repo_id)
557
558 now = _now()
559 for i, status in enumerate(("failed", "quarantined")):
560 index_terminal = _job_row(
561 repo_id, "mpack.index",
562 {"mpack_key": "sha256:" + hashlib.sha256(f"mwp4-07-{status}".encode()).hexdigest(),
563 "head": _fake_commit_id(f"h-{status}"), "branch": "main"},
564 # Distinct created_at per row to avoid PK collision in compute_job_id
565 created_at=now - timedelta(seconds=10 + i),
566 status=status,
567 )
568 db_session.add(index_terminal)
569
570 prebuild = _job_row(
571 repo_id, "fetch.mpack.prebuild",
572 {"tip_commit_ids": [_fake_commit_id("mwp4-07-tip")]},
573 created_at=now,
574 )
575 db_session.add(prebuild)
576 await db_session.commit()
577
578 claimed = await claim_next_job(db_session)
579 await db_session.commit()
580
581 assert claimed is not None, (
582 "fetch.mpack.prebuild must be claimable when all mpack.index rows are terminal"
583 )
584 assert claimed.job_type == "fetch.mpack.prebuild", (
585 f"Terminal mpack.index states (failed/quarantined) must not gate the "
586 f"prebuild. Got: {claimed.job_type}"
587 )
588
589
590 @pytest.mark.asyncio
591 async def test_mwp4_08_barrier_is_per_repo(
592 db_session: AsyncSession,
593 ) -> None:
594 """MWP4_08: the barrier is scoped per repo_id. A pending mpack.index in
595 repo X must not block a fetch.mpack.prebuild in repo Y.
596
597 Pre-fix: PASSES (no barrier means per-repo isolation is trivially true).
598 Post-fix: must still PASS — the correlated subquery joins on repo_id.
599 """
600 repo_x = await create_repo(db_session, owner="gabriel")
601 repo_y = await create_repo(db_session, owner="gabriel")
602 repo_x_id = str(repo_x.repo_id)
603 repo_y_id = str(repo_y.repo_id)
604
605 now = _now()
606 # Repo X: mpack.index pending (should block repo X's prebuild, not repo Y's)
607 index_x = _job_row(
608 repo_x_id, "mpack.index",
609 {"mpack_key": "sha256:" + hashlib.sha256(b"mwp4-08-x").hexdigest(), "head": _fake_commit_id("hx"), "branch": "main"},
610 created_at=now - timedelta(seconds=5),
611 )
612 # Repo Y: fetch.mpack.prebuild with no pending mpack.index — must be claimable
613 prebuild_y = _job_row(
614 repo_y_id, "fetch.mpack.prebuild",
615 {"tip_commit_ids": [_fake_commit_id("mwp4-08-y-tip")]},
616 created_at=now,
617 )
618 db_session.add(index_x)
619 db_session.add(prebuild_y)
620 await db_session.commit()
621
622 claimed = await claim_next_job(db_session)
623 await db_session.commit()
624
625 # index_x has an earlier created_at than prebuild_y; without the per-repo
626 # scope, index_x would be claimed. With the scope, index_x is also claimable
627 # (it's not a prebuild), so it gets claimed first — that's fine. The key
628 # assertion: claim must not return None (prebuild_y must not be spuriously
629 # blocked by repo X's index).
630 assert claimed is not None, (
631 "A job must be claimable — repo Y's prebuild must not be blocked by "
632 "repo X's pending mpack.index"
633 )
634 # Either index_x (earlier timestamp) or prebuild_y — both are acceptable.
635 # The important thing is that prebuild_y is NOT blocked by a different repo's index.
636 assert claimed.job_type in ("mpack.index", "fetch.mpack.prebuild"), (
637 f"Unexpected job_type claimed: {claimed.job_type}"
638 )
639 assert claimed.repo_id != repo_x_id or claimed.job_type == "mpack.index", (
640 "If repo X's job is claimed it must be the index, not any blocked prebuild"
641 )
642
643
644 @pytest.mark.asyncio
645 async def test_mwp4_09_barrier_does_not_affect_other_job_types(
646 db_session: AsyncSession,
647 ) -> None:
648 """MWP4_09: the barrier filters only fetch.mpack.prebuild. A pending
649 mpack.index is itself claimable, and other job types (e.g. intel.structural)
650 are unaffected by the barrier.
651
652 Pre-fix: PASSES (no barrier — all types always claimable by timestamp).
653 Post-fix: must still PASS — the filter condition is
654 job_type == 'fetch.mpack.prebuild' AND EXISTS(pending index for same repo).
655 """
656 repo = await create_repo(db_session, owner="gabriel")
657 repo_id = str(repo.repo_id)
658
659 now = _now()
660 # A pending mpack.index — must still be claimable (barrier does not block index itself)
661 index_job = _job_row(
662 repo_id, "mpack.index",
663 {"mpack_key": "sha256:" + hashlib.sha256(b"mwp4-09").hexdigest(), "head": _fake_commit_id("h"), "branch": "main"},
664 created_at=now - timedelta(seconds=10),
665 )
666 # intel.structural — unrelated type, must be claimable regardless
667 intel_job = _job_row(
668 repo_id, "intel.structural",
669 {"head": _fake_commit_id("h2"), "branch": "main"},
670 created_at=now - timedelta(seconds=5),
671 )
672 db_session.add(index_job)
673 db_session.add(intel_job)
674 await db_session.commit()
675
676 # Claim twice — both jobs must be claimable (neither is a blocked prebuild)
677 first = await claim_next_job(db_session)
678 await db_session.commit()
679 second = await claim_next_job(db_session)
680 await db_session.commit()
681
682 assert first is not None, "First claim must succeed"
683 assert second is not None, "Second claim must succeed — barrier must not block non-prebuild types"
684 claimed_types = {first.job_type, second.job_type}
685 assert "mpack.index" in claimed_types, "mpack.index must be claimable"
686 assert "intel.structural" in claimed_types, "intel.structural must be claimable"
687
688
689 # ---------------------------------------------------------------------------
690 # Phase 3 — stop masking the index gap (MWP4_10)
691 # ---------------------------------------------------------------------------
692
693
694 @pytest.mark.asyncio
695 async def test_mwp4_10_fetch_not_indexed_error_propagates(
696 db_session: AsyncSession,
697 ) -> None:
698 """MWP4_10: when wire_fetch_mpack raises FetchNotIndexedError inside
699 process_fetch_mpack_prebuild_job, the handler must re-raise it instead of
700 returning a success result with tips_built=0.
701
702 This is the Phase 3 narrowing of the broad except Exception at
703 musehub_wire_fetch.py:1164-1167. After the fix, FetchNotIndexedError is
704 caught specifically, logged, and re-raised so the worker's fail_job path
705 records it as a retryable failure. The broad except Exception still handles
706 genuinely unexpected errors — it must NOT catch FetchNotIndexedError.
707
708 Setup: push one commit (creates MusehubCommit, MusehubBranch, MusehubMPackIndex),
709 then delete MusehubMPackIndex rows to simulate the state where a terminally-
710 failed mpack.index left the index gap (the Phase 2 barrier allows the prebuild
711 to run only after index work drains — terminal failures bypass the barrier).
712
713 Pre-fix (Phase 0 behavior): FAILS — process_fetch_mpack_prebuild_job swallows
714 FetchNotIndexedError and returns tips_built=0 (no propagation).
715
716 Post-fix (Phase 3): PASSES — FetchNotIndexedError propagates to the caller.
717 """
718 repo = await create_repo(db_session, owner="gabriel")
719 repo_id = str(repo.repo_id)
720
721 data = b"mwp4-10-blob-content"
722 oid = blob_id(data)
723 snap_id = hash_snapshot({"file.txt": oid})
724 cid = _pushed_cid("mwp4-10-c1", None, snap_id)
725
726 mpack_key = await _e2e_push(db_session, repo_id, cid, None, snap_id, "file.txt", oid, data)
727
728 # Delete the inline MusehubMPackIndex rows to simulate an empty index
729 # (replicates the state after a terminally-failed mpack.index job).
730 await db_session.execute(
731 delete(MusehubMPackIndex).where(MusehubMPackIndex.mpack_id == mpack_key)
732 )
733 await db_session.commit()
734
735 prebuild_job = (await db_session.execute(
736 select(MusehubBackgroundJob).where(
737 MusehubBackgroundJob.repo_id == repo_id,
738 MusehubBackgroundJob.job_type == "fetch.mpack.prebuild",
739 MusehubBackgroundJob.status == "pending",
740 ).limit(1)
741 )).scalar_one_or_none()
742 assert prebuild_job is not None, "A fetch.mpack.prebuild job must be enqueued after push"
743
744 # Phase 3: FetchNotIndexedError must propagate — not swallowed.
745 # The worker calls fail_job on this exception, giving it retry chances.
746 with pytest.raises(FetchNotIndexedError):
747 await process_fetch_mpack_prebuild_job(db_session, prebuild_job.job_id)
748 await db_session.rollback()
749
750
751 # ---------------------------------------------------------------------------
752 # Phase 4 — integration + stress (real worker claim loop)
753 # ---------------------------------------------------------------------------
754
755
756 async def _drain_via_claim_loop(
757 session: AsyncSession,
758 repo_id: str,
759 max_iters: int = 60,
760 ) -> tuple[list[str], int]:
761 """Drive claim_next_job → dispatch → complete_job, mirroring _process_one.
762
763 Returns:
764 claim_log: job_types in the order they were claimed
765 fetch_not_indexed_count: number of FetchNotIndexedError raised during drain
766 """
767 claim_log: list[str] = []
768 fetch_not_indexed_count = 0
769
770 for _ in range(max_iters):
771 job = await claim_next_job(session)
772 await session.commit()
773 if job is None:
774 break
775
776 jid = job.job_id
777 jtype = job.job_type
778 claim_log.append(jtype)
779
780 if jtype == "mpack.index":
781 await process_mpack_index_job(session, jid)
782 await session.commit()
783 elif jtype == "fetch.mpack.prebuild":
784 try:
785 await process_fetch_mpack_prebuild_job(session, jid)
786 await session.commit()
787 except FetchNotIndexedError:
788 fetch_not_indexed_count += 1
789 await session.rollback()
790 # all other types: pass (disabled in the worker)
791
792 await complete_job(session, jid)
793 await session.commit()
794
795 return claim_log, fetch_not_indexed_count
796
797
798 @pytest.mark.asyncio
799 async def test_mwp4_11_e2e_claim_loop_orders_index_before_prebuild(
800 db_session: AsyncSession,
801 ) -> None:
802 """MWP4_11 E2E: one push then real claim loop — mpack.index claimed and
803 completed before fetch.mpack.prebuild is ever claimed. A
804 MusehubFetchMPackCache row must land at the pushed tip after drain.
805
806 This mirrors the exact sequence musehub/worker.py::_process_one runs:
807 claim_next_job → dispatch → complete_job, repeat until queue empty.
808
809 The Phase 2 claim-layer barrier is the mechanism that enforces the ordering.
810 Without it, PostgreSQL is free to return the prebuild first (RC-4).
811
812 Assertions:
813 - mpack.index appears in the claim log before any fetch.mpack.prebuild
814 - A MusehubFetchMPackCache row exists for the pushed tip commit
815 """
816 repo = await create_repo(db_session, owner="gabriel")
817 repo_id = str(repo.repo_id)
818
819 data = b"mwp4-11-blob-content"
820 oid = blob_id(data)
821 snap_id = hash_snapshot({"file.txt": oid})
822 cid = _pushed_cid("mwp4-11-c1", None, snap_id)
823
824 await _e2e_push(db_session, repo_id, cid, None, snap_id, "file.txt", oid, data)
825
826 claim_log, fetch_not_indexed_count = await _drain_via_claim_loop(db_session, repo_id)
827
828 assert "mpack.index" in claim_log, (
829 "mpack.index must be claimed during the drain"
830 )
831 assert "fetch.mpack.prebuild" in claim_log, (
832 "fetch.mpack.prebuild must be claimed during the drain"
833 )
834
835 first_prebuild_pos = claim_log.index("fetch.mpack.prebuild")
836 last_index_pos = max(i for i, t in enumerate(claim_log) if t == "mpack.index")
837 assert last_index_pos < first_prebuild_pos, (
838 f"Every mpack.index claim must complete before the first fetch.mpack.prebuild "
839 f"is claimed. claim_log={claim_log}"
840 )
841
842 assert fetch_not_indexed_count == 0, (
843 f"Zero FetchNotIndexedError must occur — the barrier ensures the prebuild "
844 f"only runs after the index drains. Got {fetch_not_indexed_count} error(s)."
845 )
846
847 cache_count = (await db_session.execute(
848 select(func.count()).select_from(MusehubFetchMPackCache).where(
849 MusehubFetchMPackCache.repo_id == repo_id,
850 MusehubFetchMPackCache.tip_commit_id == cid,
851 )
852 )).scalar_one()
853 assert cache_count >= 1, (
854 f"MusehubFetchMPackCache must have at least one row for tip={cid[:20]} "
855 f"after the prebuild runs. Got {cache_count} rows."
856 )
857
858
859 @pytest.mark.asyncio
860 async def test_mwp4_12_stress_n10_pushes_drain_with_no_ordering_violation(
861 db_session: AsyncSession,
862 ) -> None:
863 """MWP4_12 stress: N=10 back-to-back pushes with no intermediate drain, then
864 claim-loop drain. Three assertions mirror the musehub#109 acceptance criteria:
865
866 (a) Every fetch.mpack.prebuild claim happens only when zero mpack.index jobs
867 are pending or running for the same repo.
868 (b) Zero FetchNotIndexedError raised during the entire drain.
869 (c) Final clone cache covers the latest tip: all cache rows point to the same
870 mpack_id (len == 1) and a row exists for the latest pushed commit.
871
872 N=10 creates N distinct mpack.index jobs (per-key dedup allows them all) and
873 a single coalesced fetch.mpack.prebuild (coarse dedup folds them to one row).
874 The MWP-3 self-coalescing fix makes the prebuild re-read live branch tips at
875 run time, so it always targets the latest tip regardless of which push enqueued
876 it. RC-4 closes the remaining gap: the prebuild cannot be claimed until every
877 mpack.index for the repo has drained.
878 """
879 repo = await create_repo(db_session, owner="gabriel")
880 repo_id = str(repo.repo_id)
881
882 N = 10
883 cids = [_fake_commit_id(f"mwp4-12-c{i}") for i in range(N)]
884
885 for i in range(N):
886 data = f"mwp4-12-blob-{i}-unique-content".encode()
887 oid = blob_id(data)
888 snap_id = hash_snapshot({f"file_{i}.txt": oid})
889 parent = cids[i - 1] if i > 0 else None
890 cids[i] = _pushed_cid(f"mwp4-12-c{i}", parent, snap_id)
891 await _e2e_push(
892 db_session, repo_id, cids[i], parent,
893 snap_id, f"file_{i}.txt", oid, data,
894 )
895
896 # Verify N mpack.index jobs and exactly 1 fetch.mpack.prebuild were enqueued.
897 index_pending = (await db_session.execute(
898 select(func.count()).select_from(MusehubBackgroundJob).where(
899 MusehubBackgroundJob.repo_id == repo_id,
900 MusehubBackgroundJob.job_type == "mpack.index",
901 MusehubBackgroundJob.status == "pending",
902 )
903 )).scalar_one()
904 assert index_pending == N, (
905 f"Expected {N} pending mpack.index jobs after {N} pushes, got {index_pending}. "
906 f"Per-key dedup (MWP-3) must allow each distinct mpack_key through."
907 )
908
909 prebuild_pending = (await db_session.execute(
910 select(func.count()).select_from(MusehubBackgroundJob).where(
911 MusehubBackgroundJob.repo_id == repo_id,
912 MusehubBackgroundJob.job_type == "fetch.mpack.prebuild",
913 MusehubBackgroundJob.status == "pending",
914 )
915 )).scalar_one()
916 assert prebuild_pending == 1, (
917 f"Coarse dedup must collapse {N} prebuild enqueues to 1 pending row. "
918 f"Got {prebuild_pending}."
919 )
920
921 # Drive the claim loop and check barrier invariant at each prebuild claim.
922 prebuild_barrier_violations: list[int] = []
923
924 for _ in range(N + 30): # N indexes + 1 prebuild + ~20 intel.* subtypes
925 job = await claim_next_job(db_session)
926 await db_session.commit()
927 if job is None:
928 break
929
930 jid = job.job_id
931 jtype = job.job_type
932
933 if jtype == "fetch.mpack.prebuild":
934 # (a) At the moment of claim, no mpack.index must be pending/running
935 # for this repo. claim_next_job committed above — DB state is stable.
936 blocking = (await db_session.execute(
937 select(func.count()).select_from(MusehubBackgroundJob).where(
938 MusehubBackgroundJob.repo_id == repo_id,
939 MusehubBackgroundJob.job_type == "mpack.index",
940 MusehubBackgroundJob.status.in_(("pending", "running")),
941 )
942 )).scalar_one()
943 if blocking > 0:
944 prebuild_barrier_violations.append(blocking)
945
946 try:
947 await process_fetch_mpack_prebuild_job(db_session, jid)
948 await db_session.commit()
949 except FetchNotIndexedError:
950 await db_session.rollback()
951
952 elif jtype == "mpack.index":
953 await process_mpack_index_job(db_session, jid)
954 await db_session.commit()
955
956 # all other types: pass (mirrors the worker's disabled-jobs path)
957
958 await complete_job(db_session, jid)
959 await db_session.commit()
960
961 # (a) No prebuild was ever claimed while mpack.index was pending/running
962 assert not prebuild_barrier_violations, (
963 f"The RC-4 barrier was violated: fetch.mpack.prebuild was claimed while "
964 f"{prebuild_barrier_violations} mpack.index job(s) were still pending/running. "
965 f"The Phase 2 correlated NOT EXISTS barrier must prevent this."
966 )
967
968 # (b) Zero FetchNotIndexedError raised during drain
969 # Re-verify by checking all prebuild jobs are 'done' (not 'failed')
970 failed_prebuild = (await db_session.execute(
971 select(func.count()).select_from(MusehubBackgroundJob).where(
972 MusehubBackgroundJob.repo_id == repo_id,
973 MusehubBackgroundJob.job_type == "fetch.mpack.prebuild",
974 MusehubBackgroundJob.status == "failed",
975 )
976 )).scalar_one()
977 assert failed_prebuild == 0, (
978 f"{failed_prebuild} fetch.mpack.prebuild job(s) ended in 'failed' state. "
979 f"The barrier must ensure the prebuild runs only after all index work drains, "
980 f"producing zero FetchNotIndexedError occurrences."
981 )
982
983 # (c) Final clone cache covers the latest tip
984 latest_tip = cids[-1]
985 cache_for_tip = (await db_session.execute(
986 select(func.count()).select_from(MusehubFetchMPackCache).where(
987 MusehubFetchMPackCache.repo_id == repo_id,
988 MusehubFetchMPackCache.tip_commit_id == latest_tip,
989 )
990 )).scalar_one()
991 assert cache_for_tip >= 1, (
992 f"MusehubFetchMPackCache must have a row for the latest tip={latest_tip[:20]} "
993 f"after a full N={N} drain. Got {cache_for_tip} rows."
994 )
995
996 all_mpack_ids = (await db_session.execute(
997 select(MusehubFetchMPackCache.mpack_id).where(
998 MusehubFetchMPackCache.repo_id == repo_id,
999 )
1000 )).scalars().all()
1001 assert len(all_mpack_ids) >= 1, "At least one cache row must exist after full drain"
1002 unique_mpacks = set(all_mpack_ids)
1003 assert len(unique_mpacks) == 1, (
1004 f"All MusehubFetchMPackCache rows for the repo must point to the same mpack_id "
1005 f"after a full drain (single combined mpack covers all tips). "
1006 f"Got {len(unique_mpacks)} distinct mpack_ids."
1007 )
File History 8 commits
sha256:cb4b05e9d6caea5f351e3cb452ee6a1db8856169328ad96bfba9ee8e04b8afa6 fix(jobs): fence every execution of a job; an expired or su… Sonnet 5.5 minor ⚠ 4 days ago
sha256:383b9b15b1a7c18348bf797dbec252b8842199567b4b04371578fedd5f7891d4 merge: integrate push-ingest commit verification into dev Opus 5.5 minor ⚠ 11 days ago
sha256:39b90cebf3ab4c72ef67a29fe9d451ba1394084edf4053dff98878651f5d2378 test: build reproducible commit ids in push fixtures Opus 5.5 11 days ago
sha256:46854101acdde2a2481a7805887ac87ce91d35d23b62541b6165d4d0b480fa05 test(RC-4): add MWP4_11 E2E and MWP4_12 stress tests — Phas… Sonnet 4.6 141 days ago
sha256:b545dbcc8513abaecc2f275c75e499cecb6d6c85541d866b6b6ae7909dc15be9 fix(RC-4): narrow FetchNotIndexedError swallow in prebuild … Sonnet 4.6 patch 141 days ago
sha256:9e522d1d8fdb969514e26b71d13470b8f18e67ad010084764d67c1ac08b779de feat(mwp4/phase2): claim-layer dependency barrier in claim_… Sonnet 4.6 patch 141 days ago
sha256:598464aa2f30443e6ab9cb1d2283b79f26896decc4dbf29b7feda42c4b6b5b37 feat(mwp4/phase1): three-tier created_at stagger for fetch.… Sonnet 4.6 patch 141 days ago
sha256:2c523da45351334b5c4dbefed4dc3dd553b3faa8737a4e6caf301e5dc82141be test(mwp4): Phase 0 RED reproduction tests for RC-4 ordering race Sonnet 4.6 141 days ago