test_mwp4_prebuild_ordering.py
python
sha256:cb4b05e9d6caea5f351e3cb452ee6a1db8856169328ad96bfba9ee8e04b8afa6
fix(jobs): fence every execution of a job; an expired or su…
Sonnet 5.5
minor
⚠ breaking
4 days ago
| 1 | """MWP-4 — Gate fetch.mpack.prebuild behind mpack.index (fixes RC-4). |
| 2 | |
| 3 | Sub-ticket: musehub#109 — https://staging.musehub.ai/gabriel/musehub/issues/109 |
| 4 | Master: muse#58 — https://staging.musehub.ai/gabriel/muse/issues/58 |
| 5 | Predecessors: |
| 6 | musehub#106 (MWP-1, generation authority) — closed. |
| 7 | musehub#107 (MWP-2, walk fallback) — closed. |
| 8 | musehub#108 (MWP-3, job-enqueue idempotency) — closed. |
| 9 | |
| 10 | Root cause (RC-4): ``claim_next_job`` orders only by ``created_at`` with no |
| 11 | tiebreaker. When ``mpack.index`` and ``fetch.mpack.prebuild`` share the same |
| 12 | ``created_at`` (the normal case from ``enqueue_push_intel``), or the prebuild |
| 13 | lands with an earlier timestamp due to cross-push interleaving, PostgreSQL may |
| 14 | return the prebuild first. The prebuild then calls ``wire_fetch_mpack`` with |
| 15 | ``force_build=True``, hits the index-coverage check at |
| 16 | ``musehub_wire_fetch.py:681–694``, and raises ``FetchNotIndexedError`` because |
| 17 | ``MusehubMPackIndex`` is still empty. The broad ``except Exception`` at |
| 18 | ``musehub_wire_fetch.py:1164–1167`` swallows the error silently: ``tips_built`` |
| 19 | stays ``0``, no ``MusehubFetchMPackCache`` row is written, and the next clone |
| 20 | returns 503. |
| 21 | |
| 22 | Phase 0 — RED reproduction (no production code changes): |
| 23 | MWP4_01 RED — ``claim_next_job`` returns the prebuild before the index |
| 24 | when the prebuild has an earlier ``created_at``. Asserts |
| 25 | the desired correct behavior → FAILS before Phase 2 fix. |
| 26 | MWP4_02 — handler-level proof: prebuild run before index swallows |
| 27 | FetchNotIndexedError and writes no cache row. Asserts the |
| 28 | broken outcome → PASSES before fix (proves the bug). |
| 29 | Must be updated in Phase 3 when the swallow is narrowed. |
| 30 | """ |
| 31 | |
| 32 | from __future__ import annotations |
| 33 | |
| 34 | import hashlib |
| 35 | from datetime import datetime, timedelta, timezone |
| 36 | |
| 37 | import pytest |
| 38 | from sqlalchemy import delete, func, select |
| 39 | from sqlalchemy.ext.asyncio import AsyncSession |
| 40 | |
| 41 | from muse.core.ids import hash_commit, hash_snapshot |
| 42 | from muse.core.mpack import build_wire_mpack |
| 43 | from muse.core.types import blob_id |
| 44 | from musehub.core.genesis import compute_job_id |
| 45 | from musehub.db.musehub_jobs_models import MusehubBackgroundJob |
| 46 | from musehub.db.musehub_repo_models import MusehubFetchMPackCache, MusehubMPackIndex |
| 47 | from musehub.services.musehub_jobs import claim_next_job, complete_job, enqueue_push_intel |
| 48 | from musehub.services.musehub_wire_fetch import process_fetch_mpack_prebuild_job |
| 49 | from musehub.services.musehub_wire_push import process_mpack_index_job, wire_push_unpack_mpack |
| 50 | from musehub.services.musehub_wire_shared import FetchNotIndexedError |
| 51 | from tests.factories import create_repo |
| 52 | |
| 53 | |
| 54 | # --------------------------------------------------------------------------- |
| 55 | # Helpers |
| 56 | # --------------------------------------------------------------------------- |
| 57 | |
| 58 | |
| 59 | def _fake_commit_id(seed: str) -> str: |
| 60 | return "sha256:" + hashlib.sha256(seed.encode()).hexdigest() |
| 61 | |
| 62 | |
| 63 | def _now() -> datetime: |
| 64 | return datetime.now(tz=timezone.utc) |
| 65 | |
| 66 | |
| 67 | def _job_row( |
| 68 | repo_id: str, |
| 69 | job_type: str, |
| 70 | payload: dict, |
| 71 | created_at: datetime, |
| 72 | status: str = "pending", |
| 73 | ) -> MusehubBackgroundJob: |
| 74 | """Build a MusehubBackgroundJob row ready to be added to a session.""" |
| 75 | job_id = compute_job_id(repo_id, job_type, created_at.isoformat()) |
| 76 | return MusehubBackgroundJob( |
| 77 | job_id=job_id, |
| 78 | repo_id=repo_id, |
| 79 | job_type=job_type, |
| 80 | payload=payload, |
| 81 | status=status, |
| 82 | created_at=created_at, |
| 83 | attempt=0, |
| 84 | ) |
| 85 | |
| 86 | |
| 87 | _E2E_OWNER = "gabriel" |
| 88 | |
| 89 | |
| 90 | # id -> (seed, committed_at) for commits built by _e2e_raw_commit and pushed; |
| 91 | # push ingest verifies that each pushed id reproduces from its fields. |
| 92 | _PUSHED: dict[str, tuple[str, str]] = {} |
| 93 | |
| 94 | |
| 95 | def _pushed_cid(seed: str, parent: str | None, snap_id: str) -> str: |
| 96 | """The id of the commit _e2e_raw_commit builds for *seed*.""" |
| 97 | at = _now().isoformat() |
| 98 | cid = hash_commit(parent_ids=[parent] if parent else [], snapshot_id=snap_id, message=f"commit {seed}", |
| 99 | committed_at_iso=at, author=_E2E_OWNER, signer_public_key="") |
| 100 | _PUSHED[cid] = (seed, at) |
| 101 | return cid |
| 102 | |
| 103 | |
| 104 | def _e2e_raw_commit( |
| 105 | cid: str, |
| 106 | parent: str | None, |
| 107 | snap_id: str, |
| 108 | branch: str = "main", |
| 109 | ) -> dict: |
| 110 | seed, at = _PUSHED.get(cid, (cid[:12], _now().isoformat())) |
| 111 | return { |
| 112 | "commit_id": cid, |
| 113 | "branch": branch, |
| 114 | "message": f"commit {seed}", |
| 115 | "author": _E2E_OWNER, |
| 116 | "committed_at": at, |
| 117 | "parent_commit_id": parent, |
| 118 | "parent2_commit_id": None, |
| 119 | "snapshot_id": snap_id, |
| 120 | "agent_id": "", |
| 121 | "model_id": "", |
| 122 | "toolchain_id": "", |
| 123 | "sem_ver_bump": "none", |
| 124 | "breaking_changes": [], |
| 125 | "signature": "", |
| 126 | "signer_key_id": "", |
| 127 | "signer_public_key": "", |
| 128 | "prompt_hash": "", |
| 129 | } |
| 130 | |
| 131 | |
| 132 | async def _e2e_push( |
| 133 | session: AsyncSession, |
| 134 | repo_id: str, |
| 135 | cid: str, |
| 136 | parent: str | None, |
| 137 | snap_id: str, |
| 138 | fpath: str, |
| 139 | oid: str, |
| 140 | data: bytes, |
| 141 | branch: str = "main", |
| 142 | ) -> str: |
| 143 | """Build a single-commit mpack, store it in the conftest MemoryBackend, push it. |
| 144 | |
| 145 | The conftest autouse fixture patches ``musehub.storage.backends.get_backend`` |
| 146 | to a ``MemoryBackend`` instance shared across all modules. Pre-seeding |
| 147 | ``backend._mpacks[key]`` is equivalent to a MinIO PUT — both |
| 148 | ``wire_push_unpack_mpack`` and ``process_mpack_index_job`` call |
| 149 | ``get_backend()`` and see the same bytes. |
| 150 | |
| 151 | Returns the ``mpack_key`` so callers can locate the enqueued index job. |
| 152 | """ |
| 153 | import musehub.storage.backends as _b_mod |
| 154 | backend = _b_mod.get_backend() |
| 155 | |
| 156 | mpack_bytes = build_wire_mpack({ |
| 157 | "commits": [_e2e_raw_commit(cid, parent, snap_id, branch=branch)], |
| 158 | "snapshots": [{ |
| 159 | "snapshot_id": snap_id, |
| 160 | "parent_snapshot_id": None, |
| 161 | "delta_upsert": {fpath: oid}, |
| 162 | "delta_remove": [], |
| 163 | "directories": [], |
| 164 | }], |
| 165 | "blobs": [{"object_id": oid, "content": data}], |
| 166 | "tags": [], |
| 167 | }) |
| 168 | mpack_key = blob_id(mpack_bytes) |
| 169 | backend._mpacks[mpack_key] = mpack_bytes |
| 170 | |
| 171 | await wire_push_unpack_mpack( |
| 172 | session, |
| 173 | repo_id, |
| 174 | mpack_key, |
| 175 | pusher_id=_E2E_OWNER, |
| 176 | branch=branch, |
| 177 | head_commit_id=cid, |
| 178 | commits_count=1, |
| 179 | blobs_count=1, |
| 180 | force=True, |
| 181 | ) |
| 182 | await session.commit() |
| 183 | return mpack_key |
| 184 | |
| 185 | |
| 186 | # --------------------------------------------------------------------------- |
| 187 | # MWP4_01 — claim_next_job returns prebuild before a pending index (RED) |
| 188 | # --------------------------------------------------------------------------- |
| 189 | |
| 190 | |
| 191 | @pytest.mark.asyncio |
| 192 | async def test_mwp4_01_claim_returns_prebuild_before_index( |
| 193 | db_session: AsyncSession, |
| 194 | ) -> None: |
| 195 | """MWP4_01 RED: claim_next_job must NOT return fetch.mpack.prebuild while |
| 196 | an mpack.index for the same repo is pending. |
| 197 | |
| 198 | Setup: insert a fetch.mpack.prebuild with created_at = now-5s (earlier |
| 199 | than the mpack.index at now). This simulates the cross-push race where |
| 200 | push A's prebuild is enqueued before push B's index job arrives. |
| 201 | |
| 202 | Pre-fix (expected to FAIL): |
| 203 | claim_next_job orders solely by created_at; the prebuild (earlier |
| 204 | timestamp) is returned ahead of the index. The assertion |
| 205 | ``claimed.job_type == "mpack.index"`` therefore fails. |
| 206 | |
| 207 | Post-fix (Phase 2 — correlated NOT EXISTS barrier): |
| 208 | The barrier prevents fetch.mpack.prebuild from being claimed while any |
| 209 | mpack.index for the same repo_id is pending or running. The index is |
| 210 | returned instead, and the prebuild waits until the index drains. |
| 211 | """ |
| 212 | repo = await create_repo(db_session, owner="gabriel") |
| 213 | repo_id = str(repo.repo_id) |
| 214 | |
| 215 | now = _now() |
| 216 | earlier = now - timedelta(seconds=5) |
| 217 | |
| 218 | # Insert prebuild FIRST with an earlier timestamp to force it ahead in the |
| 219 | # ORDER BY created_at ordering — this is the race we're fixing. |
| 220 | prebuild = _job_row( |
| 221 | repo_id, |
| 222 | "fetch.mpack.prebuild", |
| 223 | {"tip_commit_ids": [_fake_commit_id("mwp4-01-tip")]}, |
| 224 | created_at=earlier, |
| 225 | ) |
| 226 | index = _job_row( |
| 227 | repo_id, |
| 228 | "mpack.index", |
| 229 | { |
| 230 | "mpack_key": "sha256:" + hashlib.sha256(b"mwp4-01-mpack").hexdigest(), |
| 231 | "head": _fake_commit_id("mwp4-01-head"), |
| 232 | "branch": "main", |
| 233 | }, |
| 234 | created_at=now, |
| 235 | ) |
| 236 | db_session.add(prebuild) |
| 237 | db_session.add(index) |
| 238 | await db_session.commit() |
| 239 | |
| 240 | # Claim the next job. |
| 241 | # Pre-fix: returns the prebuild (earliest created_at wins) → assertion fails. |
| 242 | # Post-fix: barrier skips the prebuild; returns the index instead. |
| 243 | claimed = await claim_next_job(db_session) |
| 244 | await db_session.commit() |
| 245 | |
| 246 | assert claimed is not None, "Expected a job to be claimed — queue has two pending rows." |
| 247 | assert claimed.job_type == "mpack.index", ( |
| 248 | f"claim_next_job must skip fetch.mpack.prebuild while mpack.index is " |
| 249 | f"pending for the same repo_id, but returned '{claimed.job_type}'. " |
| 250 | f"The prebuild must be gated behind all pending index jobs (RC-4 barrier)." |
| 251 | ) |
| 252 | |
| 253 | |
| 254 | # --------------------------------------------------------------------------- |
| 255 | # MWP4_02 — prebuild run before index swallows FetchNotIndexedError (bug proof) |
| 256 | # --------------------------------------------------------------------------- |
| 257 | |
| 258 | |
| 259 | @pytest.mark.asyncio |
| 260 | async def test_mwp4_02_prebuild_swallows_fetch_not_indexed_error( |
| 261 | db_session: AsyncSession, |
| 262 | ) -> None: |
| 263 | """MWP4_02 (updated for Phase 3): process_fetch_mpack_prebuild_job raises |
| 264 | FetchNotIndexedError when MusehubMPackIndex is empty — no longer swallowed. |
| 265 | |
| 266 | Setup: push one commit so MusehubObject, MusehubCommit, and MusehubBranch |
| 267 | rows exist. wire_push_unpack_mpack writes MusehubMPackIndex rows at step 7d |
| 268 | synchronously, so we explicitly delete them after the push to simulate the |
| 269 | race state: DB has commits/snapshots but MusehubMPackIndex is empty (as if |
| 270 | the mpack.index job hasn't run yet and has also terminally failed, bypassing |
| 271 | the Phase 2 barrier). |
| 272 | |
| 273 | Phase 0 behavior (pre-Phase 3): FetchNotIndexedError was silently swallowed |
| 274 | by the broad ``except Exception``; tips_built==0, no cache row. |
| 275 | |
| 276 | Phase 3 behavior: FetchNotIndexedError is caught specifically and re-raised |
| 277 | so the worker's fail_job records it as a failed job (retryable). The broad |
| 278 | ``except Exception`` still handles genuinely unexpected errors. |
| 279 | |
| 280 | With the Phase 2 barrier in place this path is only reachable when an |
| 281 | mpack.index has terminally failed — exactly the case that should surface as |
| 282 | a failed job, not a silent empty cache. |
| 283 | """ |
| 284 | repo = await create_repo(db_session, owner="gabriel") |
| 285 | repo_id = str(repo.repo_id) |
| 286 | |
| 287 | data = b"mwp4-02-blob-content" |
| 288 | oid = blob_id(data) |
| 289 | snap_id = hash_snapshot({"file.txt": oid}) |
| 290 | cid = _pushed_cid("mwp4-02-c1", None, snap_id) |
| 291 | |
| 292 | # Push one commit — creates MusehubObject, MusehubCommit, MusehubBranch, |
| 293 | # enqueues mpack.index + fetch.mpack.prebuild, AND writes MusehubMPackIndex |
| 294 | # rows at step 7d of wire_push_unpack_mpack (synchronous inline indexing). |
| 295 | mpack_key = await _e2e_push(db_session, repo_id, cid, None, snap_id, "file.txt", oid, data) |
| 296 | |
| 297 | # Simulate the RC-4 race: delete the inline MusehubMPackIndex rows that the |
| 298 | # push wrote. This replicates the state where the prebuild job is claimed |
| 299 | # before the mpack.index job has had a chance to run — MusehubCommit, |
| 300 | # MusehubSnapshot, and MusehubBranch are populated, but MusehubMPackIndex |
| 301 | # is empty for this mpack_key. |
| 302 | await db_session.execute( |
| 303 | delete(MusehubMPackIndex).where(MusehubMPackIndex.mpack_id == mpack_key) |
| 304 | ) |
| 305 | await db_session.commit() |
| 306 | |
| 307 | # Locate the prebuild job enqueued by wire_push_unpack_mpack. |
| 308 | prebuild_job = (await db_session.execute( |
| 309 | select(MusehubBackgroundJob).where( |
| 310 | MusehubBackgroundJob.repo_id == repo_id, |
| 311 | MusehubBackgroundJob.job_type == "fetch.mpack.prebuild", |
| 312 | MusehubBackgroundJob.status == "pending", |
| 313 | ).limit(1) |
| 314 | )).scalar_one_or_none() |
| 315 | assert prebuild_job is not None, ( |
| 316 | "wire_push_unpack_mpack must enqueue a fetch.mpack.prebuild job after a push." |
| 317 | ) |
| 318 | |
| 319 | # Phase 3: FetchNotIndexedError must propagate (no longer swallowed). |
| 320 | # The worker's fail_job path will record this as a retryable failure. |
| 321 | with pytest.raises(FetchNotIndexedError): |
| 322 | await process_fetch_mpack_prebuild_job(db_session, prebuild_job.job_id) |
| 323 | await db_session.rollback() |
| 324 | |
| 325 | |
| 326 | # --------------------------------------------------------------------------- |
| 327 | # MWP4_03 — fetch.mpack.prebuild created_at is strictly after mpack.index |
| 328 | # --------------------------------------------------------------------------- |
| 329 | |
| 330 | |
| 331 | @pytest.mark.asyncio |
| 332 | async def test_mwp4_03_prebuild_created_at_after_index( |
| 333 | db_session: AsyncSession, |
| 334 | ) -> None: |
| 335 | """MWP4_03: enqueue_push_intel must assign fetch.mpack.prebuild a created_at |
| 336 | strictly greater than mpack.index and strictly less than intel.code.* subtypes. |
| 337 | |
| 338 | This is the Phase 1 defense-in-depth stagger: |
| 339 | |
| 340 | t+0s mpack.index, intel.code, intel.structural, gc, ... (foundation) |
| 341 | t+1s fetch.mpack.prebuild (NEW middle tier) |
| 342 | t+2s intel.code.* subtypes (subtype tier) |
| 343 | |
| 344 | Pre-fix: FAILS — fetch.mpack.prebuild and mpack.index share the same |
| 345 | created_at (both are in _FOUNDATION_TYPES at t+0s). |
| 346 | |
| 347 | Post-fix (Phase 1): PASSES — fetch.mpack.prebuild gets created_at_prebuild |
| 348 | (t+1s) while mpack.index stays at the foundation tier (t+0s). |
| 349 | """ |
| 350 | repo = await create_repo(db_session, owner="gabriel") |
| 351 | repo_id = str(repo.repo_id) |
| 352 | fake_head = _fake_commit_id("mwp4-03-head") |
| 353 | fake_mpack_key = "sha256:" + hashlib.sha256(b"mwp4-03-mpack").hexdigest() |
| 354 | |
| 355 | await enqueue_push_intel( |
| 356 | db_session, |
| 357 | repo_id=repo_id, |
| 358 | head=fake_head, |
| 359 | domain_id=None, # triggers code-domain subtypes (intel.code.*) |
| 360 | branch="main", |
| 361 | owner=None, |
| 362 | mpack_key=fake_mpack_key, |
| 363 | ) |
| 364 | await db_session.commit() |
| 365 | |
| 366 | # Fetch created_at for mpack.index |
| 367 | index_row = (await db_session.execute( |
| 368 | select(MusehubBackgroundJob.created_at).where( |
| 369 | MusehubBackgroundJob.repo_id == repo_id, |
| 370 | MusehubBackgroundJob.job_type == "mpack.index", |
| 371 | ).limit(1) |
| 372 | )).scalar_one_or_none() |
| 373 | assert index_row is not None, "mpack.index job must be enqueued by enqueue_push_intel" |
| 374 | |
| 375 | # Fetch created_at for fetch.mpack.prebuild |
| 376 | prebuild_row = (await db_session.execute( |
| 377 | select(MusehubBackgroundJob.created_at).where( |
| 378 | MusehubBackgroundJob.repo_id == repo_id, |
| 379 | MusehubBackgroundJob.job_type == "fetch.mpack.prebuild", |
| 380 | ).limit(1) |
| 381 | )).scalar_one_or_none() |
| 382 | assert prebuild_row is not None, "fetch.mpack.prebuild job must be enqueued by enqueue_push_intel" |
| 383 | |
| 384 | # Fetch created_at for any intel.code.* subtype (the t+2s tier) |
| 385 | subtype_row = (await db_session.execute( |
| 386 | select(MusehubBackgroundJob.created_at).where( |
| 387 | MusehubBackgroundJob.repo_id == repo_id, |
| 388 | MusehubBackgroundJob.job_type == "intel.code.coupling", |
| 389 | ).limit(1) |
| 390 | )).scalar_one_or_none() |
| 391 | assert subtype_row is not None, ( |
| 392 | "intel.code.coupling job must be enqueued when domain_id=None — " |
| 393 | "needed to verify the three-tier ordering" |
| 394 | ) |
| 395 | |
| 396 | index_at = index_row |
| 397 | prebuild_at = prebuild_row |
| 398 | subtype_at = subtype_row |
| 399 | |
| 400 | # Tier 1 → Tier 2: prebuild strictly after index |
| 401 | assert prebuild_at > index_at, ( |
| 402 | f"fetch.mpack.prebuild created_at ({prebuild_at.isoformat()}) must be " |
| 403 | f"strictly greater than mpack.index created_at ({index_at.isoformat()}). " |
| 404 | f"Pre-fix: they share the same timestamp (both in _FOUNDATION_TYPES at t+0s). " |
| 405 | f"Phase 1 fix: assign fetch.mpack.prebuild created_at_prebuild = t+1s." |
| 406 | ) |
| 407 | |
| 408 | # Tier 2 → Tier 3: prebuild strictly before intel.code.* subtypes |
| 409 | assert prebuild_at < subtype_at, ( |
| 410 | f"fetch.mpack.prebuild created_at ({prebuild_at.isoformat()}) must be " |
| 411 | f"strictly less than intel.code.* subtype created_at ({subtype_at.isoformat()}). " |
| 412 | f"The three-tier ordering must be: foundation < prebuild < subtypes." |
| 413 | ) |
| 414 | |
| 415 | |
| 416 | # --------------------------------------------------------------------------- |
| 417 | # Phase 2 — claim-layer dependency barrier (MWP4_04 – MWP4_09) |
| 418 | # --------------------------------------------------------------------------- |
| 419 | |
| 420 | |
| 421 | @pytest.mark.asyncio |
| 422 | async def test_mwp4_04_prebuild_not_claimed_while_index_pending( |
| 423 | db_session: AsyncSession, |
| 424 | ) -> None: |
| 425 | """MWP4_04: claim_next_job must NOT return fetch.mpack.prebuild while any |
| 426 | mpack.index for the same repo is pending — even when the prebuild has an |
| 427 | earlier created_at (the exact cross-push race from MWP4_01). |
| 428 | |
| 429 | Pre-fix: FAILS — claim returns the prebuild (earlier timestamp wins). |
| 430 | Post-fix (Phase 2 barrier): PASSES — the correlated NOT EXISTS filter |
| 431 | blocks the prebuild; claim returns the mpack.index instead. |
| 432 | """ |
| 433 | repo = await create_repo(db_session, owner="gabriel") |
| 434 | repo_id = str(repo.repo_id) |
| 435 | |
| 436 | now = _now() |
| 437 | earlier = now - timedelta(seconds=5) |
| 438 | |
| 439 | prebuild = _job_row( |
| 440 | repo_id, "fetch.mpack.prebuild", |
| 441 | {"tip_commit_ids": [_fake_commit_id("mwp4-04-tip")]}, |
| 442 | created_at=earlier, |
| 443 | ) |
| 444 | index = _job_row( |
| 445 | repo_id, "mpack.index", |
| 446 | {"mpack_key": "sha256:" + hashlib.sha256(b"mwp4-04").hexdigest(), "head": _fake_commit_id("h"), "branch": "main"}, |
| 447 | created_at=now, |
| 448 | ) |
| 449 | db_session.add(prebuild) |
| 450 | db_session.add(index) |
| 451 | await db_session.commit() |
| 452 | |
| 453 | claimed = await claim_next_job(db_session) |
| 454 | await db_session.commit() |
| 455 | |
| 456 | assert claimed is not None, "A claimable job must exist (mpack.index is pending)" |
| 457 | assert claimed.job_type == "mpack.index", ( |
| 458 | f"claim_next_job must skip fetch.mpack.prebuild while mpack.index is " |
| 459 | f"pending for the same repo — the Phase 2 barrier. Got: {claimed.job_type}" |
| 460 | ) |
| 461 | |
| 462 | |
| 463 | @pytest.mark.asyncio |
| 464 | async def test_mwp4_05_prebuild_claimable_after_index_done( |
| 465 | db_session: AsyncSession, |
| 466 | ) -> None: |
| 467 | """MWP4_05: once mpack.index is in terminal state 'done', the prebuild |
| 468 | must be claimable on the next claim_next_job call. |
| 469 | |
| 470 | Pre-fix: PASSES (no barrier means prebuild is always claimable by timestamp). |
| 471 | Post-fix: must still PASS — 'done' is not a blocking state. |
| 472 | """ |
| 473 | repo = await create_repo(db_session, owner="gabriel") |
| 474 | repo_id = str(repo.repo_id) |
| 475 | |
| 476 | now = _now() |
| 477 | index_done = _job_row( |
| 478 | repo_id, "mpack.index", |
| 479 | {"mpack_key": "sha256:" + hashlib.sha256(b"mwp4-05").hexdigest(), "head": _fake_commit_id("h"), "branch": "main"}, |
| 480 | created_at=now - timedelta(seconds=10), |
| 481 | status="done", |
| 482 | ) |
| 483 | prebuild = _job_row( |
| 484 | repo_id, "fetch.mpack.prebuild", |
| 485 | {"tip_commit_ids": [_fake_commit_id("mwp4-05-tip")]}, |
| 486 | created_at=now, |
| 487 | ) |
| 488 | db_session.add(index_done) |
| 489 | db_session.add(prebuild) |
| 490 | await db_session.commit() |
| 491 | |
| 492 | claimed = await claim_next_job(db_session) |
| 493 | await db_session.commit() |
| 494 | |
| 495 | assert claimed is not None, "fetch.mpack.prebuild must be claimable when mpack.index is done" |
| 496 | assert claimed.job_type == "fetch.mpack.prebuild", ( |
| 497 | f"mpack.index in 'done' state must not block the prebuild. Got: {claimed.job_type}" |
| 498 | ) |
| 499 | |
| 500 | |
| 501 | @pytest.mark.asyncio |
| 502 | async def test_mwp4_06_prebuild_blocked_while_index_running( |
| 503 | db_session: AsyncSession, |
| 504 | ) -> None: |
| 505 | """MWP4_06: an mpack.index in 'running' state (already claimed by another |
| 506 | worker) must also block the prebuild — the barrier covers both pending and |
| 507 | running. |
| 508 | |
| 509 | Setup: mpack.index is running (claimed), fetch.mpack.prebuild is pending. |
| 510 | Expected: claim_next_job returns None (the only pending job is blocked). |
| 511 | |
| 512 | Pre-fix: FAILS — prebuild is returned (no barrier). |
| 513 | Post-fix: PASSES — running mpack.index blocks the prebuild. |
| 514 | """ |
| 515 | repo = await create_repo(db_session, owner="gabriel") |
| 516 | repo_id = str(repo.repo_id) |
| 517 | |
| 518 | now = _now() |
| 519 | index_running = _job_row( |
| 520 | repo_id, "mpack.index", |
| 521 | {"mpack_key": "sha256:" + hashlib.sha256(b"mwp4-06").hexdigest(), "head": _fake_commit_id("h"), "branch": "main"}, |
| 522 | created_at=now - timedelta(seconds=10), |
| 523 | status="running", |
| 524 | ) |
| 525 | prebuild = _job_row( |
| 526 | repo_id, "fetch.mpack.prebuild", |
| 527 | {"tip_commit_ids": [_fake_commit_id("mwp4-06-tip")]}, |
| 528 | created_at=now, |
| 529 | ) |
| 530 | db_session.add(index_running) |
| 531 | db_session.add(prebuild) |
| 532 | await db_session.commit() |
| 533 | |
| 534 | claimed = await claim_next_job(db_session) |
| 535 | await db_session.commit() |
| 536 | |
| 537 | assert claimed is None, ( |
| 538 | f"claim_next_job must return None when the only pending job is a " |
| 539 | f"fetch.mpack.prebuild blocked by a running mpack.index. Got: " |
| 540 | f"{claimed.job_type if claimed else None}" |
| 541 | ) |
| 542 | |
| 543 | |
| 544 | @pytest.mark.asyncio |
| 545 | async def test_mwp4_07_terminal_index_does_not_block_prebuild( |
| 546 | db_session: AsyncSession, |
| 547 | ) -> None: |
| 548 | """MWP4_07: mpack.index rows in terminal states (failed, quarantined) must |
| 549 | NOT block the prebuild. A permanently-failed index should surface via |
| 550 | fail_job, not silently hold up the prebuild forever. |
| 551 | |
| 552 | Pre-fix: PASSES (no barrier means prebuild is always claimable). |
| 553 | Post-fix: must still PASS — only pending/running are blocking states. |
| 554 | """ |
| 555 | repo = await create_repo(db_session, owner="gabriel") |
| 556 | repo_id = str(repo.repo_id) |
| 557 | |
| 558 | now = _now() |
| 559 | for i, status in enumerate(("failed", "quarantined")): |
| 560 | index_terminal = _job_row( |
| 561 | repo_id, "mpack.index", |
| 562 | {"mpack_key": "sha256:" + hashlib.sha256(f"mwp4-07-{status}".encode()).hexdigest(), |
| 563 | "head": _fake_commit_id(f"h-{status}"), "branch": "main"}, |
| 564 | # Distinct created_at per row to avoid PK collision in compute_job_id |
| 565 | created_at=now - timedelta(seconds=10 + i), |
| 566 | status=status, |
| 567 | ) |
| 568 | db_session.add(index_terminal) |
| 569 | |
| 570 | prebuild = _job_row( |
| 571 | repo_id, "fetch.mpack.prebuild", |
| 572 | {"tip_commit_ids": [_fake_commit_id("mwp4-07-tip")]}, |
| 573 | created_at=now, |
| 574 | ) |
| 575 | db_session.add(prebuild) |
| 576 | await db_session.commit() |
| 577 | |
| 578 | claimed = await claim_next_job(db_session) |
| 579 | await db_session.commit() |
| 580 | |
| 581 | assert claimed is not None, ( |
| 582 | "fetch.mpack.prebuild must be claimable when all mpack.index rows are terminal" |
| 583 | ) |
| 584 | assert claimed.job_type == "fetch.mpack.prebuild", ( |
| 585 | f"Terminal mpack.index states (failed/quarantined) must not gate the " |
| 586 | f"prebuild. Got: {claimed.job_type}" |
| 587 | ) |
| 588 | |
| 589 | |
| 590 | @pytest.mark.asyncio |
| 591 | async def test_mwp4_08_barrier_is_per_repo( |
| 592 | db_session: AsyncSession, |
| 593 | ) -> None: |
| 594 | """MWP4_08: the barrier is scoped per repo_id. A pending mpack.index in |
| 595 | repo X must not block a fetch.mpack.prebuild in repo Y. |
| 596 | |
| 597 | Pre-fix: PASSES (no barrier means per-repo isolation is trivially true). |
| 598 | Post-fix: must still PASS — the correlated subquery joins on repo_id. |
| 599 | """ |
| 600 | repo_x = await create_repo(db_session, owner="gabriel") |
| 601 | repo_y = await create_repo(db_session, owner="gabriel") |
| 602 | repo_x_id = str(repo_x.repo_id) |
| 603 | repo_y_id = str(repo_y.repo_id) |
| 604 | |
| 605 | now = _now() |
| 606 | # Repo X: mpack.index pending (should block repo X's prebuild, not repo Y's) |
| 607 | index_x = _job_row( |
| 608 | repo_x_id, "mpack.index", |
| 609 | {"mpack_key": "sha256:" + hashlib.sha256(b"mwp4-08-x").hexdigest(), "head": _fake_commit_id("hx"), "branch": "main"}, |
| 610 | created_at=now - timedelta(seconds=5), |
| 611 | ) |
| 612 | # Repo Y: fetch.mpack.prebuild with no pending mpack.index — must be claimable |
| 613 | prebuild_y = _job_row( |
| 614 | repo_y_id, "fetch.mpack.prebuild", |
| 615 | {"tip_commit_ids": [_fake_commit_id("mwp4-08-y-tip")]}, |
| 616 | created_at=now, |
| 617 | ) |
| 618 | db_session.add(index_x) |
| 619 | db_session.add(prebuild_y) |
| 620 | await db_session.commit() |
| 621 | |
| 622 | claimed = await claim_next_job(db_session) |
| 623 | await db_session.commit() |
| 624 | |
| 625 | # index_x has an earlier created_at than prebuild_y; without the per-repo |
| 626 | # scope, index_x would be claimed. With the scope, index_x is also claimable |
| 627 | # (it's not a prebuild), so it gets claimed first — that's fine. The key |
| 628 | # assertion: claim must not return None (prebuild_y must not be spuriously |
| 629 | # blocked by repo X's index). |
| 630 | assert claimed is not None, ( |
| 631 | "A job must be claimable — repo Y's prebuild must not be blocked by " |
| 632 | "repo X's pending mpack.index" |
| 633 | ) |
| 634 | # Either index_x (earlier timestamp) or prebuild_y — both are acceptable. |
| 635 | # The important thing is that prebuild_y is NOT blocked by a different repo's index. |
| 636 | assert claimed.job_type in ("mpack.index", "fetch.mpack.prebuild"), ( |
| 637 | f"Unexpected job_type claimed: {claimed.job_type}" |
| 638 | ) |
| 639 | assert claimed.repo_id != repo_x_id or claimed.job_type == "mpack.index", ( |
| 640 | "If repo X's job is claimed it must be the index, not any blocked prebuild" |
| 641 | ) |
| 642 | |
| 643 | |
| 644 | @pytest.mark.asyncio |
| 645 | async def test_mwp4_09_barrier_does_not_affect_other_job_types( |
| 646 | db_session: AsyncSession, |
| 647 | ) -> None: |
| 648 | """MWP4_09: the barrier filters only fetch.mpack.prebuild. A pending |
| 649 | mpack.index is itself claimable, and other job types (e.g. intel.structural) |
| 650 | are unaffected by the barrier. |
| 651 | |
| 652 | Pre-fix: PASSES (no barrier — all types always claimable by timestamp). |
| 653 | Post-fix: must still PASS — the filter condition is |
| 654 | job_type == 'fetch.mpack.prebuild' AND EXISTS(pending index for same repo). |
| 655 | """ |
| 656 | repo = await create_repo(db_session, owner="gabriel") |
| 657 | repo_id = str(repo.repo_id) |
| 658 | |
| 659 | now = _now() |
| 660 | # A pending mpack.index — must still be claimable (barrier does not block index itself) |
| 661 | index_job = _job_row( |
| 662 | repo_id, "mpack.index", |
| 663 | {"mpack_key": "sha256:" + hashlib.sha256(b"mwp4-09").hexdigest(), "head": _fake_commit_id("h"), "branch": "main"}, |
| 664 | created_at=now - timedelta(seconds=10), |
| 665 | ) |
| 666 | # intel.structural — unrelated type, must be claimable regardless |
| 667 | intel_job = _job_row( |
| 668 | repo_id, "intel.structural", |
| 669 | {"head": _fake_commit_id("h2"), "branch": "main"}, |
| 670 | created_at=now - timedelta(seconds=5), |
| 671 | ) |
| 672 | db_session.add(index_job) |
| 673 | db_session.add(intel_job) |
| 674 | await db_session.commit() |
| 675 | |
| 676 | # Claim twice — both jobs must be claimable (neither is a blocked prebuild) |
| 677 | first = await claim_next_job(db_session) |
| 678 | await db_session.commit() |
| 679 | second = await claim_next_job(db_session) |
| 680 | await db_session.commit() |
| 681 | |
| 682 | assert first is not None, "First claim must succeed" |
| 683 | assert second is not None, "Second claim must succeed — barrier must not block non-prebuild types" |
| 684 | claimed_types = {first.job_type, second.job_type} |
| 685 | assert "mpack.index" in claimed_types, "mpack.index must be claimable" |
| 686 | assert "intel.structural" in claimed_types, "intel.structural must be claimable" |
| 687 | |
| 688 | |
| 689 | # --------------------------------------------------------------------------- |
| 690 | # Phase 3 — stop masking the index gap (MWP4_10) |
| 691 | # --------------------------------------------------------------------------- |
| 692 | |
| 693 | |
| 694 | @pytest.mark.asyncio |
| 695 | async def test_mwp4_10_fetch_not_indexed_error_propagates( |
| 696 | db_session: AsyncSession, |
| 697 | ) -> None: |
| 698 | """MWP4_10: when wire_fetch_mpack raises FetchNotIndexedError inside |
| 699 | process_fetch_mpack_prebuild_job, the handler must re-raise it instead of |
| 700 | returning a success result with tips_built=0. |
| 701 | |
| 702 | This is the Phase 3 narrowing of the broad except Exception at |
| 703 | musehub_wire_fetch.py:1164-1167. After the fix, FetchNotIndexedError is |
| 704 | caught specifically, logged, and re-raised so the worker's fail_job path |
| 705 | records it as a retryable failure. The broad except Exception still handles |
| 706 | genuinely unexpected errors — it must NOT catch FetchNotIndexedError. |
| 707 | |
| 708 | Setup: push one commit (creates MusehubCommit, MusehubBranch, MusehubMPackIndex), |
| 709 | then delete MusehubMPackIndex rows to simulate the state where a terminally- |
| 710 | failed mpack.index left the index gap (the Phase 2 barrier allows the prebuild |
| 711 | to run only after index work drains — terminal failures bypass the barrier). |
| 712 | |
| 713 | Pre-fix (Phase 0 behavior): FAILS — process_fetch_mpack_prebuild_job swallows |
| 714 | FetchNotIndexedError and returns tips_built=0 (no propagation). |
| 715 | |
| 716 | Post-fix (Phase 3): PASSES — FetchNotIndexedError propagates to the caller. |
| 717 | """ |
| 718 | repo = await create_repo(db_session, owner="gabriel") |
| 719 | repo_id = str(repo.repo_id) |
| 720 | |
| 721 | data = b"mwp4-10-blob-content" |
| 722 | oid = blob_id(data) |
| 723 | snap_id = hash_snapshot({"file.txt": oid}) |
| 724 | cid = _pushed_cid("mwp4-10-c1", None, snap_id) |
| 725 | |
| 726 | mpack_key = await _e2e_push(db_session, repo_id, cid, None, snap_id, "file.txt", oid, data) |
| 727 | |
| 728 | # Delete the inline MusehubMPackIndex rows to simulate an empty index |
| 729 | # (replicates the state after a terminally-failed mpack.index job). |
| 730 | await db_session.execute( |
| 731 | delete(MusehubMPackIndex).where(MusehubMPackIndex.mpack_id == mpack_key) |
| 732 | ) |
| 733 | await db_session.commit() |
| 734 | |
| 735 | prebuild_job = (await db_session.execute( |
| 736 | select(MusehubBackgroundJob).where( |
| 737 | MusehubBackgroundJob.repo_id == repo_id, |
| 738 | MusehubBackgroundJob.job_type == "fetch.mpack.prebuild", |
| 739 | MusehubBackgroundJob.status == "pending", |
| 740 | ).limit(1) |
| 741 | )).scalar_one_or_none() |
| 742 | assert prebuild_job is not None, "A fetch.mpack.prebuild job must be enqueued after push" |
| 743 | |
| 744 | # Phase 3: FetchNotIndexedError must propagate — not swallowed. |
| 745 | # The worker calls fail_job on this exception, giving it retry chances. |
| 746 | with pytest.raises(FetchNotIndexedError): |
| 747 | await process_fetch_mpack_prebuild_job(db_session, prebuild_job.job_id) |
| 748 | await db_session.rollback() |
| 749 | |
| 750 | |
| 751 | # --------------------------------------------------------------------------- |
| 752 | # Phase 4 — integration + stress (real worker claim loop) |
| 753 | # --------------------------------------------------------------------------- |
| 754 | |
| 755 | |
| 756 | async def _drain_via_claim_loop( |
| 757 | session: AsyncSession, |
| 758 | repo_id: str, |
| 759 | max_iters: int = 60, |
| 760 | ) -> tuple[list[str], int]: |
| 761 | """Drive claim_next_job → dispatch → complete_job, mirroring _process_one. |
| 762 | |
| 763 | Returns: |
| 764 | claim_log: job_types in the order they were claimed |
| 765 | fetch_not_indexed_count: number of FetchNotIndexedError raised during drain |
| 766 | """ |
| 767 | claim_log: list[str] = [] |
| 768 | fetch_not_indexed_count = 0 |
| 769 | |
| 770 | for _ in range(max_iters): |
| 771 | job = await claim_next_job(session) |
| 772 | await session.commit() |
| 773 | if job is None: |
| 774 | break |
| 775 | |
| 776 | jid = job.job_id |
| 777 | jtype = job.job_type |
| 778 | claim_log.append(jtype) |
| 779 | |
| 780 | if jtype == "mpack.index": |
| 781 | await process_mpack_index_job(session, jid) |
| 782 | await session.commit() |
| 783 | elif jtype == "fetch.mpack.prebuild": |
| 784 | try: |
| 785 | await process_fetch_mpack_prebuild_job(session, jid) |
| 786 | await session.commit() |
| 787 | except FetchNotIndexedError: |
| 788 | fetch_not_indexed_count += 1 |
| 789 | await session.rollback() |
| 790 | # all other types: pass (disabled in the worker) |
| 791 | |
| 792 | await complete_job(session, jid) |
| 793 | await session.commit() |
| 794 | |
| 795 | return claim_log, fetch_not_indexed_count |
| 796 | |
| 797 | |
| 798 | @pytest.mark.asyncio |
| 799 | async def test_mwp4_11_e2e_claim_loop_orders_index_before_prebuild( |
| 800 | db_session: AsyncSession, |
| 801 | ) -> None: |
| 802 | """MWP4_11 E2E: one push then real claim loop — mpack.index claimed and |
| 803 | completed before fetch.mpack.prebuild is ever claimed. A |
| 804 | MusehubFetchMPackCache row must land at the pushed tip after drain. |
| 805 | |
| 806 | This mirrors the exact sequence musehub/worker.py::_process_one runs: |
| 807 | claim_next_job → dispatch → complete_job, repeat until queue empty. |
| 808 | |
| 809 | The Phase 2 claim-layer barrier is the mechanism that enforces the ordering. |
| 810 | Without it, PostgreSQL is free to return the prebuild first (RC-4). |
| 811 | |
| 812 | Assertions: |
| 813 | - mpack.index appears in the claim log before any fetch.mpack.prebuild |
| 814 | - A MusehubFetchMPackCache row exists for the pushed tip commit |
| 815 | """ |
| 816 | repo = await create_repo(db_session, owner="gabriel") |
| 817 | repo_id = str(repo.repo_id) |
| 818 | |
| 819 | data = b"mwp4-11-blob-content" |
| 820 | oid = blob_id(data) |
| 821 | snap_id = hash_snapshot({"file.txt": oid}) |
| 822 | cid = _pushed_cid("mwp4-11-c1", None, snap_id) |
| 823 | |
| 824 | await _e2e_push(db_session, repo_id, cid, None, snap_id, "file.txt", oid, data) |
| 825 | |
| 826 | claim_log, fetch_not_indexed_count = await _drain_via_claim_loop(db_session, repo_id) |
| 827 | |
| 828 | assert "mpack.index" in claim_log, ( |
| 829 | "mpack.index must be claimed during the drain" |
| 830 | ) |
| 831 | assert "fetch.mpack.prebuild" in claim_log, ( |
| 832 | "fetch.mpack.prebuild must be claimed during the drain" |
| 833 | ) |
| 834 | |
| 835 | first_prebuild_pos = claim_log.index("fetch.mpack.prebuild") |
| 836 | last_index_pos = max(i for i, t in enumerate(claim_log) if t == "mpack.index") |
| 837 | assert last_index_pos < first_prebuild_pos, ( |
| 838 | f"Every mpack.index claim must complete before the first fetch.mpack.prebuild " |
| 839 | f"is claimed. claim_log={claim_log}" |
| 840 | ) |
| 841 | |
| 842 | assert fetch_not_indexed_count == 0, ( |
| 843 | f"Zero FetchNotIndexedError must occur — the barrier ensures the prebuild " |
| 844 | f"only runs after the index drains. Got {fetch_not_indexed_count} error(s)." |
| 845 | ) |
| 846 | |
| 847 | cache_count = (await db_session.execute( |
| 848 | select(func.count()).select_from(MusehubFetchMPackCache).where( |
| 849 | MusehubFetchMPackCache.repo_id == repo_id, |
| 850 | MusehubFetchMPackCache.tip_commit_id == cid, |
| 851 | ) |
| 852 | )).scalar_one() |
| 853 | assert cache_count >= 1, ( |
| 854 | f"MusehubFetchMPackCache must have at least one row for tip={cid[:20]} " |
| 855 | f"after the prebuild runs. Got {cache_count} rows." |
| 856 | ) |
| 857 | |
| 858 | |
| 859 | @pytest.mark.asyncio |
| 860 | async def test_mwp4_12_stress_n10_pushes_drain_with_no_ordering_violation( |
| 861 | db_session: AsyncSession, |
| 862 | ) -> None: |
| 863 | """MWP4_12 stress: N=10 back-to-back pushes with no intermediate drain, then |
| 864 | claim-loop drain. Three assertions mirror the musehub#109 acceptance criteria: |
| 865 | |
| 866 | (a) Every fetch.mpack.prebuild claim happens only when zero mpack.index jobs |
| 867 | are pending or running for the same repo. |
| 868 | (b) Zero FetchNotIndexedError raised during the entire drain. |
| 869 | (c) Final clone cache covers the latest tip: all cache rows point to the same |
| 870 | mpack_id (len == 1) and a row exists for the latest pushed commit. |
| 871 | |
| 872 | N=10 creates N distinct mpack.index jobs (per-key dedup allows them all) and |
| 873 | a single coalesced fetch.mpack.prebuild (coarse dedup folds them to one row). |
| 874 | The MWP-3 self-coalescing fix makes the prebuild re-read live branch tips at |
| 875 | run time, so it always targets the latest tip regardless of which push enqueued |
| 876 | it. RC-4 closes the remaining gap: the prebuild cannot be claimed until every |
| 877 | mpack.index for the repo has drained. |
| 878 | """ |
| 879 | repo = await create_repo(db_session, owner="gabriel") |
| 880 | repo_id = str(repo.repo_id) |
| 881 | |
| 882 | N = 10 |
| 883 | cids = [_fake_commit_id(f"mwp4-12-c{i}") for i in range(N)] |
| 884 | |
| 885 | for i in range(N): |
| 886 | data = f"mwp4-12-blob-{i}-unique-content".encode() |
| 887 | oid = blob_id(data) |
| 888 | snap_id = hash_snapshot({f"file_{i}.txt": oid}) |
| 889 | parent = cids[i - 1] if i > 0 else None |
| 890 | cids[i] = _pushed_cid(f"mwp4-12-c{i}", parent, snap_id) |
| 891 | await _e2e_push( |
| 892 | db_session, repo_id, cids[i], parent, |
| 893 | snap_id, f"file_{i}.txt", oid, data, |
| 894 | ) |
| 895 | |
| 896 | # Verify N mpack.index jobs and exactly 1 fetch.mpack.prebuild were enqueued. |
| 897 | index_pending = (await db_session.execute( |
| 898 | select(func.count()).select_from(MusehubBackgroundJob).where( |
| 899 | MusehubBackgroundJob.repo_id == repo_id, |
| 900 | MusehubBackgroundJob.job_type == "mpack.index", |
| 901 | MusehubBackgroundJob.status == "pending", |
| 902 | ) |
| 903 | )).scalar_one() |
| 904 | assert index_pending == N, ( |
| 905 | f"Expected {N} pending mpack.index jobs after {N} pushes, got {index_pending}. " |
| 906 | f"Per-key dedup (MWP-3) must allow each distinct mpack_key through." |
| 907 | ) |
| 908 | |
| 909 | prebuild_pending = (await db_session.execute( |
| 910 | select(func.count()).select_from(MusehubBackgroundJob).where( |
| 911 | MusehubBackgroundJob.repo_id == repo_id, |
| 912 | MusehubBackgroundJob.job_type == "fetch.mpack.prebuild", |
| 913 | MusehubBackgroundJob.status == "pending", |
| 914 | ) |
| 915 | )).scalar_one() |
| 916 | assert prebuild_pending == 1, ( |
| 917 | f"Coarse dedup must collapse {N} prebuild enqueues to 1 pending row. " |
| 918 | f"Got {prebuild_pending}." |
| 919 | ) |
| 920 | |
| 921 | # Drive the claim loop and check barrier invariant at each prebuild claim. |
| 922 | prebuild_barrier_violations: list[int] = [] |
| 923 | |
| 924 | for _ in range(N + 30): # N indexes + 1 prebuild + ~20 intel.* subtypes |
| 925 | job = await claim_next_job(db_session) |
| 926 | await db_session.commit() |
| 927 | if job is None: |
| 928 | break |
| 929 | |
| 930 | jid = job.job_id |
| 931 | jtype = job.job_type |
| 932 | |
| 933 | if jtype == "fetch.mpack.prebuild": |
| 934 | # (a) At the moment of claim, no mpack.index must be pending/running |
| 935 | # for this repo. claim_next_job committed above — DB state is stable. |
| 936 | blocking = (await db_session.execute( |
| 937 | select(func.count()).select_from(MusehubBackgroundJob).where( |
| 938 | MusehubBackgroundJob.repo_id == repo_id, |
| 939 | MusehubBackgroundJob.job_type == "mpack.index", |
| 940 | MusehubBackgroundJob.status.in_(("pending", "running")), |
| 941 | ) |
| 942 | )).scalar_one() |
| 943 | if blocking > 0: |
| 944 | prebuild_barrier_violations.append(blocking) |
| 945 | |
| 946 | try: |
| 947 | await process_fetch_mpack_prebuild_job(db_session, jid) |
| 948 | await db_session.commit() |
| 949 | except FetchNotIndexedError: |
| 950 | await db_session.rollback() |
| 951 | |
| 952 | elif jtype == "mpack.index": |
| 953 | await process_mpack_index_job(db_session, jid) |
| 954 | await db_session.commit() |
| 955 | |
| 956 | # all other types: pass (mirrors the worker's disabled-jobs path) |
| 957 | |
| 958 | await complete_job(db_session, jid) |
| 959 | await db_session.commit() |
| 960 | |
| 961 | # (a) No prebuild was ever claimed while mpack.index was pending/running |
| 962 | assert not prebuild_barrier_violations, ( |
| 963 | f"The RC-4 barrier was violated: fetch.mpack.prebuild was claimed while " |
| 964 | f"{prebuild_barrier_violations} mpack.index job(s) were still pending/running. " |
| 965 | f"The Phase 2 correlated NOT EXISTS barrier must prevent this." |
| 966 | ) |
| 967 | |
| 968 | # (b) Zero FetchNotIndexedError raised during drain |
| 969 | # Re-verify by checking all prebuild jobs are 'done' (not 'failed') |
| 970 | failed_prebuild = (await db_session.execute( |
| 971 | select(func.count()).select_from(MusehubBackgroundJob).where( |
| 972 | MusehubBackgroundJob.repo_id == repo_id, |
| 973 | MusehubBackgroundJob.job_type == "fetch.mpack.prebuild", |
| 974 | MusehubBackgroundJob.status == "failed", |
| 975 | ) |
| 976 | )).scalar_one() |
| 977 | assert failed_prebuild == 0, ( |
| 978 | f"{failed_prebuild} fetch.mpack.prebuild job(s) ended in 'failed' state. " |
| 979 | f"The barrier must ensure the prebuild runs only after all index work drains, " |
| 980 | f"producing zero FetchNotIndexedError occurrences." |
| 981 | ) |
| 982 | |
| 983 | # (c) Final clone cache covers the latest tip |
| 984 | latest_tip = cids[-1] |
| 985 | cache_for_tip = (await db_session.execute( |
| 986 | select(func.count()).select_from(MusehubFetchMPackCache).where( |
| 987 | MusehubFetchMPackCache.repo_id == repo_id, |
| 988 | MusehubFetchMPackCache.tip_commit_id == latest_tip, |
| 989 | ) |
| 990 | )).scalar_one() |
| 991 | assert cache_for_tip >= 1, ( |
| 992 | f"MusehubFetchMPackCache must have a row for the latest tip={latest_tip[:20]} " |
| 993 | f"after a full N={N} drain. Got {cache_for_tip} rows." |
| 994 | ) |
| 995 | |
| 996 | all_mpack_ids = (await db_session.execute( |
| 997 | select(MusehubFetchMPackCache.mpack_id).where( |
| 998 | MusehubFetchMPackCache.repo_id == repo_id, |
| 999 | ) |
| 1000 | )).scalars().all() |
| 1001 | assert len(all_mpack_ids) >= 1, "At least one cache row must exist after full drain" |
| 1002 | unique_mpacks = set(all_mpack_ids) |
| 1003 | assert len(unique_mpacks) == 1, ( |
| 1004 | f"All MusehubFetchMPackCache rows for the repo must point to the same mpack_id " |
| 1005 | f"after a full drain (single combined mpack covers all tips). " |
| 1006 | f"Got {len(unique_mpacks)} distinct mpack_ids." |
| 1007 | ) |
File History
8 commits
sha256:cb4b05e9d6caea5f351e3cb452ee6a1db8856169328ad96bfba9ee8e04b8afa6
fix(jobs): fence every execution of a job; an expired or su…
Sonnet 5.5
minor
⚠
4 days ago
sha256:383b9b15b1a7c18348bf797dbec252b8842199567b4b04371578fedd5f7891d4
merge: integrate push-ingest commit verification into dev
Opus 5.5
minor
⚠
11 days ago
sha256:39b90cebf3ab4c72ef67a29fe9d451ba1394084edf4053dff98878651f5d2378
test: build reproducible commit ids in push fixtures
Opus 5.5
11 days ago
sha256:46854101acdde2a2481a7805887ac87ce91d35d23b62541b6165d4d0b480fa05
test(RC-4): add MWP4_11 E2E and MWP4_12 stress tests — Phas…
Sonnet 4.6
141 days ago
sha256:b545dbcc8513abaecc2f275c75e499cecb6d6c85541d866b6b6ae7909dc15be9
fix(RC-4): narrow FetchNotIndexedError swallow in prebuild …
Sonnet 4.6
patch
141 days ago
sha256:9e522d1d8fdb969514e26b71d13470b8f18e67ad010084764d67c1ac08b779de
feat(mwp4/phase2): claim-layer dependency barrier in claim_…
Sonnet 4.6
patch
141 days ago
sha256:598464aa2f30443e6ab9cb1d2283b79f26896decc4dbf29b7feda42c4b6b5b37
feat(mwp4/phase1): three-tier created_at stagger for fetch.…
Sonnet 4.6
patch
141 days ago
sha256:2c523da45351334b5c4dbefed4dc3dd553b3faa8737a4e6caf301e5dc82141be
test(mwp4): Phase 0 RED reproduction tests for RC-4 ordering race
Sonnet 4.6
141 days ago