gabriel / musehub public
test_fetch_mpack_prebuild.py python
699 lines 31.1 KB Hotspot Cold
Raw
sha256:1609a886bd55c9b689a2dda5b8ec9710a27030ff3cb80936f44f7b10a2f0b177 test(prebuild): RED — an unrecoverable missing blob must fa… Sonnet 5.5 4 days ago
1 """TDD — fetch.mpack.prebuild job handler and wire_fetch_mpack cache (issue #92 Phases 2–5).
2
3 Test IDs:
4 FMC_07 Unit: mock wire_fetch_mpack, confirm cache rows written for each tip,
5 confirm existing fresh entries are skipped
6 FMC_08 Integration: insert a job row, run the handler, verify
7 MusehubFetchMPackCache row exists with correct repo_id/tip/mpack_id
8 FMC_13 Unit: cache hit returns presigned URL without entering blob-load path
9 FMC_14 Unit: cache miss builds and writes cache row
10 FMC_18 Integration: enqueue_push_intel inserts fetch.mpack.prebuild with branch tips
11 FMC_20 Unit: gc_fetch_mpack_cache deletes expired rows + R2 objects; fresh rows untouched
12 """
13 from __future__ import annotations
14
15 import hashlib
16 from datetime import datetime, timedelta, timezone
17 from unittest.mock import AsyncMock, patch
18
19 import pytest
20 from sqlalchemy import select
21 from sqlalchemy.ext.asyncio import AsyncSession
22
23 from musehub.core.genesis import compute_job_id
24 from musehub.db.musehub_jobs_models import MusehubBackgroundJob
25 from musehub.db.musehub_repo_models import MusehubFetchMPackCache
26 from musehub.services.musehub_gc import gc_fetch_mpack_cache
27 from musehub.services.musehub_jobs import enqueue_push_intel
28 from musehub.services.musehub_wire_fetch import process_fetch_mpack_prebuild_job, wire_fetch_mpack
29 from musehub.services.musehub_wire_shared import MPackNotReadyError
30 from tests.factories import create_branch, create_repo
31
32
33 def _now() -> datetime:
34 return datetime.now(tz=timezone.utc)
35
36
37 def _fake_commit_id(seed: str) -> str:
38 return "sha256:" + hashlib.sha256(seed.encode()).hexdigest()
39
40
41 def _fake_mpack_id(seed: str) -> str:
42 return "sha256:" + hashlib.sha256(f"mpack-{seed}".encode()).hexdigest()
43
44
45 async def _insert_job(
46 session: AsyncSession,
47 repo_id: str,
48 tip_commit_ids: list[str],
49 ) -> str:
50 now = _now()
51 job_id = compute_job_id(repo_id, "fetch.mpack.prebuild", now.isoformat())
52 session.add(MusehubBackgroundJob(
53 job_id=job_id,
54 repo_id=repo_id,
55 job_type="fetch.mpack.prebuild",
56 payload={"tip_commit_ids": tip_commit_ids},
57 status="pending",
58 created_at=now,
59 attempt=0,
60 ))
61 await session.flush()
62 return job_id
63
64
65 # ── FMC_07 ────────────────────────────────────────────────────────────────────
66
67 @pytest.mark.tier2
68 async def test_fmc_07_builds_uncached_tips_in_one_combined_mpack(db_session: AsyncSession) -> None:
69 """FMC_07a: the handler builds all uncached tips in a SINGLE combined wire_fetch_mpack
70 call (want=[all uncached tips]) — not one call per tip. The per-tip cache rows are
71 written inside wire_fetch_mpack (covered by FMC_14), so it is mocked here and we assert
72 the handler's orchestration + counts only."""
73 repo = await create_repo(db_session, owner="gabriel", visibility="public")
74 tip_a = _fake_commit_id("tip-a")
75 tip_b = _fake_commit_id("tip-b")
76 combined_mpack = _fake_mpack_id("combined")
77
78 # The handler reads live branch tips from MusehubBranch (Phase 1 fix) —
79 # create branches so the handler sees tip_a and tip_b as the live tip set.
80 await create_branch(db_session, repo.repo_id, name="main", head_commit_id=tip_a)
81 await create_branch(db_session, repo.repo_id, name="dev", head_commit_id=tip_b)
82 job_id = await _insert_job(db_session, repo.repo_id, [tip_a, tip_b])
83 await db_session.commit()
84
85 with patch(
86 "musehub.services.musehub_wire_fetch.wire_fetch_mpack",
87 new_callable=AsyncMock,
88 return_value={"mpack_id": combined_mpack, "mpack_url": "https://r2.example/c", "commit_count": 2, "blob_count": 5},
89 ) as mock_build:
90 result = await process_fetch_mpack_prebuild_job(db_session, job_id)
91 await db_session.commit()
92
93 # One combined build covering both uncached tips — not one call per tip.
94 assert mock_build.call_count == 1
95 assert set(mock_build.call_args.kwargs["want"]) == {tip_a, tip_b}
96 assert result["tips_requested"] == 2
97 assert result["tips_built"] == 2
98 assert result["tips_skipped"] == 0
99
100
101 @pytest.mark.tier2
102 async def test_fmc_07b_skips_tips_with_fresh_cache(db_session: AsyncSession) -> None:
103 """FMC_07b: prebuild skips ALL tips only when every tip shares the same cached mpack_id.
104
105 When one tip is already cached with a DIFFERENT mpack_id than would be built for a
106 new tip, the prebuild must rebuild ALL tips together so that every tip ends up
107 pointing to the same combined mpack_id — required for the clone cache-hit check
108 (len(mpack_ids)==1 across all want tips).
109 """
110 repo = await create_repo(db_session, owner="gabriel", visibility="public")
111 tip_cached = _fake_commit_id("tip-cached")
112 tip_new = _fake_commit_id("tip-new")
113 existing_mpack = _fake_mpack_id("existing")
114 new_mpack = _fake_mpack_id("new")
115
116 # Pre-populate a fresh cache entry for tip_cached (different mpack_id from what
117 # the combined build will produce — diverged state).
118 cache_id = hashlib.sha256((repo.repo_id + tip_cached).encode()).hexdigest()
119 db_session.add(MusehubFetchMPackCache(
120 cache_id=cache_id,
121 repo_id=repo.repo_id,
122 tip_commit_id=tip_cached,
123 mpack_id=existing_mpack,
124 created_at=_now(),
125 expires_at=_now() + timedelta(days=7),
126 ))
127 # The handler reads live branch tips from MusehubBranch (Phase 1 fix).
128 await create_branch(db_session, repo.repo_id, name="main", head_commit_id=tip_cached)
129 await create_branch(db_session, repo.repo_id, name="dev", head_commit_id=tip_new)
130 job_id = await _insert_job(db_session, repo.repo_id, [tip_cached, tip_new])
131 await db_session.commit()
132
133 with patch(
134 "musehub.services.musehub_wire_fetch.wire_fetch_mpack",
135 new_callable=AsyncMock,
136 return_value={"mpack_id": new_mpack, "mpack_url": "https://r2.example/new", "commit_count": 2, "blob_count": 2},
137 ) as mock_build:
138 result = await process_fetch_mpack_prebuild_job(db_session, job_id)
139 await db_session.commit()
140
141 # Both tips must be rebuilt together so they share one mpack_id.
142 assert mock_build.call_count == 1
143 actual_want = mock_build.call_args[1].get("want") or mock_build.call_args[0][2]
144 assert set(actual_want) == {tip_cached, tip_new}, (
145 f"want must include ALL tips; got {actual_want}"
146 )
147 assert result["tips_built"] == 2
148 assert result["tips_skipped"] == 0
149
150 # The previously-cached entry must be updated to the new combined mpack_id.
151 cached_row = (await db_session.execute(
152 select(MusehubFetchMPackCache)
153 .where(MusehubFetchMPackCache.repo_id == repo.repo_id)
154 .where(MusehubFetchMPackCache.tip_commit_id == tip_cached)
155 )).scalar_one()
156 assert cached_row.mpack_id == new_mpack, (
157 "existing cache entry must be updated to the combined mpack_id"
158 )
159
160
161 @pytest.mark.tier2
162 async def test_fmc_07c_empty_payload_is_a_noop(db_session: AsyncSession) -> None:
163 """FMC_07c: job with no tip_commit_ids returns zeros without calling wire_fetch_mpack."""
164 repo = await create_repo(db_session, owner="gabriel", visibility="public")
165 job_id = await _insert_job(db_session, repo.repo_id, [])
166 await db_session.commit()
167
168 with patch(
169 "musehub.services.musehub_wire_fetch.wire_fetch_mpack",
170 new_callable=AsyncMock,
171 ) as mock_build:
172 result = await process_fetch_mpack_prebuild_job(db_session, job_id)
173
174 assert mock_build.call_count == 0
175 assert result["tips_requested"] == 0
176 assert result["tips_built"] == 0
177
178
179 # ── FMC_08 ────────────────────────────────────────────────────────────────────
180
181 @pytest.mark.tier2
182 async def test_fmc_08_cache_row_has_correct_fields(db_session: AsyncSession) -> None:
183 """FMC_08: running the handler end to end (through a REAL wire_fetch_mpack) writes a
184 cache row with matching repo_id, tip, and mpack_id. Cache rows are written inside
185 wire_fetch_mpack, so this exercises the full handler -> build -> cache-write chain
186 rather than mocking the build away. Only the expensive externals (DAG walk, mpack
187 bytes, storage) are stubbed — same pattern as FMC_14."""
188 from types import SimpleNamespace
189
190 repo = await create_repo(db_session, owner="gabriel", visibility="public")
191 tip = _fake_commit_id("integration-tip")
192 built_mpack = _fake_mpack_id("integration")
193
194 # The handler reads live branch tips from MusehubBranch (Phase 1 fix).
195 await create_branch(db_session, repo.repo_id, name="main", head_commit_id=tip)
196 job_id = await _insert_job(db_session, repo.repo_id, [tip])
197 await db_session.commit()
198
199 mock_backend = AsyncMock()
200 mock_backend.put_mpack.return_value = None
201 mock_backend.presign_mpack_get.return_value = "https://r2.example/int"
202 mock_backend.delete.return_value = None
203 fake_proxy = SimpleNamespace(commit_id=tip, snapshot_id=None, parent_ids=[])
204
205 with patch("musehub.services.musehub_wire_fetch.get_backend", return_value=mock_backend), \
206 patch("musehub.services.musehub_wire_fetch._wants_in_repo", new_callable=AsyncMock, side_effect=lambda _s, _r, want: set(want)), \
207 patch("musehub.services.musehub_wire_fetch._walk_commit_delta", new_callable=AsyncMock,
208 return_value={tip: fake_proxy}), \
209 patch("muse.core.mpack.build_wire_mpack", return_value=b"MUSE\x00fake-mpack"), \
210 patch("musehub.services.musehub_wire_fetch.blob_id", return_value=built_mpack):
211 result = await process_fetch_mpack_prebuild_job(db_session, job_id)
212 await db_session.commit()
213
214 assert result["tips_built"] == 1
215
216 row = (await db_session.execute(
217 select(MusehubFetchMPackCache)
218 .where(MusehubFetchMPackCache.repo_id == repo.repo_id)
219 .where(MusehubFetchMPackCache.tip_commit_id == tip)
220 )).scalar_one()
221
222 assert row.repo_id == repo.repo_id
223 assert row.tip_commit_id == tip
224 assert row.mpack_id == built_mpack
225 assert row.expires_at > _now()
226
227
228 # ── FMC_13 ────────────────────────────────────────────────────────────────────
229
230 @pytest.mark.tier2
231 async def test_fmc_13_cache_hit_returns_presigned_url_without_blob_load(db_session: AsyncSession) -> None:
232 """FMC_13: cache hit returns the presigned URL immediately; blob-load path is never entered."""
233 repo = await create_repo(db_session, owner="gabriel", visibility="public")
234 tip = _fake_commit_id("fmc13-tip")
235 cached_mpack = _fake_mpack_id("fmc13-tip")
236 expected_url = "https://r2.example/cached-fmc13"
237
238 # Pre-populate a fresh cache entry.
239 cache_id = hashlib.sha256((repo.repo_id + tip).encode()).hexdigest()
240 db_session.add(MusehubFetchMPackCache(
241 cache_id=cache_id,
242 repo_id=repo.repo_id,
243 tip_commit_id=tip,
244 mpack_id=cached_mpack,
245 created_at=_now(),
246 expires_at=_now() + timedelta(days=7),
247 ))
248 await db_session.commit()
249
250 mock_backend = AsyncMock()
251 mock_backend.presign_mpack_get.return_value = expected_url
252
253 with patch("musehub.services.musehub_wire_fetch.get_backend", return_value=mock_backend), \
254 patch("musehub.services.musehub_wire_fetch._walk_commit_delta", new_callable=AsyncMock) as mock_walk:
255 result = await wire_fetch_mpack(db_session, repo.repo_id, want=[tip], have=[])
256
257 # The expensive DAG walk must never have been called.
258 mock_walk.assert_not_called()
259 # The returned URL must match the presigned URL for the cached mpack.
260 assert result["mpack_url"] == expected_url
261 assert result["mpack_id"] == cached_mpack
262
263
264 @pytest.mark.tier2
265 async def test_fmc_13b_duplicate_branch_heads_share_one_cache_tip(db_session: AsyncSession) -> None:
266 """FMC_13b: main/dev at one commit is one cache requirement, not two.
267
268 ``muse clone`` sends one want entry per branch, so two branches at the same
269 commit produce duplicate commit IDs. The cache is keyed by unique commit
270 ID and must still hit instead of rebuilding forever.
271 """
272 repo = await create_repo(db_session, owner="gabriel", visibility="public")
273 shared_tip = _fake_commit_id("fmc13b-main-dev-tip")
274 cached_mpack = _fake_mpack_id("fmc13b-main-dev-tip")
275 expected_url = "https://r2.example/cached-fmc13b"
276
277 cache_id = hashlib.sha256((repo.repo_id + shared_tip).encode()).hexdigest()
278 db_session.add(MusehubFetchMPackCache(
279 cache_id=cache_id,
280 repo_id=repo.repo_id,
281 tip_commit_id=shared_tip,
282 mpack_id=cached_mpack,
283 created_at=_now(),
284 expires_at=_now() + timedelta(days=7),
285 ))
286 await db_session.commit()
287
288 mock_backend = AsyncMock()
289 mock_backend.presign_mpack_get.return_value = expected_url
290
291 with patch("musehub.services.musehub_wire_fetch.get_backend", return_value=mock_backend), \
292 patch("musehub.services.musehub_wire_fetch._walk_commit_delta", new_callable=AsyncMock) as mock_walk:
293 result = await wire_fetch_mpack(
294 db_session,
295 repo.repo_id,
296 want=[shared_tip, shared_tip],
297 have=[],
298 )
299
300 mock_walk.assert_not_called()
301 assert result["mpack_url"] == expected_url
302 assert result["mpack_id"] == cached_mpack
303
304
305 @pytest.mark.tier2
306 async def test_fmc_13c_prebuild_deduplicates_live_branch_heads(db_session: AsyncSession) -> None:
307 """FMC_13c: a prebuild skips one cached commit shared by two branches."""
308 repo = await create_repo(db_session, owner="gabriel", visibility="public")
309 shared_tip = _fake_commit_id("fmc13c-main-dev-tip")
310 cached_mpack = _fake_mpack_id("fmc13c-main-dev-tip")
311
312 await create_branch(db_session, repo.repo_id, name="main", head_commit_id=shared_tip)
313 await create_branch(db_session, repo.repo_id, name="dev", head_commit_id=shared_tip)
314 job_id = await _insert_job(db_session, repo.repo_id, [shared_tip, shared_tip])
315 cache_id = hashlib.sha256((repo.repo_id + shared_tip).encode()).hexdigest()
316 db_session.add(MusehubFetchMPackCache(
317 cache_id=cache_id,
318 repo_id=repo.repo_id,
319 tip_commit_id=shared_tip,
320 mpack_id=cached_mpack,
321 created_at=_now(),
322 expires_at=_now() + timedelta(days=7),
323 ))
324 await db_session.commit()
325
326 with patch(
327 "musehub.services.musehub_wire_fetch.wire_fetch_mpack",
328 new_callable=AsyncMock,
329 ) as mock_build:
330 result = await process_fetch_mpack_prebuild_job(db_session, job_id)
331
332 mock_build.assert_not_called()
333 assert result["tips_requested"] == 1
334 assert result["tips_built"] == 0
335 assert result["tips_skipped"] == 1
336
337
338 # ── FMC_14 ────────────────────────────────────────────────────────────────────
339
340 @pytest.mark.tier2
341 async def test_fmc_14_cache_miss_builds_and_writes_cache_row(db_session: AsyncSession) -> None:
342 """FMC_14: on a cache miss, wire_fetch_mpack builds the mpack and writes a cache row."""
343 from types import SimpleNamespace
344
345 repo = await create_repo(db_session, owner="gabriel", visibility="public")
346 tip = _fake_commit_id("fmc14-tip")
347 built_mpack = _fake_mpack_id("fmc14-tip")
348 built_url = "https://r2.example/built-fmc14"
349
350 # No cache row exists — this is a cold miss.
351 mock_backend = AsyncMock()
352 mock_backend.put_mpack.return_value = None
353 mock_backend.presign_mpack_get.return_value = built_url
354 mock_backend.delete.return_value = None
355
356 # _walk_commit_delta returns one proxy commit with no snapshot so that
357 # wire_fetch_mpack proceeds to build an empty-but-valid mpack without
358 # needing real commit/snapshot/object rows in the DB.
359 fake_proxy = SimpleNamespace(commit_id=tip, snapshot_id=None, parent_ids=[])
360
361 with patch("musehub.services.musehub_wire_fetch.get_backend", return_value=mock_backend), \
362 patch("musehub.services.musehub_wire_fetch._wants_in_repo", new_callable=AsyncMock, side_effect=lambda _s, _r, want: set(want)), \
363 patch("musehub.services.musehub_wire_fetch._walk_commit_delta", new_callable=AsyncMock,
364 return_value={tip: fake_proxy}), \
365 patch("muse.core.mpack.build_wire_mpack", return_value=b"MUSE\x00fake-mpack"), \
366 patch("musehub.services.musehub_wire_fetch.blob_id", return_value=built_mpack):
367 result = await wire_fetch_mpack(db_session, repo.repo_id, want=[tip], have=[], force_build=True)
368 await db_session.commit()
369
370 assert result["mpack_url"] == built_url
371 assert result["mpack_id"] == built_mpack
372
373 # The cache row must have been written.
374 row = (await db_session.execute(
375 select(MusehubFetchMPackCache)
376 .where(MusehubFetchMPackCache.repo_id == repo.repo_id)
377 .where(MusehubFetchMPackCache.tip_commit_id == tip)
378 )).scalar_one()
379 assert row.mpack_id == built_mpack
380 assert row.expires_at > _now()
381
382
383 # ── FMC_18 ────────────────────────────────────────────────────────────────────
384
385 @pytest.mark.tier2
386 async def test_fmc_18_enqueue_push_intel_creates_prebuild_job_with_branch_tips(
387 db_session: AsyncSession,
388 ) -> None:
389 """FMC_18: enqueue_push_intel enqueues fetch.mpack.prebuild with all branch tip commit IDs."""
390 repo = await create_repo(db_session, owner="gabriel", visibility="public")
391
392 tip_a = _fake_commit_id("branch-main")
393 tip_b = _fake_commit_id("branch-dev")
394 await create_branch(db_session, repo.repo_id, name="main", head_commit_id=tip_a)
395 await create_branch(db_session, repo.repo_id, name="dev", head_commit_id=tip_b)
396
397 await enqueue_push_intel(
398 db_session,
399 repo.repo_id,
400 head=tip_a,
401 domain_id=None,
402 branch="main",
403 )
404 await db_session.commit()
405
406 job_row = (await db_session.execute(
407 select(MusehubBackgroundJob)
408 .where(MusehubBackgroundJob.repo_id == repo.repo_id)
409 .where(MusehubBackgroundJob.job_type == "fetch.mpack.prebuild")
410 .where(MusehubBackgroundJob.status == "pending")
411 )).scalar_one()
412
413 tip_ids = set(job_row.payload.get("tip_commit_ids", []))
414 assert tip_a in tip_ids, f"main tip {tip_a[:20]} missing from payload"
415 assert tip_b in tip_ids, f"dev tip {tip_b[:20]} missing from payload"
416
417
418 # ── FMC_20 ────────────────────────────────────────────────────────────────────
419
420 @pytest.mark.tier2
421 async def test_fmc_20_gc_deletes_expired_rows_and_r2_objects_leaves_fresh_untouched(
422 db_session: AsyncSession,
423 ) -> None:
424 """FMC_20: gc_fetch_mpack_cache deletes expired rows + R2 objects; fresh rows survive."""
425 repo = await create_repo(db_session, owner="gabriel", visibility="public")
426
427 tip_expired_a = _fake_commit_id("expired-a")
428 tip_expired_b = _fake_commit_id("expired-b")
429 tip_fresh = _fake_commit_id("fresh")
430
431 mpack_expired_a = _fake_mpack_id("expired-a")
432 mpack_expired_b = _fake_mpack_id("expired-b")
433 mpack_fresh = _fake_mpack_id("fresh")
434
435 past = _now() - timedelta(days=1)
436 future = _now() + timedelta(days=6)
437
438 for tip, mpack, exp in [
439 (tip_expired_a, mpack_expired_a, past),
440 (tip_expired_b, mpack_expired_b, past),
441 (tip_fresh, mpack_fresh, future),
442 ]:
443 cache_id = hashlib.sha256((repo.repo_id + tip).encode()).hexdigest()
444 db_session.add(MusehubFetchMPackCache(
445 cache_id=cache_id,
446 repo_id=repo.repo_id,
447 tip_commit_id=tip,
448 mpack_id=mpack,
449 created_at=_now(),
450 expires_at=exp,
451 ))
452 await db_session.commit()
453
454 mock_backend = AsyncMock()
455 mock_backend.delete.return_value = None
456
457 with patch("musehub.services.musehub_gc.get_backend", return_value=mock_backend):
458 n_deleted = await gc_fetch_mpack_cache(db_session, repo.repo_id)
459 await db_session.commit()
460
461 assert n_deleted == 2
462
463 # R2 delete called exactly once for each expired mpack — in any order.
464 deleted_mpack_ids = {call.args[0] for call in mock_backend.delete.call_args_list}
465 assert mpack_expired_a in deleted_mpack_ids
466 assert mpack_expired_b in deleted_mpack_ids
467 assert mpack_fresh not in deleted_mpack_ids
468
469 # Expired rows gone from DB.
470 remaining = (await db_session.execute(
471 select(MusehubFetchMPackCache)
472 .where(MusehubFetchMPackCache.repo_id == repo.repo_id)
473 )).scalars().all()
474 remaining_tips = {r.tip_commit_id for r in remaining}
475 assert tip_expired_a not in remaining_tips
476 assert tip_expired_b not in remaining_tips
477
478 # Fresh row survives.
479 assert tip_fresh in remaining_tips
480
481
482 # ── FMC_21 ────────────────────────────────────────────────────────────────────
483 # Regression for musehub issue #39 (muse clone fails / hangs): a repo that goes
484 # stale (cache TTL expires with no further pushes) previously had no self-heal
485 # path — wire_fetch_mpack would raise MPackNotReadyError() forever on every
486 # retry, since nothing ever enqueued a new fetch.mpack.prebuild job for the
487 # current tips. Reproduced live against gabriel/muse on 2026-09-09: cache rows
488 # for both branch heads existed but had expired 5 days earlier, and the client
489 # retried the full 120s budget every single attempt with zero chance of success.
490
491 @pytest.mark.tier2
492 async def test_fmc_21_cache_miss_enqueues_prebuild_job(db_session: AsyncSession) -> None:
493 """FMC_21: on a real (non-force_build) cache miss, a fetch.mpack.prebuild job is enqueued."""
494 repo = await create_repo(db_session, owner="gabriel", visibility="public")
495 tip = _fake_commit_id("fmc21-tip")
496 await create_branch(db_session, repo.repo_id, name="main", head_commit_id=tip)
497
498 with pytest.raises(MPackNotReadyError):
499 await wire_fetch_mpack(db_session, repo.repo_id, want=[tip], have=[])
500 await db_session.commit()
501
502 job_row = (await db_session.execute(
503 select(MusehubBackgroundJob)
504 .where(MusehubBackgroundJob.repo_id == repo.repo_id)
505 .where(MusehubBackgroundJob.job_type == "fetch.mpack.prebuild")
506 .where(MusehubBackgroundJob.status == "pending")
507 )).scalar_one()
508 assert tip in set(job_row.payload.get("tip_commit_ids", []))
509
510
511 @pytest.mark.tier2
512 async def test_fmc_21b_repeated_cache_miss_does_not_duplicate_pending_job(db_session: AsyncSession) -> None:
513 """FMC_21b: a second MISS (e.g. the client's own retry loop) does not enqueue a duplicate job."""
514 repo = await create_repo(db_session, owner="gabriel", visibility="public")
515 tip = _fake_commit_id("fmc21b-tip")
516 await create_branch(db_session, repo.repo_id, name="main", head_commit_id=tip)
517
518 for _ in range(3):
519 with pytest.raises(MPackNotReadyError):
520 await wire_fetch_mpack(db_session, repo.repo_id, want=[tip], have=[])
521 await db_session.commit()
522
523 rows = (await db_session.execute(
524 select(MusehubBackgroundJob)
525 .where(MusehubBackgroundJob.repo_id == repo.repo_id)
526 .where(MusehubBackgroundJob.job_type == "fetch.mpack.prebuild")
527 .where(MusehubBackgroundJob.status == "pending")
528 )).scalars().all()
529 assert len(rows) == 1
530
531
532 @pytest.mark.tier2
533 async def test_fmc_21c_expired_cache_self_heals_end_to_end(db_session: AsyncSession) -> None:
534 """FMC_21c: expired cache -> MISS enqueues job -> running the job -> next fetch is a HIT.
535
536 This is the exact sequence that was broken for gabriel/muse: a dormant
537 repo's cache entry ages past its TTL and, before this fix, could never
538 recover without a fresh push.
539 """
540 from types import SimpleNamespace
541
542 repo = await create_repo(db_session, owner="gabriel", visibility="public")
543 tip = _fake_commit_id("fmc21c-tip")
544 stale_mpack = _fake_mpack_id("fmc21c-stale")
545 await create_branch(db_session, repo.repo_id, name="main", head_commit_id=tip)
546
547 # Seed an *expired* cache row -- exactly what we found live on staging.
548 cache_id = hashlib.sha256((repo.repo_id + tip).encode()).hexdigest()
549 db_session.add(MusehubFetchMPackCache(
550 cache_id=cache_id,
551 repo_id=repo.repo_id,
552 tip_commit_id=tip,
553 mpack_id=stale_mpack,
554 created_at=_now() - timedelta(days=8),
555 expires_at=_now() - timedelta(days=1),
556 ))
557 await db_session.commit()
558
559 # Step 1 -- MISS (expired row doesn't count as a hit) must self-heal.
560 with pytest.raises(MPackNotReadyError):
561 await wire_fetch_mpack(db_session, repo.repo_id, want=[tip], have=[])
562 await db_session.commit()
563
564 job_row = (await db_session.execute(
565 select(MusehubBackgroundJob)
566 .where(MusehubBackgroundJob.repo_id == repo.repo_id)
567 .where(MusehubBackgroundJob.job_type == "fetch.mpack.prebuild")
568 .where(MusehubBackgroundJob.status == "pending")
569 )).scalar_one()
570
571 # Step 2 -- run the job the worker would have picked up.
572 fresh_mpack = _fake_mpack_id("fmc21c-fresh")
573 fake_proxy = SimpleNamespace(commit_id=tip, snapshot_id=None, parent_ids=[])
574 mock_backend = AsyncMock()
575 mock_backend.put_mpack.return_value = None
576 mock_backend.presign_mpack_get.return_value = "https://r2.example/fresh-fmc21c"
577 mock_backend.delete.return_value = None
578
579 with patch("musehub.services.musehub_wire_fetch.get_backend", return_value=mock_backend), \
580 patch("musehub.services.musehub_wire_fetch._wants_in_repo", new_callable=AsyncMock, side_effect=lambda _s, _r, want: set(want)), \
581 patch("musehub.services.musehub_wire_fetch._walk_commit_delta", new_callable=AsyncMock,
582 return_value={tip: fake_proxy}), \
583 patch("muse.core.mpack.build_wire_mpack", return_value=b"MUSE\x00fake-mpack"), \
584 patch("musehub.services.musehub_wire_fetch.blob_id", return_value=fresh_mpack):
585 await process_fetch_mpack_prebuild_job(db_session, job_row.job_id)
586 await db_session.commit()
587
588 # Step 3 -- the next fetch is now a real cache HIT on the freshly built mpack.
589 with patch("musehub.services.musehub_wire_fetch.get_backend", return_value=mock_backend), \
590 patch("musehub.services.musehub_wire_fetch._walk_commit_delta", new_callable=AsyncMock) as mock_walk:
591 result = await wire_fetch_mpack(db_session, repo.repo_id, want=[tip], have=[])
592
593 mock_walk.assert_not_called()
594 assert result["mpack_id"] == fresh_mpack
595 assert result["mpack_id"] != stale_mpack
596
597
598 # ── MWP9 supersede-retry ────────────────────────────────────────────────────
599
600 @pytest.mark.tier2
601 async def test_mwp9_prebuild_rebuilds_when_superseded_during_build(db_session: AsyncSession) -> None:
602 """RED before the fix (found live, musehub#108 MWP9): a bulk repair batch's
603 20 commit-signature fixes landed while a single coalesced prebuild job was
604 already mid-build. Only 1 of 20 made it into that job's cached mpack,
605 because the build's data read started before the other 19 writes landed,
606 and nothing told the job its coverage was stale once it finished.
607
608 Simulates this by having the mocked wire_fetch_mpack set the job's own
609 superseded_at (as enqueue_job's dedup-skip path would, if another
610 enqueue_job call landed while this job was 'running') on its first call,
611 then return a second, different mpack_id on a required second call. The
612 handler must detect the stale coverage and rebuild once more before
613 finishing, so the final cache entry points to the SECOND (fresh) mpack,
614 not the first (stale) one.
615 """
616 repo = await create_repo(db_session, owner="gabriel", visibility="public")
617 tip = _fake_commit_id("tip-mwp9")
618 stale_mpack = _fake_mpack_id("stale")
619 fresh_mpack = _fake_mpack_id("fresh")
620
621 await create_branch(db_session, repo.repo_id, name="main", head_commit_id=tip)
622 job_id = await _insert_job(db_session, repo.repo_id, [tip])
623 await db_session.execute(
624 MusehubBackgroundJob.__table__.update()
625 .where(MusehubBackgroundJob.job_id == job_id)
626 .values(status="running", claimed_at=_now())
627 )
628 await db_session.commit()
629
630 call_results = iter([
631 {"mpack_id": stale_mpack, "mpack_url": "https://r2.example/stale", "commit_count": 1, "blob_count": 1},
632 {"mpack_id": fresh_mpack, "mpack_url": "https://r2.example/fresh", "commit_count": 1, "blob_count": 1},
633 ])
634
635 async def _fake_build(*args, **kwargs):
636 result = next(call_results)
637 if result["mpack_id"] == stale_mpack:
638 # Simulate enqueue_job's dedup-skip path stamping this running
639 # job's superseded_at because a new trigger arrived mid-build.
640 row = (await db_session.execute(
641 select(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == job_id)
642 )).scalar_one()
643 row.superseded_at = _now()
644 await db_session.flush()
645 return result
646
647 with patch(
648 "musehub.services.musehub_wire_fetch.wire_fetch_mpack",
649 new_callable=AsyncMock,
650 side_effect=_fake_build,
651 ) as mock_build:
652 result = await process_fetch_mpack_prebuild_job(db_session, job_id)
653 await db_session.commit()
654
655 assert mock_build.call_count == 2, (
656 "the handler must rebuild once more after detecting it was superseded "
657 "mid-build, instead of finishing with stale coverage"
658 )
659 assert result["tips_built"] == 1
660
661 cache_row = (await db_session.execute(
662 select(MusehubFetchMPackCache)
663 .where(MusehubFetchMPackCache.repo_id == repo.repo_id)
664 .where(MusehubFetchMPackCache.tip_commit_id == tip)
665 )).scalar_one()
666 assert cache_row.mpack_id == fresh_mpack, (
667 "the final cached mpack must be the fresh rebuild, not the stale one "
668 "whose data read predated the superseding write"
669 )
670
671 job_row = (await db_session.execute(
672 select(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == job_id)
673 )).scalar_one()
674 assert job_row.superseded_at is None, (
675 "the flag must be cleared once its rebuild obligation has been honored"
676 )
677
678
679 @pytest.mark.tier2
680 async def test_mwp9_prebuild_does_not_rebuild_when_not_superseded(db_session: AsyncSession) -> None:
681 """GREEN guard: the common case (no concurrent trigger) must still build
682 exactly once -- the supersede-retry loop must not add a spurious rebuild."""
683 repo = await create_repo(db_session, owner="gabriel", visibility="public")
684 tip = _fake_commit_id("tip-mwp9-nosuper")
685 mpack = _fake_mpack_id("single")
686
687 await create_branch(db_session, repo.repo_id, name="main", head_commit_id=tip)
688 job_id = await _insert_job(db_session, repo.repo_id, [tip])
689 await db_session.commit()
690
691 with patch(
692 "musehub.services.musehub_wire_fetch.wire_fetch_mpack",
693 new_callable=AsyncMock,
694 return_value={"mpack_id": mpack, "mpack_url": "https://r2.example/x", "commit_count": 1, "blob_count": 1},
695 ) as mock_build:
696 await process_fetch_mpack_prebuild_job(db_session, job_id)
697 await db_session.commit()
698
699 assert mock_build.call_count == 1
File History 14 commits
sha256:1609a886bd55c9b689a2dda5b8ec9710a27030ff3cb80936f44f7b10a2f0b177 test(prebuild): RED — an unrecoverable missing blob must fa… Sonnet 5.5 4 days ago
sha256:12582616eae9996b8ff470c96ea8ed1b0b0184d90251506e3eb83738c2717b98 test(fetch): prebuild tests patch build_wire_mpack_buffer, … Opus 5.5 4 days ago
sha256:4defb0407ee7fe26b1addb8df5cf3d31dae4ebf1ab92eeca1c05348b9d4c8458 merge: integrate R1 into the publishable #114 line (private… Opus 5.5 minor ⚠ 12 days ago
sha256:83227647a82e1733a20b0b6d225df3d153769630d3c710d255e0c1b8aab8ae61 Co-Authored-By: Claude Opus 5.5 <[email protected]> Opus 5.5 minor ⚠ 13 days ago
sha256:388b0d5926748aed8e7b2941523f5fb2b6991d500eea4c3aac01c5e1e07d2d86 fix: fetch.mpack.prebuild rebuilds when superseded mid-build Sonnet 5 minor ⚠ 26 days ago
sha256:efcebc1682a099c4e426c776ad4f4d0c29f2e8e68fd8b17e170f55d16c6f866f fix(#39): fetch/mpack cache MISS self-heals instead of retr… Sonnet 5 patch 68 days ago
sha256:969ddc5e88776e70af33c016b8777bb721356508ae5f304b3fb46c451494ece7 test(mwp3): Phase 4 — fix adjacent suite regressions, lock … Sonnet 4.6 141 days ago
sha256:068c4d6fa3a2c2e6caf48ed7dd56dff70b848431e3330da7d32f1b7453ea29a9 fix: prebuild upserts all tips to combined mpack_id after build Sonnet 4.6 patch 149 days ago
sha256:97c7eb42fab4fd882f88ce01cf6f18d999560962d504651a86c0a90ef0da6af1 feat: deprecate on-demand mpack build; cache miss → 503 MPa… Sonnet 4.6 minor ⚠ 149 days ago
sha256:dc28fb2384d12a52d4b4fea7743873940b89d9d08ce298f96d0fdc8d694724d4 test+types: green the musehub suite and ratchet typing audi… Opus 4.8 minor ⚠ 153 days ago
sha256:f58d788df3ccdda8f8987b428418db655a38582309239b99d7b9715ea6dff618 feat(#92): phase 5 — GC expired fetch mpack cache entries (… Sonnet 4.6 patch 157 days ago
sha256:d50f9cf9829dfbe35721a23b81ad256c729ddf9dd565a0a9e56d27847e255632 feat(#92): phase 4 — enqueue fetch.mpack.prebuild on push (… Sonnet 4.6 patch 157 days ago
sha256:1c5b7a0aba79472f4b10e52326dc010bdab1a498c9e195593d0707860478a034 feat(#92): phase 3 — cache lookup in wire_fetch_mpack (FMC_… Sonnet 4.6 patch 157 days ago
sha256:0e447fc3f6b7887d5d9e86b557c659ef7d0b05e2e09ddb0cb551ada240e48a51 feat(phase2): fetch.mpack.prebuild job handler + worker dis… Sonnet 4.6 patch 157 days ago