gabriel / musehub public
test_wire_push_external_parent_reconstruction.py python
307 lines 12.0 KB Cold
Raw
sha256:383b9b15b1a7c18348bf797dbec252b8842199567b4b04371578fedd5f7891d4 merge: integrate push-ingest commit verification into dev Opus 5.5 minor ⚠ breaking 11 days ago
1 """TDD — musehub#134: wire_push_unpack_mpack must not persist an empty base when
2 a pushed snapshot's parent is an EXTERNAL delta-only (or missing) snapshot.
3
4 Root cause, in ``musehub_wire_push.py``:
5
6 _psnap_rows = (await session.execute(
7 select(MusehubSnapshot.snapshot_id, MusehubSnapshot.manifest_blob)
8 .where(MusehubSnapshot.snapshot_id.in_(_external_parent_sids))
9 )).all()
10 for _psid, _pblob in _psnap_rows:
11 if _pblob:
12 _parent_snap_manifests[_psid] = dict(_msgpack.unpackb(_pblob, raw=False))
13 ...
14 _parent_base = _parent_snap_manifests.get(_parent_sid) or {}
15
16 Only checks ``manifest_blob`` directly. A delta-only external parent
17 (``manifest_blob=NULL``, reconstructible via ``delta_blob`` + ``parent_snapshot_id``)
18 is silently dropped, and the child's delta is applied on top of ``{}`` — a manifest
19 that is missing every inherited file and can never hash back to its own snapshot_id.
20
21 This test calls ``wire_push_unpack_mpack`` DIRECTLY with a hand-built wire mpack that
22 contains ONLY the new commit (its parent snapshot is genuinely absent from the
23 payload) — this is the only reliable way to force the external-parent path. A
24 CLI-level ``muse push`` integration test (see ``test_push_delta_only_parent_manifest.py``)
25 cannot force it: ``muse push``'s ``have`` negotiation is a literal-branch-tip match
26 with no ancestor-closure expansion (``muse/core/mpack.py::walk_commits`` —
27 ``prune=lambda cid: cid in have_set``), so a real second push naturally resends the
28 whole chain back to the nearest branch tip, making the "external" parent an in-payload
29 sibling instead — which is exactly why that existing test passes without exercising
30 this bug.
31
32 RED before the fix (D's stored manifest drops f1/f2, inherited from delta-only B).
33 GREEN after (parent reconstructed from its delta chain before applying D's delta).
34 """
35 from __future__ import annotations
36
37 import datetime
38 from unittest.mock import AsyncMock, MagicMock, patch
39
40 import msgpack
41 import pytest
42 from sqlalchemy.ext.asyncio import AsyncSession
43
44 from muse.core.ids import hash_commit, hash_snapshot
45 from muse.core.mpack import build_wire_mpack
46 from muse.core.types import blob_id
47 from musehub.core.genesis import compute_identity_id
48 from musehub.db.musehub_repo_models import (
49 MusehubCommit,
50 MusehubCommitGraph,
51 MusehubSnapshot,
52 MusehubSnapshotRef,
53 )
54 from musehub.services.musehub_repository import create_repo
55 from musehub.services.musehub_wire_push import wire_push_unpack_mpack
56
57 _OWNER = "gabriel"
58 _IDENTITY_ID = compute_identity_id(b"gabriel")
59
60
61 def _cid(seed: str) -> str:
62 """An id for a row seeded straight into the database (never pushed)."""
63 return blob_id(f"ext-parent-commit-{seed}".encode())
64
65
66 _SEEDS: dict[str, str] = {}
67
68
69 def _pushed_cid(seed: str, parent: str | None, snap_id: str) -> str:
70 """The id of the commit _raw_commit builds for *seed*; push ingest verifies it."""
71 cid = hash_commit(parent_ids=[parent] if parent else [], snapshot_id=snap_id, message=f"commit {seed}",
72 committed_at_iso=_now().isoformat(), author=_OWNER, signer_public_key="")
73 _SEEDS[cid] = seed
74 return cid
75
76
77 def _now() -> datetime.datetime:
78 return datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
79
80
81 def _raw_commit(cid: str, parent: str | None, snap_id: str, *, branch: str = "feat") -> dict:
82 return {
83 "commit_id": cid,
84 "branch": branch,
85 "message": f"commit {_SEEDS.get(cid, cid[:12])}",
86 "author": _OWNER,
87 "committed_at": _now().isoformat(),
88 "parent_commit_id": parent,
89 "parent2_commit_id": None,
90 "snapshot_id": snap_id,
91 "agent_id": "",
92 "model_id": "",
93 "toolchain_id": "",
94 "sem_ver_bump": "none",
95 "breaking_changes": [],
96 "signature": "",
97 "signer_key_id": "",
98 "signer_public_key": "",
99 "prompt_hash": "",
100 }
101
102
103 def _mock_backend(mpack_bytes: bytes) -> MagicMock:
104 backend = MagicMock()
105 backend.get_mpack = AsyncMock(return_value=mpack_bytes)
106 backend.put = AsyncMock(return_value=None)
107 backend.put_mpack = AsyncMock(return_value=None)
108 backend.quarantine_mpack = AsyncMock(return_value=None)
109 backend.presign_get = AsyncMock(return_value="")
110 backend.presign_mpack_get = AsyncMock(return_value="")
111 return backend
112
113
114 @pytest.mark.asyncio
115 async def test_child_of_external_delta_only_parent_reconstructs_full_manifest(
116 db_session: AsyncSession,
117 ) -> None:
118 """RED: D's parent B is delta-only and NOT included in D's push payload.
119
120 Seeded directly (simulating an earlier, separate push that already landed
121 A as root/full-manifest and B as a middle/delta-only snapshot — exactly
122 what a real ``muse push`` of A->B->C produces):
123
124 A: root, full manifest {f1.txt}
125 B: delta-only, manifest_blob=NULL, delta_blob={+f2.txt}, parent=A
126
127 Then push ONLY commit D (parent=B, delta={+f4.txt}) — B is genuinely
128 external: absent from this push's own commits/snapshots list entirely.
129 """
130 repo = await create_repo(
131 db_session,
132 name="ext-parent-repro",
133 owner=_OWNER,
134 owner_user_id=_IDENTITY_ID,
135 visibility="public",
136 initialize=False,
137 )
138 await db_session.commit()
139
140 oid1 = blob_id(b"content-f1")
141 oid2 = blob_id(b"content-f2")
142 oid4 = blob_id(b"content-f4")
143
144 manifest_a = {"f1.txt": oid1}
145 manifest_b = {"f1.txt": oid1, "f2.txt": oid2}
146 snap_a = hash_snapshot(manifest_a)
147 snap_b = hash_snapshot(manifest_b)
148
149 # Seed A: root, full manifest (exactly what a real push stores for a batch root).
150 db_session.add(MusehubSnapshot(
151 snapshot_id=snap_a,
152 directories=[],
153 manifest_blob=msgpack.packb(manifest_a, use_bin_type=True),
154 entry_count=len(manifest_a),
155 created_at=_now(),
156 parent_snapshot_id=None,
157 delta_blob=None,
158 ))
159 # Seed B: delta-only middle snapshot — manifest_blob=NULL, exactly what a real
160 # push stores for a non-root/non-head snapshot.
161 db_session.add(MusehubSnapshot(
162 snapshot_id=snap_b,
163 directories=[],
164 manifest_blob=None,
165 entry_count=len(manifest_b),
166 created_at=_now(),
167 parent_snapshot_id=snap_a,
168 delta_blob=msgpack.packb({"add": {"f2.txt": oid2}}, use_bin_type=True),
169 ))
170 db_session.add(MusehubSnapshotRef(repo_id=repo.repo_id, snapshot_id=snap_a))
171 db_session.add(MusehubSnapshotRef(repo_id=repo.repo_id, snapshot_id=snap_b))
172
173 # Seed A/B's musehub_commits rows (authoritative DAG) + A's commit_graph row
174 # (anchor, generation 0) — exactly what an earlier real push of A->B already
175 # landed on the server before this test's push of D.
176 a_commit = _cid("a")
177 b_commit = _cid("b")
178 db_session.add(MusehubCommit(
179 commit_id=a_commit, branch="feat", message="A", author=_OWNER,
180 timestamp=_now(), parent_ids=[], snapshot_id=snap_a,
181 ))
182 db_session.add(MusehubCommit(
183 commit_id=b_commit, branch="feat", message="B", author=_OWNER,
184 timestamp=_now(), parent_ids=[a_commit], snapshot_id=snap_b,
185 ))
186 db_session.add(MusehubCommitGraph(
187 commit_id=a_commit, parent_ids=[], generation=0, snapshot_id=snap_a, created_at=_now(),
188 ))
189 await db_session.commit()
190
191 manifest_d = {"f1.txt": oid1, "f2.txt": oid2, "f4.txt": oid4}
192 snap_d = hash_snapshot(manifest_d)
193 d_commit = _pushed_cid("d", b_commit, snap_d)
194
195 # The push payload for D ONLY — B is genuinely absent, exactly as a real
196 # incremental push would send once `have` correctly recognizes B is already
197 # on the remote (see module docstring for why muse push's CLI negotiation
198 # can't reliably force this in an integration test).
199 mpack_bytes = build_wire_mpack({
200 "commits": [_raw_commit(d_commit, b_commit, snap_d)],
201 "snapshots": [{
202 "snapshot_id": snap_d,
203 "parent_snapshot_id": snap_b,
204 "delta_upsert": {"f4.txt": oid4},
205 "delta_remove": [],
206 "directories": [],
207 }],
208 "blobs": [{"object_id": oid4, "content": b"content-f4"}],
209 "tags": [],
210 })
211 mpack_key = blob_id(mpack_bytes)
212 backend = _mock_backend(mpack_bytes)
213
214 with patch("musehub.services.musehub_wire.get_backend", return_value=backend), \
215 patch("musehub.services.musehub_wire_push.get_backend", return_value=backend), \
216 patch("musehub.storage.backends.get_backend", return_value=backend):
217 await wire_push_unpack_mpack(
218 db_session,
219 repo.repo_id,
220 mpack_key,
221 pusher_id=_OWNER,
222 branch="feat",
223 head_commit_id=d_commit,
224 commits_count=1,
225 blobs_count=1,
226 force=True,
227 )
228 await db_session.flush()
229
230 stored = await db_session.get(MusehubSnapshot, snap_d)
231 assert stored is not None, "D's snapshot row was not written at all"
232 assert stored.manifest_blob is not None, "D is the head of its own push batch — must get a full manifest_blob"
233
234 stored_manifest = dict(msgpack.unpackb(stored.manifest_blob, raw=False))
235 must_have = {"f1.txt", "f2.txt", "f4.txt"}
236 assert must_have <= set(stored_manifest), (
237 f"D's stored manifest dropped files inherited from the external delta-only "
238 f"parent B (base resolved to empty instead of being reconstructed).\n"
239 f" stored = {sorted(stored_manifest)}\n"
240 f" must_have = {sorted(must_have)}\n"
241 f" missing = {sorted(must_have - set(stored_manifest))}"
242 )
243 assert hash_snapshot(stored_manifest, list(stored.directories or [])) == snap_d, (
244 f"stored manifest does not reproduce snapshot_id {snap_d[:18]} — "
245 f"the exact corruption reproduced live on staging.musehub.ai/gabriel/musehub"
246 )
247
248
249 @pytest.mark.asyncio
250 async def test_push_rejected_when_external_parent_is_a_total_phantom(
251 db_session: AsyncSession,
252 ) -> None:
253 """RED: D's parent snapshot has NO row at all — not delta-only, genuinely absent
254 (the exact state found on staging: a snapshot_id referenced as a parent with no
255 backing musehub_snapshots row and no musehub_commits row either). Reconstruction
256 cannot recover any content. The push must be REJECTED, not silently persisted
257 with a manifest that will never hash-verify again.
258 """
259 repo = await create_repo(
260 db_session,
261 name="phantom-parent-repro",
262 owner=_OWNER,
263 owner_user_id=_IDENTITY_ID,
264 visibility="public",
265 initialize=False,
266 )
267 await db_session.commit()
268
269 oid4 = blob_id(b"content-f4-phantom")
270 phantom_snap_id = hash_snapshot({"never.txt": blob_id(b"never-existed")})
271 manifest_d = {"never.txt": blob_id(b"never-existed"), "f4.txt": oid4}
272 snap_d = hash_snapshot(manifest_d)
273
274 d_commit = _pushed_cid("phantom-d", _cid("phantom-parent-commit"), snap_d)
275 mpack_bytes = build_wire_mpack({
276 "commits": [_raw_commit(d_commit, _cid("phantom-parent-commit"), snap_d)],
277 "snapshots": [{
278 "snapshot_id": snap_d,
279 "parent_snapshot_id": phantom_snap_id,
280 "delta_upsert": {"f4.txt": oid4},
281 "delta_remove": [],
282 "directories": [],
283 }],
284 "blobs": [{"object_id": oid4, "content": b"content-f4-phantom"}],
285 "tags": [],
286 })
287 mpack_key = blob_id(mpack_bytes)
288 backend = _mock_backend(mpack_bytes)
289
290 with patch("musehub.services.musehub_wire.get_backend", return_value=backend), \
291 patch("musehub.services.musehub_wire_push.get_backend", return_value=backend), \
292 patch("musehub.storage.backends.get_backend", return_value=backend):
293 with pytest.raises(ValueError, match="cannot be reconstructed"):
294 await wire_push_unpack_mpack(
295 db_session,
296 repo.repo_id,
297 mpack_key,
298 pusher_id=_OWNER,
299 branch="feat",
300 head_commit_id=d_commit,
301 commits_count=1,
302 blobs_count=1,
303 force=True,
304 )
305
306 stored = await db_session.get(MusehubSnapshot, snap_d)
307 assert stored is None, "a snapshot that can never be verified must never be persisted"
File History 4 commits
sha256:383b9b15b1a7c18348bf797dbec252b8842199567b4b04371578fedd5f7891d4 merge: integrate push-ingest commit verification into dev Opus 5.5 minor ⚠ 11 days ago
sha256:39b90cebf3ab4c72ef67a29fe9d451ba1394084edf4053dff98878651f5d2378 test: build reproducible commit ids in push fixtures Opus 5.5 11 days ago
sha256:281d7e2bfa6d4378354b0287c78e3bf1902581c486d072d3352c9d645ba81f63 fix(musehub#134): reconstruct external delta-only parent in… Sonnet 5 patch 132 days ago
sha256:b916bfaf72c56574764215fb36e245869f2cf36c7fab7e38afbc195d8d105a54 test(red): musehub#134 — push must reconstruct external del… Sonnet 5 132 days ago