gabriel / muse public
test_mpack_delta_format.py python
261 lines 9.3 KB
Raw
sha256:51ce277f663e01a43eaffbe77509b1de7ac2d4251b55d23306304bcdeb92c90d feat(pack): delta-encode snapshots in MPackBundle wire format Sonnet 4.6 minor ⚠ breaking 131 days ago
1 """TDD — MPackBundle snapshot delta format.
2
3 Guiding principle: content-addressing is a proof, not a label.
4 snapshot_id = sha256(sorted path-NUL-oid pairs)
5
6 If we hold snapshot_id and a delta from the parent manifest, we reconstruct
7 the full manifest and hash it. If the hash matches snapshot_id, the delta
8 is correct. No external store needed. The math IS the verification.
9
10 Tests:
11 1. build_mpack emits SnapshotDeltaDict entries (delta_add/delta_remove),
12 never a full manifest blob per snapshot after the first one.
13 2. Delta chain reconstruction: apply each delta → hash matches snapshot_id.
14 3. Bundle wire size is < 10% of the equivalent full-manifest bundle for a
15 100-commit chain where each commit changes one file.
16 4. apply_mpack round-trips delta bundles: snapshots written to local store
17 have the correct full manifest.
18 """
19 from __future__ import annotations
20
21 import datetime
22 import hashlib
23 import pathlib
24
25 import pytest
26
27 from muse.core.object_store import write_object
28 from muse.core.pack import MPackBundle, apply_mpack, build_mpack
29 from muse.core.paths import muse_dir
30 from muse.core.snapshot import compute_snapshot_id
31 from muse.core.store import (
32 CommitRecord,
33 SnapshotRecord,
34 read_snapshot,
35 write_branch_ref,
36 write_commit,
37 write_snapshot,
38 )
39 from muse.core.types import blob_id
40
41
42 # ---------------------------------------------------------------------------
43 # Helpers
44 # ---------------------------------------------------------------------------
45
46 def _make_repo(tmp: pathlib.Path) -> pathlib.Path:
47 tmp.mkdir(parents=True, exist_ok=True)
48 dot = muse_dir(tmp)
49 dot.mkdir()
50 (dot / "repo.json").write_text('{"repo_id":"delta-test","owner":"gabriel"}')
51 for d in ("commits", "snapshots", "objects"):
52 (dot / d).mkdir()
53 (dot / "refs" / "heads").mkdir(parents=True)
54 (dot / "HEAD").write_text("ref: refs/heads/main\n")
55 (dot / "config.toml").write_text("")
56 return tmp
57
58
59 _N_BASE_FILES = 50
60 _N_COMMITS = 100
61 _BLOB_SIZE = 256
62
63
64 def _make_blob(tag: str) -> tuple[str, bytes]:
65 raw = tag.encode() + b"x" * _BLOB_SIZE
66 return blob_id(raw), raw
67
68
69 def _populate_chain(repo: pathlib.Path) -> tuple[str, list[str]]:
70 """Create _N_BASE_FILES blobs + _N_COMMITS commits, each changing one file.
71
72 Returns (head_commit_id, ordered_snapshot_ids_oldest_first).
73 """
74 base_blobs: dict[str, tuple[str, bytes]] = {}
75 for i in range(_N_BASE_FILES):
76 oid, raw = _make_blob(f"base-{i:04d}")
77 write_object(repo, oid, raw)
78 base_blobs[f"file_{i:04d}.txt"] = (oid, raw)
79
80 base_manifest = {path: oid for path, (oid, _) in base_blobs.items()}
81
82 parent: str | None = None
83 tip = ""
84 ts = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
85 snapshot_ids: list[str] = []
86
87 for i in range(_N_COMMITS):
88 # Each commit changes exactly one file.
89 new_oid, new_raw = _make_blob(f"commit-{i:05d}-variant")
90 write_object(repo, new_oid, new_raw)
91 manifest = dict(base_manifest)
92 manifest[f"file_{i % _N_BASE_FILES:04d}.txt"] = new_oid
93
94 sid = compute_snapshot_id(manifest)
95 write_snapshot(repo, SnapshotRecord(snapshot_id=sid, manifest=manifest))
96 snapshot_ids.append(sid)
97
98 cid = _make_commit_id(parent, sid, f"c{i:05d}", ts.isoformat())
99 rec = CommitRecord(
100 repo_id="delta-test",
101 commit_id=cid,
102 branch="main",
103 snapshot_id=sid,
104 message=f"c{i:05d}",
105 committed_at=ts,
106 parent_commit_id=parent,
107 parent2_commit_id=None,
108 author="gabriel",
109 metadata={},
110 structured_delta=None,
111 sem_ver_bump="none",
112 breaking_changes=[],
113 agent_id="", model_id="", toolchain_id="",
114 prompt_hash="", signature="", signer_key_id="",
115 )
116 write_commit(repo, rec)
117 parent = cid
118 tip = cid
119 ts += datetime.timedelta(seconds=60)
120
121 write_branch_ref(repo, "main", tip)
122 return tip, snapshot_ids
123
124
125 def _make_commit_id(parent: str | None, sid: str, msg: str, ts: str) -> str:
126 from muse.core.snapshot import compute_commit_id
127 return compute_commit_id(
128 parent_ids=[parent] if parent else [],
129 snapshot_id=sid,
130 message=msg,
131 committed_at_iso=ts,
132 author="gabriel",
133 )
134
135
136 def _reconstruct_from_deltas(bundle: MPackBundle) -> dict[str, dict[str, str]]:
137 """Apply the delta chain and return {snapshot_id: full_manifest}."""
138 from muse.core.snapshot import compute_snapshot_id as csi
139 resolved: dict[str, dict[str, str]] = {}
140 for snap in bundle.get("snapshots") or []:
141 sid = snap["snapshot_id"]
142 parent_sid = snap.get("parent_snapshot_id")
143 delta_add: dict[str, str] = snap.get("delta_add") or {}
144 delta_remove: list[str] = snap.get("delta_remove") or []
145
146 base = dict(resolved[parent_sid]) if parent_sid and parent_sid in resolved else {}
147 base.update(delta_add)
148 for path in delta_remove:
149 base.pop(path, None)
150
151 # The hash IS the proof.
152 assert csi(base) == sid, f"hash mismatch for {sid[:16]}"
153 resolved[sid] = base
154 return resolved
155
156
157 # ---------------------------------------------------------------------------
158 # Tests
159 # ---------------------------------------------------------------------------
160
161 def test_bundle_snapshots_are_deltas(tmp_path: pathlib.Path) -> None:
162 """build_mpack emits snapshot deltas, not full manifests."""
163 repo = _make_repo(tmp_path / "repo")
164 head, _ = _populate_chain(repo)
165
166 bundle = build_mpack(repo, [head], have=[])
167
168 snaps = bundle.get("snapshots") or []
169 assert len(snaps) == _N_COMMITS, f"expected {_N_COMMITS} snapshots, got {len(snaps)}"
170
171 for snap in snaps:
172 assert "delta_add" in snap, f"missing delta_add in snapshot {snap.get('snapshot_id', '?')[:16]}"
173 assert "delta_remove" in snap, f"missing delta_remove"
174 assert "manifest" not in snap, "full manifest must not be present — delta format only"
175
176
177 def test_delta_reconstruction_proves_snapshot_id(tmp_path: pathlib.Path) -> None:
178 """Applying each delta and hashing the result must equal snapshot_id."""
179 repo = _make_repo(tmp_path / "repo")
180 head, snapshot_ids = _populate_chain(repo)
181
182 bundle = build_mpack(repo, [head], have=[])
183
184 # Will assert inside _reconstruct_from_deltas if any hash mismatches.
185 resolved = _reconstruct_from_deltas(bundle)
186
187 assert set(resolved.keys()) == set(snapshot_ids), "not all snapshots resolved"
188
189
190 def test_only_first_snapshot_has_full_manifest(tmp_path: pathlib.Path) -> None:
191 """All snapshots after the first should have delta_add < full manifest size."""
192 repo = _make_repo(tmp_path / "repo")
193 head, _ = _populate_chain(repo)
194
195 bundle = build_mpack(repo, [head], have=[])
196 snaps = bundle.get("snapshots") or []
197
198 # First snapshot: delta_add == full manifest (no parent), so len == N_BASE_FILES.
199 assert len(snaps[0].get("delta_add", {})) == _N_BASE_FILES
200
201 # All subsequent snapshots change exactly one file → delta_add has 1 or 2 entries
202 # (1 add + maybe 1 implicit change if same path reverted).
203 for snap in snaps[1:]:
204 n_add = len(snap.get("delta_add", {}))
205 assert n_add < _N_BASE_FILES, (
206 f"snapshot {snap['snapshot_id'][:16]} delta_add has {n_add} entries — "
207 f"should be a small delta, not a full manifest copy"
208 )
209
210
211 def test_delta_bundle_smaller_than_full_manifest(tmp_path: pathlib.Path) -> None:
212 """Delta bundle wire bytes must be < 10% of a hypothetical full-manifest bundle."""
213 import msgpack
214
215 repo = _make_repo(tmp_path / "repo")
216 head, snapshot_ids = _populate_chain(repo)
217
218 delta_bundle = build_mpack(repo, [head], have=[])
219 delta_bytes = len(msgpack.packb(delta_bundle, use_bin_type=True))
220
221 # Build a synthetic "full manifest" bundle for size comparison.
222 full_snap_size = sum(
223 len(msgpack.packb({
224 "snapshot_id": sid,
225 "manifest": (read_snapshot(repo, sid) or SnapshotRecord(snapshot_id=sid, manifest={})).manifest,
226 }, use_bin_type=True))
227 for sid in snapshot_ids
228 )
229 delta_snap_size = sum(
230 len(msgpack.packb(snap, use_bin_type=True))
231 for snap in (delta_bundle.get("snapshots") or [])
232 )
233
234 ratio = delta_snap_size / full_snap_size
235 assert ratio < 0.10, (
236 f"Delta snapshots are {ratio:.1%} of full-manifest size — expected < 10%.\n"
237 f" delta_snap_bytes={delta_snap_size} full_snap_bytes={full_snap_size}"
238 )
239 _ = delta_bytes # measured; useful for manual inspection
240
241
242 def test_apply_mpack_reconstructs_snapshots_from_deltas(tmp_path: pathlib.Path) -> None:
243 """apply_mpack writes correct full SnapshotRecords from delta bundles."""
244 src = _make_repo(tmp_path / "src")
245 head, snapshot_ids = _populate_chain(src)
246
247 bundle = build_mpack(src, [head], have=[])
248
249 dst = _make_repo(tmp_path / "dst")
250 result = apply_mpack(dst, bundle)
251
252 assert result["snapshots_written"] == _N_COMMITS
253
254 # Every snapshot in dst must have the full correct manifest.
255 for sid in snapshot_ids:
256 snap = read_snapshot(dst, sid)
257 assert snap is not None, f"snapshot {sid[:16]} not written to dst"
258 assert compute_snapshot_id(snap.manifest) == sid, (
259 f"manifest hash mismatch for {sid[:16]}: "
260 f"compute_snapshot_id gives {compute_snapshot_id(snap.manifest)[:16]}"
261 )
File History 1 commit
sha256:51ce277f663e01a43eaffbe77509b1de7ac2d4251b55d23306304bcdeb92c90d feat(pack): delta-encode snapshots in MPackBundle wire format Sonnet 4.6 minor ⚠ 131 days ago