gabriel / muse public
test_push_object_delta.py python
367 lines 13.6 KB
Raw
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9 docs: expand cache plan with all seven testing tiers and do… Sonnet 4.6 131 days ago
1 """TDD — push only sends objects that are genuinely new.
2
3 Root cause
4 ----------
5 ``walk_commits`` and ``collect_object_ids`` collect ALL objects from the
6 snapshots of new commits without subtracting objects already present in the
7 ``have`` commits' snapshots.
8
9 A snapshot is a full manifest of the repo state at a point in time — it
10 includes every file, not just changed ones. So for a 900-object repo, 1 new
11 commit still sends all 900 objects instead of just the 1–2 that changed.
12
13 The fix: subtract objects reachable from ``have`` commits' snapshots.
14
15 new_objects = objects_in_new_snapshots − objects_in_have_snapshots
16
17 Coverage
18 --------
19 I Unit — collect_object_ids: unchanged objects excluded when have is set
20 II Unit — collect_object_ids: new object (not in have snapshot) is included
21 III Unit — collect_object_ids: object removed in new commit is excluded
22 IV Unit — walk_commits: all_object_ids obeys the same delta semantics
23 V Unit — multi-file repo: 1 changed file → 1 object sent, not all files
24 VI Integration — 10-file repo, 9 unchanged, 1 changed → only 1 object pushed
25 VII Regression — have=[] sends all objects (no regression)
26 VIII Regression — have commit with no local snapshot handled gracefully
27 """
28 from __future__ import annotations
29
30 import datetime
31 import json
32 import pathlib
33
34 import pytest
35
36 from muse._version import __version__
37 from muse.core.object_store import write_object
38 from muse.core.pack import collect_object_ids, walk_commits
39 from muse.core.snapshot import compute_commit_id, compute_snapshot_id
40 from muse.core.store import (
41 CommitRecord,
42 SnapshotRecord,
43 write_commit,
44 write_snapshot,
45 )
46 from muse.core._types import Manifest, blob_id
47
48
49 # ---------------------------------------------------------------------------
50 # Helpers
51 # ---------------------------------------------------------------------------
52
53
54 def _oid(content: bytes) -> str:
55 return blob_id(content)
56
57
58 def _repo(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> pathlib.Path:
59 muse = tmp_path / ".muse"
60 for d in ("commits", "snapshots", "objects", "refs/heads", "remotes"):
61 (muse / d).mkdir(parents=True, exist_ok=True)
62 (muse / "HEAD").write_text("ref: refs/heads/main\n")
63 (muse / "repo.json").write_text(
64 json.dumps({"repo_id": "test-repo", "schema_version": __version__, "domain": "code"})
65 )
66 (muse / "config.toml").write_text('[remotes.origin]\nurl = "https://hub.example.com/r"\n')
67 monkeypatch.setenv("MUSE_REPO_ROOT", str(tmp_path))
68 monkeypatch.chdir(tmp_path)
69 return tmp_path
70
71
72 def _write_commit(
73 root: pathlib.Path,
74 manifest: Manifest,
75 *,
76 parent_id: str | None = None,
77 ) -> CommitRecord:
78 """Write objects, snapshot, and commit; return the CommitRecord."""
79 for oid, raw in [(oid, None) for oid in manifest.values()]:
80 # objects were written by the caller via _write_object
81 pass
82 snap_id = compute_snapshot_id(manifest)
83 write_snapshot(root, SnapshotRecord(snapshot_id=snap_id, manifest=manifest))
84 ts = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
85 parent_ids = [parent_id] if parent_id else []
86 cid = compute_commit_id(
87 repo_id="test-repo",
88 parent_ids=parent_ids,
89 snapshot_id=snap_id,
90 message="test",
91 committed_at_iso=ts.isoformat(),
92 )
93 commit = CommitRecord(
94 commit_id=cid,
95 repo_id="test-repo",
96 created_on_branch="main",
97 snapshot_id=snap_id,
98 message="test",
99 committed_at=ts,
100 parent_commit_id=parent_id,
101 )
102 write_commit(root, commit)
103 return commit
104
105
106 def _write_object(root: pathlib.Path, content: bytes) -> str:
107 oid = _oid(content)
108 write_object(root, oid, content)
109 return oid
110
111
112 # ---------------------------------------------------------------------------
113 # I — unchanged objects are excluded when have is set
114 # ---------------------------------------------------------------------------
115
116
117 class TestCollectObjectIdsDelta:
118 def test_unchanged_object_excluded_when_in_have_snapshot(
119 self, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch
120 ) -> None:
121 """Object present in both have-commit and new-commit snapshot → not sent."""
122 root = _repo(tmp_path, monkeypatch)
123 unchanged = _write_object(root, b"unchanged file")
124
125 # Commit A (the server has this)
126 commit_a = _write_commit(root, {"file.txt": unchanged})
127
128 # Commit B (new — same file, no changes)
129 commit_b = _write_commit(root, {"file.txt": unchanged}, parent_id=commit_a.commit_id)
130
131 result = collect_object_ids(root, [commit_b.commit_id], have=[commit_a.commit_id])
132
133 assert unchanged not in result, (
134 "Object present in have-snapshot must not be re-sent"
135 )
136
137 def test_new_object_included_when_not_in_have_snapshot(
138 self, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch
139 ) -> None:
140 """Object only in new-commit snapshot → must be sent."""
141 root = _repo(tmp_path, monkeypatch)
142 old_file = _write_object(root, b"old file content")
143 new_file = _write_object(root, b"brand new file")
144
145 commit_a = _write_commit(root, {"old.txt": old_file})
146 commit_b = _write_commit(
147 root,
148 {"old.txt": old_file, "new.txt": new_file},
149 parent_id=commit_a.commit_id,
150 )
151
152 result = collect_object_ids(root, [commit_b.commit_id], have=[commit_a.commit_id])
153
154 assert new_file in result, "New object not in have-snapshot must be sent"
155 assert old_file not in result, "Object already in have-snapshot must not be sent"
156
157 def test_removed_object_excluded(
158 self, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch
159 ) -> None:
160 """Object present in have-snapshot but deleted in new commit → not sent."""
161 root = _repo(tmp_path, monkeypatch)
162 kept = _write_object(root, b"kept file")
163 removed = _write_object(root, b"file that gets deleted")
164
165 commit_a = _write_commit(root, {"kept.txt": kept, "gone.txt": removed})
166 # Commit B removes gone.txt
167 commit_b = _write_commit(root, {"kept.txt": kept}, parent_id=commit_a.commit_id)
168
169 result = collect_object_ids(root, [commit_b.commit_id], have=[commit_a.commit_id])
170
171 assert removed not in result
172 assert kept not in result # still unchanged
173
174 def test_empty_delta_when_no_changes(
175 self, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch
176 ) -> None:
177 """Identical snapshot in new commit → zero objects sent."""
178 root = _repo(tmp_path, monkeypatch)
179 obj = _write_object(root, b"content")
180
181 commit_a = _write_commit(root, {"f.txt": obj})
182 # Commit B — identical snapshot (content unchanged)
183 commit_b = _write_commit(root, {"f.txt": obj}, parent_id=commit_a.commit_id)
184
185 result = collect_object_ids(root, [commit_b.commit_id], have=[commit_a.commit_id])
186
187 assert result == [], f"Expected no objects to send, got {result}"
188
189
190 # ---------------------------------------------------------------------------
191 # IV — walk_commits obeys the same delta semantics
192 # ---------------------------------------------------------------------------
193
194
195 class TestWalkCommitsDelta:
196 def test_walk_all_object_ids_excludes_have_objects(
197 self, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch
198 ) -> None:
199 """walk_commits.all_object_ids must subtract have-snapshot objects."""
200 root = _repo(tmp_path, monkeypatch)
201 shared = _write_object(root, b"shared across commits")
202 new_obj = _write_object(root, b"only in new commit")
203
204 commit_a = _write_commit(root, {"shared.txt": shared})
205 commit_b = _write_commit(
206 root,
207 {"shared.txt": shared, "new.txt": new_obj},
208 parent_id=commit_a.commit_id,
209 )
210
211 walk = walk_commits(root, [commit_b.commit_id], have=[commit_a.commit_id])
212
213 assert new_obj in walk["all_object_ids"]
214 assert shared not in walk["all_object_ids"], (
215 "walk_commits must exclude objects already in have-snapshot"
216 )
217
218
219 # ---------------------------------------------------------------------------
220 # V — multi-file repo: only the changed file is sent
221 # ---------------------------------------------------------------------------
222
223
224 class TestMultiFileDelta:
225 def test_only_changed_file_sent_in_10_file_repo(
226 self, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch
227 ) -> None:
228 """10-file repo: 9 unchanged + 1 modified → only 1 object sent."""
229 root = _repo(tmp_path, monkeypatch)
230
231 # Create 10 files in commit A
232 files_a: Manifest = {}
233 for i in range(10):
234 content = f"file {i} original content".encode()
235 oid = _write_object(root, content)
236 files_a[f"file{i:02d}.mid"] = oid
237
238 commit_a = _write_commit(root, files_a)
239
240 # Commit B: modify only file05.mid
241 files_b = dict(files_a)
242 modified_oid = _write_object(root, b"file 5 modified content")
243 files_b["file05.mid"] = modified_oid
244
245 commit_b = _write_commit(root, files_b, parent_id=commit_a.commit_id)
246
247 result = collect_object_ids(root, [commit_b.commit_id], have=[commit_a.commit_id])
248
249 assert result == [modified_oid], (
250 f"Expected only 1 modified object, got {len(result)}: {result}"
251 )
252
253
254 # ---------------------------------------------------------------------------
255 # VI — integration: 1 added file in large repo
256 # ---------------------------------------------------------------------------
257
258
259 class TestLargeRepoDelta:
260 def test_one_added_file_sends_one_object(
261 self, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch
262 ) -> None:
263 """100-file repo, add 1 new file → 1 object sent."""
264 root = _repo(tmp_path, monkeypatch)
265
266 files_a: Manifest = {}
267 for i in range(100):
268 oid = _write_object(root, f"track {i} content".encode())
269 files_a[f"track{i:03d}.mid"] = oid
270
271 commit_a = _write_commit(root, files_a)
272
273 # Add one new file
274 new_oid = _write_object(root, b"brand new track content")
275 files_b = {**files_a, "new_track.mid": new_oid}
276 commit_b = _write_commit(root, files_b, parent_id=commit_a.commit_id)
277
278 result = collect_object_ids(root, [commit_b.commit_id], have=[commit_a.commit_id])
279
280 assert result == [new_oid], (
281 f"Expected exactly 1 new object, got {len(result)}"
282 )
283
284
285 # ---------------------------------------------------------------------------
286 # VII — regression: have=[] sends all objects
287 # ---------------------------------------------------------------------------
288
289
290 class TestNoHaveRegression:
291 def test_no_have_sends_all_objects(
292 self, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch
293 ) -> None:
294 """Without have, all objects in the commit graph are returned."""
295 root = _repo(tmp_path, monkeypatch)
296 obj1 = _write_object(root, b"obj1")
297 obj2 = _write_object(root, b"obj2")
298
299 commit_a = _write_commit(root, {"a.txt": obj1})
300 commit_b = _write_commit(root, {"a.txt": obj1, "b.txt": obj2}, parent_id=commit_a.commit_id)
301
302 result = collect_object_ids(root, [commit_b.commit_id], have=[])
303
304 assert obj1 in result
305 assert obj2 in result
306
307
308 # ---------------------------------------------------------------------------
309 # VIII — graceful handling: have commit has no local snapshot
310 # ---------------------------------------------------------------------------
311
312
313 class TestMissingHaveSnapshot:
314 def test_missing_have_snapshot_treated_as_no_have(
315 self, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch
316 ) -> None:
317 """If a have-commit's snapshot isn't local, don't crash — send the objects."""
318 root = _repo(tmp_path, monkeypatch)
319 obj = _write_object(root, b"some object")
320
321 # Write only a commit record without writing its snapshot locally
322 snap_id = compute_snapshot_id({"f.txt": obj})
323 ts = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
324 fake_have_cid = compute_commit_id(
325 repo_id="test-repo",
326 parent_ids=[],
327 snapshot_id=snap_id,
328 message="phantom",
329 committed_at_iso=ts.isoformat(),
330 )
331 phantom_commit = CommitRecord(
332 commit_id=fake_have_cid,
333 repo_id="test-repo",
334 created_on_branch="main",
335 snapshot_id=snap_id,
336 message="phantom",
337 committed_at=ts,
338 )
339 write_commit(root, phantom_commit)
340 # Note: snapshot is NOT written locally
341
342 new_obj = _write_object(root, b"new object")
343 snap2_id = compute_snapshot_id({"f.txt": obj, "g.txt": new_obj})
344 write_snapshot(root, SnapshotRecord(snapshot_id=snap2_id, manifest={"f.txt": obj, "g.txt": new_obj}))
345 write_object(root, obj, b"some object")
346 cid2 = compute_commit_id(
347 repo_id="test-repo",
348 parent_ids=[fake_have_cid],
349 snapshot_id=snap2_id,
350 message="real",
351 committed_at_iso=ts.isoformat(),
352 )
353 commit2 = CommitRecord(
354 commit_id=cid2,
355 repo_id="test-repo",
356 created_on_branch="main",
357 snapshot_id=snap2_id,
358 message="real",
359 committed_at=ts,
360 parent_commit_id=fake_have_cid,
361 )
362 write_commit(root, commit2)
363
364 # Should not crash; since have-snapshot is missing, objects may be over-sent
365 # but must not raise
366 result = collect_object_ids(root, [cid2], have=[fake_have_cid])
367 assert isinstance(result, list)
File History 1 commit
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9 docs: expand cache plan with all seven testing tiers and do… Sonnet 4.6 131 days ago