gabriel / muse public
test_mpack_e2e.py python
504 lines 18.0 KB
Raw
sha256:51ce277f663e01a43eaffbe77509b1de7ac2d4251b55d23306304bcdeb92c90d feat(pack): delta-encode snapshots in MPackBundle wire format Sonnet 4.6 minor ⚠ breaking 121 days ago
1 """End-to-end integration tests for ``muse push local`` + ``muse pull`` round-trip.
2
3 These tests exercise the full MPack protocol pipeline:
4
5 - push.py pre-compression fix (raw bytes, encoding="raw")
6 - LocalFileTransport.push_stream (direct filesystem write)
7 - apply_mpack writing objects/commits/snapshots/refs to the remote
8 - muse pull fetching commits and objects back from the remote
9
10 All tests use the CliRunner from tests.cli_test_helper and
11 LocalFileTransport (file:// URL) so no HTTP server is required.
12 """
13 from __future__ import annotations
14 from collections.abc import Mapping
15
16 import datetime
17 import json
18 import pathlib
19
20 import pytest
21
22 from muse.core.compression import choose_compression
23 from muse.core.object_store import object_path, write_object
24 from muse.core.snapshot import compute_commit_id, compute_snapshot_id
25 from muse.core.store import CommitRecord, SnapshotRecord, commit_path, write_commit, write_snapshot
26 from tests.cli_test_helper import CliRunner, InvokeResult
27 from muse.core.types import blob_id
28 from muse.core.paths import config_toml_path, heads_dir, muse_dir, ref_path
29
30 runner = CliRunner()
31
32
33 def _parse_json_output(result: InvokeResult) -> Mapping[str, object]:
34 """Extract the JSON object from push/pull output.
35
36 CliRunner combines stdout (JSON) and stderr (progress lines).
37 Find the first line that parses as a JSON object.
38 """
39 for line in result.output.splitlines():
40 line = line.strip()
41 if line.startswith("{"):
42 return json.loads(line)
43 raise ValueError(f"No JSON line found in output:\n{result.output!r}")
44
45
46 # ---------------------------------------------------------------------------
47 # Repo setup helpers
48 # ---------------------------------------------------------------------------
49
50
51 def _make_repo(path: pathlib.Path, *, repo_id: str = "test-repo", domain: str = "code") -> pathlib.Path:
52 """Create a minimal muse repo at *path* and return the root."""
53 muse = muse_dir(path)
54 for sub in ("objects", "commits", "snapshots", "refs/heads"):
55 (muse / sub).mkdir(parents=True)
56 (muse / "HEAD").write_text("ref: refs/heads/main")
57 (muse / "repo.json").write_text(
58 json.dumps({"repo_id": repo_id, "domain": domain, "default_branch": "main"})
59 )
60 return path
61
62
63 def _write_config_toml(repo: pathlib.Path, remotes: Mapping[str, str]) -> None:
64 """Write .muse/config.toml with one [remotes.<name>] section per entry."""
65 lines = ["[remotes]\n"]
66 for name, url in remotes.items():
67 lines.append(f'[remotes.{name}]\n')
68 lines.append(f'url = "{url}"\n')
69 (config_toml_path(repo)).write_text("".join(lines))
70
71
72 def _snap(repo: pathlib.Path, manifest: Mapping[str, str] | None = None) -> str:
73 m = manifest or {}
74 snap_id = compute_snapshot_id(m)
75 write_snapshot(repo, SnapshotRecord(
76 snapshot_id=snap_id,
77 manifest=m,
78 created_at=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc),
79 ))
80 return snap_id
81
82
83 def _commit(
84 repo: pathlib.Path,
85 snap_id: str,
86 *,
87 parent: str | None = None,
88 message: str = "test commit",
89 branch: str = "main",
90 ) -> str:
91 committed_at = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
92 parent_ids: list[str] = [parent] if parent else []
93 commit_id = compute_commit_id(
94 parent_ids=parent_ids,
95 snapshot_id=snap_id,
96 message=message,
97 committed_at_iso=committed_at.isoformat(),
98 )
99 write_commit(repo, CommitRecord(
100 repo_id="test-repo",
101 commit_id=commit_id,
102 branch=branch,
103 snapshot_id=snap_id,
104 message=message,
105 committed_at=committed_at,
106 parent_commit_id=parent,
107 ))
108 return commit_id
109
110
111 def _set_ref(repo: pathlib.Path, branch: str, commit_id: str) -> None:
112 ref_dir = heads_dir(repo)
113 ref_dir.mkdir(parents=True, exist_ok=True)
114 (ref_dir / branch).write_text(commit_id)
115
116
117 def _push(src: pathlib.Path, remote: str = "local", branch: str = "main", *extra: str) -> InvokeResult:
118 from muse.cli.app import main as cli
119 return runner.invoke(
120 cli,
121 ["push", remote, branch, "--json", *extra],
122 env={"MUSE_REPO_ROOT": str(src)},
123 )
124
125
126 def _pull(dst: pathlib.Path, remote: str = "origin", branch: str = "main", *extra: str) -> InvokeResult:
127 from muse.cli.app import main as cli
128 return runner.invoke(
129 cli,
130 ["pull", remote, branch, *extra],
131 env={"MUSE_REPO_ROOT": str(dst)},
132 )
133
134
135 def _object_path(repo: pathlib.Path, oid: str) -> pathlib.Path:
136 """Return the on-disk path for a content-addressed object."""
137 return object_path(repo, oid)
138
139
140
141 def _branch_ref(repo: pathlib.Path, branch: str) -> str | None:
142 ref = ref_path(repo, branch)
143 return ref.read_text().strip() if ref.exists() else None
144
145
146 # ---------------------------------------------------------------------------
147 # Basic push round-trip
148 # ---------------------------------------------------------------------------
149
150
151 class TestPushBasic:
152 def test_push_single_commit(self, tmp_path: pathlib.Path) -> None:
153 """A single commit with no objects is pushed; dst has the commit and ref."""
154 src = _make_repo(tmp_path / "src")
155 dst = _make_repo(tmp_path / "dst")
156 _write_config_toml(src, {"local": dst.as_uri()})
157
158 sid = _snap(src)
159 cid = _commit(src, sid, message="initial")
160 _set_ref(src, "main", cid)
161
162 result = _push(src)
163 assert result.exit_code == 0, result.output
164
165 data = _parse_json_output(result)
166 assert data["status"] == "pushed"
167 assert data["branch"] == "main"
168 assert data["commits_sent"] == 1
169
170 assert commit_path(dst, cid).exists(), "commit file missing from remote"
171 assert _branch_ref(dst, "main") == cid, "remote branch ref not updated"
172
173 def test_push_with_objects(self, tmp_path: pathlib.Path) -> None:
174 """Objects referenced by the commit are transferred to the remote."""
175 src = _make_repo(tmp_path / "src")
176 dst = _make_repo(tmp_path / "dst")
177 _write_config_toml(src, {"local": dst.as_uri()})
178
179 content = b"hello muse e2e push"
180 oid = blob_id(content)
181 write_object(src, oid, content)
182
183 sid = _snap(src, {"hello.txt": oid})
184 cid = _commit(src, sid, message="add hello.txt")
185 _set_ref(src, "main", cid)
186
187 result = _push(src)
188 assert result.exit_code == 0, result.output
189
190 data = _parse_json_output(result)
191 assert data["status"] == "pushed"
192 assert data["objects_sent"] == 1
193
194 assert _object_path(dst, oid).exists(), "object file missing from remote"
195
196 def test_push_reports_compression_type(self, tmp_path: pathlib.Path) -> None:
197 """Objects use choose_compression() — zstd when available, zlib fallback."""
198 src = _make_repo(tmp_path / "src")
199 dst = _make_repo(tmp_path / "dst")
200 _write_config_toml(src, {"local": dst.as_uri()})
201
202 content = b"compression selection test " * 100
203 oid = blob_id(content)
204 write_object(src, oid, content)
205
206 sid = _snap(src, {"data.bin": oid})
207 cid = _commit(src, sid)
208 _set_ref(src, "main", cid)
209
210 result = _push(src)
211 assert result.exit_code == 0, result.output
212
213 # The remote object should exist regardless of which algorithm was chosen.
214 assert _object_path(dst, oid).exists()
215 # Verify the correct algorithm would have been selected (no assertion on
216 # the actual stored bytes — that is an implementation detail of transport).
217 expected_algo = choose_compression()
218 assert expected_algo in ("zstd", "zlib"), f"unexpected algorithm: {expected_algo}"
219
220 def test_push_up_to_date(self, tmp_path: pathlib.Path) -> None:
221 """Second push of the same HEAD reports up_to_date."""
222 src = _make_repo(tmp_path / "src")
223 dst = _make_repo(tmp_path / "dst")
224 _write_config_toml(src, {"local": dst.as_uri()})
225
226 sid = _snap(src)
227 cid = _commit(src, sid)
228 _set_ref(src, "main", cid)
229
230 r1 = _push(src)
231 assert r1.exit_code == 0
232
233 r2 = _push(src)
234 assert r2.exit_code == 0
235 data = _parse_json_output(r2)
236 assert data["status"] == "up_to_date"
237 assert data["commits_sent"] == 0
238
239 def test_push_multi_commit_chain(self, tmp_path: pathlib.Path) -> None:
240 """A chain of commits is pushed in full on first push."""
241 src = _make_repo(tmp_path / "src")
242 dst = _make_repo(tmp_path / "dst")
243 _write_config_toml(src, {"local": dst.as_uri()})
244
245 sid = _snap(src)
246 c1 = _commit(src, sid, message="first")
247 c2 = _commit(src, sid, parent=c1, message="second")
248 c3 = _commit(src, sid, parent=c2, message="third")
249 _set_ref(src, "main", c3)
250
251 result = _push(src)
252 assert result.exit_code == 0, result.output
253
254 data = _parse_json_output(result)
255 assert data["status"] == "pushed"
256 assert data["commits_sent"] == 3
257
258 for cid in (c1, c2, c3):
259 assert commit_path(dst, cid).exists(), f"commit {cid[:8]} missing"
260 assert _branch_ref(dst, "main") == c3
261
262 def test_push_incremental_second_commit(self, tmp_path: pathlib.Path) -> None:
263 """Second push sends only the new commit, not the already-transferred one."""
264 src = _make_repo(tmp_path / "src")
265 dst = _make_repo(tmp_path / "dst")
266 _write_config_toml(src, {"local": dst.as_uri()})
267
268 sid = _snap(src)
269 c1 = _commit(src, sid, message="initial")
270 _set_ref(src, "main", c1)
271
272 r1 = _push(src)
273 assert r1.exit_code == 0
274 assert _parse_json_output(r1)["commits_sent"] == 1
275
276 c2 = _commit(src, sid, parent=c1, message="follow-up")
277 _set_ref(src, "main", c2)
278
279 r2 = _push(src)
280 assert r2.exit_code == 0
281 data2 = _parse_json_output(r2)
282 assert data2["status"] == "pushed"
283 assert data2["commits_sent"] == 1 # only the new commit
284
285 assert _branch_ref(dst, "main") == c2
286
287
288 # ---------------------------------------------------------------------------
289 # Push error conditions
290 # ---------------------------------------------------------------------------
291
292
293 class TestPushErrors:
294 def test_push_no_remote_configured(self, tmp_path: pathlib.Path) -> None:
295 """Push to an unconfigured remote exits with error."""
296 src = _make_repo(tmp_path / "src")
297 sid = _snap(src)
298 cid = _commit(src, sid)
299 _set_ref(src, "main", cid)
300
301 result = _push(src, remote="nonexistent")
302 assert result.exit_code != 0
303 data = _parse_json_output(result)
304 assert data["error"] == "remote_not_configured"
305
306 def test_push_no_commits(self, tmp_path: pathlib.Path) -> None:
307 """Push with no commits on the branch exits with error."""
308 src = _make_repo(tmp_path / "src")
309 dst = _make_repo(tmp_path / "dst")
310 _write_config_toml(src, {"local": dst.as_uri()})
311
312 result = _push(src)
313 assert result.exit_code != 0
314 data = _parse_json_output(result)
315 assert "error" in data
316
317 def test_push_dry_run(self, tmp_path: pathlib.Path) -> None:
318 """--dry-run returns status dry_run without writing to remote."""
319 src = _make_repo(tmp_path / "src")
320 dst = _make_repo(tmp_path / "dst")
321 _write_config_toml(src, {"local": dst.as_uri()})
322
323 sid = _snap(src)
324 cid = _commit(src, sid, message="dry run test")
325 _set_ref(src, "main", cid)
326
327 result = _push(src, "local", "main", "--dry-run")
328 assert result.exit_code == 0, result.output
329 data = _parse_json_output(result)
330 assert data["status"] == "dry_run"
331 assert data["dry_run"] is True
332
333 # Remote must not have been modified.
334 assert _branch_ref(dst, "main") is None
335
336 def test_push_diverged_rejected_without_force(self, tmp_path: pathlib.Path) -> None:
337 """Push is rejected when the remote branch has diverged (non-fast-forward)."""
338 src = _make_repo(tmp_path / "src")
339 dst = _make_repo(tmp_path / "dst")
340 _write_config_toml(src, {"local": dst.as_uri()})
341
342 sid = _snap(src)
343 c1 = _commit(src, sid, message="shared root")
344 _set_ref(src, "main", c1)
345
346 # First push establishes c1 on the remote.
347 r1 = _push(src)
348 assert r1.exit_code == 0
349
350 # Advance the remote independently (simulate another push from elsewhere).
351 c_remote = _commit(dst, sid, parent=c1, message="remote advancement")
352 _set_ref(dst, "main", c_remote)
353
354 # Advance local independently from c1.
355 c_local = _commit(src, sid, parent=c1, message="local advancement")
356 _set_ref(src, "main", c_local)
357
358 result = _push(src)
359 assert result.exit_code != 0
360
361 def test_push_force_overwrites_diverged(self, tmp_path: pathlib.Path) -> None:
362 """--force allows pushing over a diverged remote branch."""
363 src = _make_repo(tmp_path / "src")
364 dst = _make_repo(tmp_path / "dst")
365 _write_config_toml(src, {"local": dst.as_uri()})
366
367 sid = _snap(src)
368 c1 = _commit(src, sid, message="shared root")
369 _set_ref(src, "main", c1)
370 _push(src)
371
372 # Diverge remote and local.
373 c_remote = _commit(dst, sid, parent=c1, message="remote diverge")
374 _set_ref(dst, "main", c_remote)
375
376 c_local = _commit(src, sid, parent=c1, message="local diverge")
377 _set_ref(src, "main", c_local)
378
379 result = _push(src, "local", "main", "--force")
380 assert result.exit_code == 0, result.output
381 data = _parse_json_output(result)
382 assert data["status"] == "pushed"
383 assert _branch_ref(dst, "main") == c_local
384
385
386 # ---------------------------------------------------------------------------
387 # Object content integrity
388 # ---------------------------------------------------------------------------
389
390
391 class TestObjectIntegrity:
392 def test_object_content_survives_push(self, tmp_path: pathlib.Path) -> None:
393 """Object bytes at the remote match what was written to src."""
394 src = _make_repo(tmp_path / "src")
395 dst = _make_repo(tmp_path / "dst")
396 _write_config_toml(src, {"local": dst.as_uri()})
397
398 content = b"binary\x00\x01\x02data" * 512
399 oid = blob_id(content)
400 write_object(src, oid, content)
401
402 sid = _snap(src, {"bin.dat": oid})
403 cid = _commit(src, sid)
404 _set_ref(src, "main", cid)
405
406 result = _push(src)
407 assert result.exit_code == 0, result.output
408
409 from muse.core.object_store import read_object
410 recovered = read_object(dst, oid)
411 assert recovered == content, "object content mismatch after push"
412
413 def test_multiple_objects_all_transferred(self, tmp_path: pathlib.Path) -> None:
414 """All objects in the manifest are present on the remote after push."""
415 src = _make_repo(tmp_path / "src")
416 dst = _make_repo(tmp_path / "dst")
417 _write_config_toml(src, {"local": dst.as_uri()})
418
419 manifest: dict[str, str] = {}
420 for i in range(5):
421 content = f"file content {i}".encode() * 100
422 oid = blob_id(content)
423 write_object(src, oid, content)
424 manifest[f"file{i}.txt"] = oid
425
426 sid = _snap(src, manifest)
427 cid = _commit(src, sid)
428 _set_ref(src, "main", cid)
429
430 result = _push(src)
431 assert result.exit_code == 0, result.output
432
433 data = _parse_json_output(result)
434 assert data["objects_sent"] == 5
435
436 from muse.core.object_store import read_object
437 for oid in manifest.values():
438 assert read_object(dst, oid) is not None, f"object {oid[:16]} missing"
439
440 def test_dedup_objects_not_resent(self, tmp_path: pathlib.Path) -> None:
441 """Objects already on the remote are not re-transferred on subsequent push."""
442 src = _make_repo(tmp_path / "src")
443 dst = _make_repo(tmp_path / "dst")
444 _write_config_toml(src, {"local": dst.as_uri()})
445
446 content = b"shared object content"
447 oid = blob_id(content)
448 write_object(src, oid, content)
449
450 sid = _snap(src, {"shared.txt": oid})
451 c1 = _commit(src, sid, message="first")
452 _set_ref(src, "main", c1)
453
454 r1 = _push(src)
455 assert r1.exit_code == 0
456 assert _parse_json_output(r1)["objects_sent"] == 1
457
458 # Add a new commit with an EMPTY snapshot — no new objects to transfer.
459 sid2 = _snap(src)
460 c2 = _commit(src, sid2, parent=c1, message="second (empty snapshot)")
461 _set_ref(src, "main", c2)
462
463 r2 = _push(src)
464 assert r2.exit_code == 0
465 data2 = _parse_json_output(r2)
466 # The second commit's snapshot has no objects — nothing new to send.
467 assert data2["objects_sent"] == 0
468
469
470 # ---------------------------------------------------------------------------
471 # Pull round-trip (fetch_stream via LocalFileTransport)
472 # ---------------------------------------------------------------------------
473
474
475 class TestPullRoundTrip:
476 def test_pull_fetches_commit_and_objects(self, tmp_path: pathlib.Path) -> None:
477 """After push→pull, a third empty repo has the commit and object."""
478 src = _make_repo(tmp_path / "src")
479 remote = _make_repo(tmp_path / "remote")
480 _write_config_toml(src, {"local": remote.as_uri()})
481
482 content = b"pull round-trip content"
483 oid = blob_id(content)
484 write_object(src, oid, content)
485
486 sid = _snap(src, {"payload.bin": oid})
487 cid = _commit(src, sid, message="push payload")
488 _set_ref(src, "main", cid)
489
490 push_result = _push(src)
491 assert push_result.exit_code == 0, push_result.output
492
493 # Set up a fresh dst that pulls from the same remote.
494 dst = _make_repo(tmp_path / "dst")
495 _write_config_toml(dst, {"origin": remote.as_uri()})
496
497 pull_result = _pull(dst)
498 assert pull_result.exit_code == 0, pull_result.output
499
500 assert commit_path(dst, cid).exists(), "commit missing after pull"
501 assert _branch_ref(dst, "main") == cid
502
503 from muse.core.object_store import read_object
504 assert read_object(dst, oid) == content, "object content mismatch after pull"
File History 1 commit
sha256:51ce277f663e01a43eaffbe77509b1de7ac2d4251b55d23306304bcdeb92c90d feat(pack): delta-encode snapshots in MPackBundle wire format Sonnet 4.6 minor 121 days ago