gabriel / muse public
test_mpack_e2e.py python
504 lines 18.0 KB
Raw
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9 docs: expand cache plan with all seven testing tiers and do… Sonnet 4.6 131 days ago
1 """End-to-end integration tests for ``muse push local`` + ``muse pull`` round-trip.
2
3 These tests exercise the full MPack protocol pipeline:
4
5 - push.py pre-compression fix (raw bytes, encoding="raw")
6 - LocalFileTransport.push_stream (direct filesystem write)
7 - apply_mpack writing objects/commits/snapshots/refs to the remote
8 - muse pull fetching commits and objects back from the remote
9
10 All tests use the CliRunner from tests.cli_test_helper and
11 LocalFileTransport (file:// URL) so no HTTP server is required.
12 """
13 from __future__ import annotations
14 from collections.abc import Mapping
15
16 import datetime
17 import json
18 import pathlib
19
20 import pytest
21
22 from muse.core.compression import choose_compression
23 from muse.core.object_store import object_path, write_object
24 from muse.core.snapshot import compute_commit_id, compute_snapshot_id
25 from muse.core.store import CommitRecord, SnapshotRecord, commit_path, write_commit, write_snapshot
26 from tests.cli_test_helper import CliRunner, InvokeResult
27 from muse.core._types import blob_id
28
29 runner = CliRunner()
30
31
32 def _parse_json_output(result: InvokeResult) -> Mapping[str, object]:
33 """Extract the JSON object from push/pull output.
34
35 CliRunner combines stdout (JSON) and stderr (progress lines).
36 Find the first line that parses as a JSON object.
37 """
38 for line in result.output.splitlines():
39 line = line.strip()
40 if line.startswith("{"):
41 return json.loads(line)
42 raise ValueError(f"No JSON line found in output:\n{result.output!r}")
43
44
45 # ---------------------------------------------------------------------------
46 # Repo setup helpers
47 # ---------------------------------------------------------------------------
48
49
50 def _make_repo(path: pathlib.Path, *, repo_id: str = "test-repo", domain: str = "code") -> pathlib.Path:
51 """Create a minimal muse repo at *path* and return the root."""
52 muse = path / ".muse"
53 for sub in ("objects", "commits", "snapshots", "refs/heads"):
54 (muse / sub).mkdir(parents=True)
55 (muse / "HEAD").write_text("ref: refs/heads/main")
56 (muse / "repo.json").write_text(
57 json.dumps({"repo_id": repo_id, "domain": domain, "default_branch": "main"})
58 )
59 return path
60
61
62 def _write_config_toml(repo: pathlib.Path, remotes: Mapping[str, str]) -> None:
63 """Write .muse/config.toml with one [remotes.<name>] section per entry."""
64 lines = ["[remotes]\n"]
65 for name, url in remotes.items():
66 lines.append(f'[remotes.{name}]\n')
67 lines.append(f'url = "{url}"\n')
68 (repo / ".muse" / "config.toml").write_text("".join(lines))
69
70
71 def _snap(repo: pathlib.Path, manifest: Mapping[str, str] | None = None) -> str:
72 m = manifest or {}
73 snap_id = compute_snapshot_id(m)
74 write_snapshot(repo, SnapshotRecord(
75 snapshot_id=snap_id,
76 manifest=m,
77 created_at=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc),
78 ))
79 return snap_id
80
81
82 def _commit(
83 repo: pathlib.Path,
84 snap_id: str,
85 *,
86 parent: str | None = None,
87 message: str = "test commit",
88 branch: str = "main",
89 ) -> str:
90 committed_at = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
91 parent_ids: list[str] = [parent] if parent else []
92 commit_id = compute_commit_id(
93 repo_id="test-repo",
94 parent_ids=parent_ids,
95 snapshot_id=snap_id,
96 message=message,
97 committed_at_iso=committed_at.isoformat(),
98 )
99 write_commit(repo, CommitRecord(
100 commit_id=commit_id,
101 repo_id="test-repo",
102 created_on_branch=branch,
103 snapshot_id=snap_id,
104 message=message,
105 committed_at=committed_at,
106 parent_commit_id=parent,
107 ))
108 return commit_id
109
110
111 def _set_ref(repo: pathlib.Path, branch: str, commit_id: str) -> None:
112 ref_dir = repo / ".muse" / "refs" / "heads"
113 ref_dir.mkdir(parents=True, exist_ok=True)
114 (ref_dir / branch).write_text(commit_id)
115
116
117 def _push(src: pathlib.Path, remote: str = "local", branch: str = "main", *extra: str) -> InvokeResult:
118 from muse.cli.app import main as cli
119 return runner.invoke(
120 cli,
121 ["push", remote, branch, "--json", *extra],
122 env={"MUSE_REPO_ROOT": str(src)},
123 )
124
125
126 def _pull(dst: pathlib.Path, remote: str = "origin", branch: str = "main", *extra: str) -> InvokeResult:
127 from muse.cli.app import main as cli
128 return runner.invoke(
129 cli,
130 ["pull", remote, branch, *extra],
131 env={"MUSE_REPO_ROOT": str(dst)},
132 )
133
134
135 def _object_path(repo: pathlib.Path, oid: str) -> pathlib.Path:
136 """Return the on-disk path for a content-addressed object."""
137 return object_path(repo, oid)
138
139
140
141 def _branch_ref(repo: pathlib.Path, branch: str) -> str | None:
142 ref = repo / ".muse" / "refs" / "heads" / branch
143 return ref.read_text().strip() if ref.exists() else None
144
145
146 # ---------------------------------------------------------------------------
147 # Basic push round-trip
148 # ---------------------------------------------------------------------------
149
150
151 class TestPushBasic:
152 def test_push_single_commit(self, tmp_path: pathlib.Path) -> None:
153 """A single commit with no objects is pushed; dst has the commit and ref."""
154 src = _make_repo(tmp_path / "src")
155 dst = _make_repo(tmp_path / "dst")
156 _write_config_toml(src, {"local": dst.as_uri()})
157
158 sid = _snap(src)
159 cid = _commit(src, sid, message="initial")
160 _set_ref(src, "main", cid)
161
162 result = _push(src)
163 assert result.exit_code == 0, result.output
164
165 data = _parse_json_output(result)
166 assert data["status"] == "pushed"
167 assert data["branch"] == "main"
168 assert data["commits_sent"] == 1
169
170 assert commit_path(dst, cid).exists(), "commit file missing from remote"
171 assert _branch_ref(dst, "main") == cid, "remote branch ref not updated"
172
173 def test_push_with_objects(self, tmp_path: pathlib.Path) -> None:
174 """Objects referenced by the commit are transferred to the remote."""
175 src = _make_repo(tmp_path / "src")
176 dst = _make_repo(tmp_path / "dst")
177 _write_config_toml(src, {"local": dst.as_uri()})
178
179 content = b"hello muse e2e push"
180 oid = blob_id(content)
181 write_object(src, oid, content)
182
183 sid = _snap(src, {"hello.txt": oid})
184 cid = _commit(src, sid, message="add hello.txt")
185 _set_ref(src, "main", cid)
186
187 result = _push(src)
188 assert result.exit_code == 0, result.output
189
190 data = _parse_json_output(result)
191 assert data["status"] == "pushed"
192 assert data["objects_sent"] == 1
193
194 assert _object_path(dst, oid).exists(), "object file missing from remote"
195
196 def test_push_reports_compression_type(self, tmp_path: pathlib.Path) -> None:
197 """Objects use choose_compression() — zstd when available, zlib fallback."""
198 src = _make_repo(tmp_path / "src")
199 dst = _make_repo(tmp_path / "dst")
200 _write_config_toml(src, {"local": dst.as_uri()})
201
202 content = b"compression selection test " * 100
203 oid = blob_id(content)
204 write_object(src, oid, content)
205
206 sid = _snap(src, {"data.bin": oid})
207 cid = _commit(src, sid)
208 _set_ref(src, "main", cid)
209
210 result = _push(src)
211 assert result.exit_code == 0, result.output
212
213 # The remote object should exist regardless of which algorithm was chosen.
214 assert _object_path(dst, oid).exists()
215 # Verify the correct algorithm would have been selected (no assertion on
216 # the actual stored bytes — that is an implementation detail of transport).
217 expected_algo = choose_compression()
218 assert expected_algo in ("zstd", "zlib"), f"unexpected algorithm: {expected_algo}"
219
220 def test_push_up_to_date(self, tmp_path: pathlib.Path) -> None:
221 """Second push of the same HEAD reports up_to_date."""
222 src = _make_repo(tmp_path / "src")
223 dst = _make_repo(tmp_path / "dst")
224 _write_config_toml(src, {"local": dst.as_uri()})
225
226 sid = _snap(src)
227 cid = _commit(src, sid)
228 _set_ref(src, "main", cid)
229
230 r1 = _push(src)
231 assert r1.exit_code == 0
232
233 r2 = _push(src)
234 assert r2.exit_code == 0
235 data = _parse_json_output(r2)
236 assert data["status"] == "up_to_date"
237 assert data["commits_sent"] == 0
238
239 def test_push_multi_commit_chain(self, tmp_path: pathlib.Path) -> None:
240 """A chain of commits is pushed in full on first push."""
241 src = _make_repo(tmp_path / "src")
242 dst = _make_repo(tmp_path / "dst")
243 _write_config_toml(src, {"local": dst.as_uri()})
244
245 sid = _snap(src)
246 c1 = _commit(src, sid, message="first")
247 c2 = _commit(src, sid, parent=c1, message="second")
248 c3 = _commit(src, sid, parent=c2, message="third")
249 _set_ref(src, "main", c3)
250
251 result = _push(src)
252 assert result.exit_code == 0, result.output
253
254 data = _parse_json_output(result)
255 assert data["status"] == "pushed"
256 assert data["commits_sent"] == 3
257
258 for cid in (c1, c2, c3):
259 assert commit_path(dst, cid).exists(), f"commit {cid[:8]} missing"
260 assert _branch_ref(dst, "main") == c3
261
262 def test_push_incremental_second_commit(self, tmp_path: pathlib.Path) -> None:
263 """Second push sends only the new commit, not the already-transferred one."""
264 src = _make_repo(tmp_path / "src")
265 dst = _make_repo(tmp_path / "dst")
266 _write_config_toml(src, {"local": dst.as_uri()})
267
268 sid = _snap(src)
269 c1 = _commit(src, sid, message="initial")
270 _set_ref(src, "main", c1)
271
272 r1 = _push(src)
273 assert r1.exit_code == 0
274 assert _parse_json_output(r1)["commits_sent"] == 1
275
276 c2 = _commit(src, sid, parent=c1, message="follow-up")
277 _set_ref(src, "main", c2)
278
279 r2 = _push(src)
280 assert r2.exit_code == 0
281 data2 = _parse_json_output(r2)
282 assert data2["status"] == "pushed"
283 assert data2["commits_sent"] == 1 # only the new commit
284
285 assert _branch_ref(dst, "main") == c2
286
287
288 # ---------------------------------------------------------------------------
289 # Push error conditions
290 # ---------------------------------------------------------------------------
291
292
293 class TestPushErrors:
294 def test_push_no_remote_configured(self, tmp_path: pathlib.Path) -> None:
295 """Push to an unconfigured remote exits with error."""
296 src = _make_repo(tmp_path / "src")
297 sid = _snap(src)
298 cid = _commit(src, sid)
299 _set_ref(src, "main", cid)
300
301 result = _push(src, remote="nonexistent")
302 assert result.exit_code != 0
303 data = _parse_json_output(result)
304 assert data["error"] == "remote_not_configured"
305
306 def test_push_no_commits(self, tmp_path: pathlib.Path) -> None:
307 """Push with no commits on the branch exits with error."""
308 src = _make_repo(tmp_path / "src")
309 dst = _make_repo(tmp_path / "dst")
310 _write_config_toml(src, {"local": dst.as_uri()})
311
312 result = _push(src)
313 assert result.exit_code != 0
314 data = _parse_json_output(result)
315 assert "error" in data
316
317 def test_push_dry_run(self, tmp_path: pathlib.Path) -> None:
318 """--dry-run returns status dry_run without writing to remote."""
319 src = _make_repo(tmp_path / "src")
320 dst = _make_repo(tmp_path / "dst")
321 _write_config_toml(src, {"local": dst.as_uri()})
322
323 sid = _snap(src)
324 cid = _commit(src, sid, message="dry run test")
325 _set_ref(src, "main", cid)
326
327 result = _push(src, "local", "main", "--dry-run")
328 assert result.exit_code == 0, result.output
329 data = _parse_json_output(result)
330 assert data["status"] == "dry_run"
331 assert data["dry_run"] is True
332
333 # Remote must not have been modified.
334 assert _branch_ref(dst, "main") is None
335
336 def test_push_diverged_rejected_without_force(self, tmp_path: pathlib.Path) -> None:
337 """Push is rejected when the remote branch has diverged (non-fast-forward)."""
338 src = _make_repo(tmp_path / "src")
339 dst = _make_repo(tmp_path / "dst")
340 _write_config_toml(src, {"local": dst.as_uri()})
341
342 sid = _snap(src)
343 c1 = _commit(src, sid, message="shared root")
344 _set_ref(src, "main", c1)
345
346 # First push establishes c1 on the remote.
347 r1 = _push(src)
348 assert r1.exit_code == 0
349
350 # Advance the remote independently (simulate another push from elsewhere).
351 c_remote = _commit(dst, sid, parent=c1, message="remote advancement")
352 _set_ref(dst, "main", c_remote)
353
354 # Advance local independently from c1.
355 c_local = _commit(src, sid, parent=c1, message="local advancement")
356 _set_ref(src, "main", c_local)
357
358 result = _push(src)
359 assert result.exit_code != 0
360
361 def test_push_force_overwrites_diverged(self, tmp_path: pathlib.Path) -> None:
362 """--force allows pushing over a diverged remote branch."""
363 src = _make_repo(tmp_path / "src")
364 dst = _make_repo(tmp_path / "dst")
365 _write_config_toml(src, {"local": dst.as_uri()})
366
367 sid = _snap(src)
368 c1 = _commit(src, sid, message="shared root")
369 _set_ref(src, "main", c1)
370 _push(src)
371
372 # Diverge remote and local.
373 c_remote = _commit(dst, sid, parent=c1, message="remote diverge")
374 _set_ref(dst, "main", c_remote)
375
376 c_local = _commit(src, sid, parent=c1, message="local diverge")
377 _set_ref(src, "main", c_local)
378
379 result = _push(src, "local", "main", "--force")
380 assert result.exit_code == 0, result.output
381 data = _parse_json_output(result)
382 assert data["status"] == "pushed"
383 assert _branch_ref(dst, "main") == c_local
384
385
386 # ---------------------------------------------------------------------------
387 # Object content integrity
388 # ---------------------------------------------------------------------------
389
390
391 class TestObjectIntegrity:
392 def test_object_content_survives_push(self, tmp_path: pathlib.Path) -> None:
393 """Object bytes at the remote match what was written to src."""
394 src = _make_repo(tmp_path / "src")
395 dst = _make_repo(tmp_path / "dst")
396 _write_config_toml(src, {"local": dst.as_uri()})
397
398 content = b"binary\x00\x01\x02data" * 512
399 oid = blob_id(content)
400 write_object(src, oid, content)
401
402 sid = _snap(src, {"bin.dat": oid})
403 cid = _commit(src, sid)
404 _set_ref(src, "main", cid)
405
406 result = _push(src)
407 assert result.exit_code == 0, result.output
408
409 from muse.core.object_store import read_object
410 recovered = read_object(dst, oid)
411 assert recovered == content, "object content mismatch after push"
412
413 def test_multiple_objects_all_transferred(self, tmp_path: pathlib.Path) -> None:
414 """All objects in the manifest are present on the remote after push."""
415 src = _make_repo(tmp_path / "src")
416 dst = _make_repo(tmp_path / "dst")
417 _write_config_toml(src, {"local": dst.as_uri()})
418
419 manifest: dict[str, str] = {}
420 for i in range(5):
421 content = f"file content {i}".encode() * 100
422 oid = blob_id(content)
423 write_object(src, oid, content)
424 manifest[f"file{i}.txt"] = oid
425
426 sid = _snap(src, manifest)
427 cid = _commit(src, sid)
428 _set_ref(src, "main", cid)
429
430 result = _push(src)
431 assert result.exit_code == 0, result.output
432
433 data = _parse_json_output(result)
434 assert data["objects_sent"] == 5
435
436 from muse.core.object_store import read_object
437 for oid in manifest.values():
438 assert read_object(dst, oid) is not None, f"object {oid[:16]} missing"
439
440 def test_dedup_objects_not_resent(self, tmp_path: pathlib.Path) -> None:
441 """Objects already on the remote are not re-transferred on subsequent push."""
442 src = _make_repo(tmp_path / "src")
443 dst = _make_repo(tmp_path / "dst")
444 _write_config_toml(src, {"local": dst.as_uri()})
445
446 content = b"shared object content"
447 oid = blob_id(content)
448 write_object(src, oid, content)
449
450 sid = _snap(src, {"shared.txt": oid})
451 c1 = _commit(src, sid, message="first")
452 _set_ref(src, "main", c1)
453
454 r1 = _push(src)
455 assert r1.exit_code == 0
456 assert _parse_json_output(r1)["objects_sent"] == 1
457
458 # Add a new commit with an EMPTY snapshot — no new objects to transfer.
459 sid2 = _snap(src)
460 c2 = _commit(src, sid2, parent=c1, message="second (empty snapshot)")
461 _set_ref(src, "main", c2)
462
463 r2 = _push(src)
464 assert r2.exit_code == 0
465 data2 = _parse_json_output(r2)
466 # The second commit's snapshot has no objects — nothing new to send.
467 assert data2["objects_sent"] == 0
468
469
470 # ---------------------------------------------------------------------------
471 # Pull round-trip (fetch_stream via LocalFileTransport)
472 # ---------------------------------------------------------------------------
473
474
475 class TestPullRoundTrip:
476 def test_pull_fetches_commit_and_objects(self, tmp_path: pathlib.Path) -> None:
477 """After push→pull, a third empty repo has the commit and object."""
478 src = _make_repo(tmp_path / "src")
479 remote = _make_repo(tmp_path / "remote")
480 _write_config_toml(src, {"local": remote.as_uri()})
481
482 content = b"pull round-trip content"
483 oid = blob_id(content)
484 write_object(src, oid, content)
485
486 sid = _snap(src, {"payload.bin": oid})
487 cid = _commit(src, sid, message="push payload")
488 _set_ref(src, "main", cid)
489
490 push_result = _push(src)
491 assert push_result.exit_code == 0, push_result.output
492
493 # Set up a fresh dst that pulls from the same remote.
494 dst = _make_repo(tmp_path / "dst")
495 _write_config_toml(dst, {"origin": remote.as_uri()})
496
497 pull_result = _pull(dst)
498 assert pull_result.exit_code == 0, pull_result.output
499
500 assert commit_path(dst, cid).exists(), "commit missing after pull"
501 assert _branch_ref(dst, "main") == cid
502
503 from muse.core.object_store import read_object
504 assert read_object(dst, oid) == content, "object content mismatch after pull"
File History 2 commits
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9 docs: expand cache plan with all seven testing tiers and do… Sonnet 4.6 131 days ago
sha256:7f9e2ef5286aedad9c1e6011b4c46ca27f39dbdad6e3409357e36b26e46b3b7c docs: docstring sprint for-each-ref→hotspots — idiomatic ru… Sonnet 4.6 patch 138 days ago