gabriel / muse public
test_mpack_e2e.py python
501 lines 18.1 KB
Raw
sha256:88ac91129873e6a496e9189515aa690eb893ae25d69c8f72af141a2be5068eb3 docs: docstring sprint contract→find-symbol — idiomatic run… Sonnet 4.6 patch 139 days ago
1 """End-to-end integration tests for ``muse push local`` + ``muse pull`` round-trip.
2
3 These tests exercise the full MPack protocol pipeline:
4
5 - push.py pre-compression fix (raw bytes, encoding="raw")
6 - LocalFileTransport.push_stream (direct filesystem write)
7 - apply_mpack writing objects/commits/snapshots/refs to the remote
8 - muse pull fetching commits and objects back from the remote
9
10 All tests use the CliRunner from tests.cli_test_helper and
11 LocalFileTransport (file:// URL) so no HTTP server is required.
12 """
13 from __future__ import annotations
14
15 import datetime
16 import hashlib
17 import json
18 import pathlib
19
20 import pytest
21
22 from muse.core.compression import choose_compression
23 from muse.core.object_store import object_path, write_object
24 from muse.core.snapshot import compute_commit_id, compute_snapshot_id
25 from muse.core.store import CommitRecord, SnapshotRecord, _commit_path as _store_commit_path, write_commit, write_snapshot
26 from tests.cli_test_helper import CliRunner, InvokeResult
27 from muse.core._types import long_id
28
29 runner = CliRunner()
30
31
32 def _parse_json_output(result: InvokeResult) -> dict:
33 """Extract the JSON object from push/pull output.
34
35 CliRunner combines stdout (JSON) and stderr (progress lines).
36 Find the first line that parses as a JSON object.
37 """
38 for line in result.output.splitlines():
39 line = line.strip()
40 if line.startswith("{"):
41 return json.loads(line)
42 raise ValueError(f"No JSON line found in output:\n{result.output!r}")
43
44
45 # ---------------------------------------------------------------------------
46 # Repo setup helpers
47 # ---------------------------------------------------------------------------
48
49
50 def _make_repo(path: pathlib.Path, *, repo_id: str = "test-repo", domain: str = "code") -> pathlib.Path:
51 """Create a minimal muse repo at *path* and return the root."""
52 muse = path / ".muse"
53 for sub in ("objects", "commits", "snapshots", "refs/heads"):
54 (muse / sub).mkdir(parents=True)
55 (muse / "HEAD").write_text("ref: refs/heads/main")
56 (muse / "repo.json").write_text(
57 json.dumps({"repo_id": repo_id, "domain": domain, "default_branch": "main"})
58 )
59 return path
60
61
62 def _write_config_toml(repo: pathlib.Path, remotes: dict[str, str]) -> None:
63 """Write .muse/config.toml with one [remotes.<name>] section per entry."""
64 lines = ["[remotes]\n"]
65 for name, url in remotes.items():
66 lines.append(f'[remotes.{name}]\n')
67 lines.append(f'url = "{url}"\n')
68 (repo / ".muse" / "config.toml").write_text("".join(lines))
69
70
71 def _snap(repo: pathlib.Path, manifest: dict[str, str] | None = None) -> str:
72 m = manifest or {}
73 snap_id = compute_snapshot_id(m)
74 write_snapshot(repo, SnapshotRecord(
75 snapshot_id=snap_id,
76 manifest=m,
77 created_at=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc),
78 ))
79 return snap_id
80
81
82 def _commit(
83 repo: pathlib.Path,
84 snap_id: str,
85 *,
86 parent: str | None = None,
87 message: str = "test commit",
88 branch: str = "main",
89 ) -> str:
90 committed_at = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
91 parent_ids: list[str] = [parent] if parent else []
92 commit_id = compute_commit_id(parent_ids, snap_id, message, committed_at.isoformat())
93 write_commit(repo, CommitRecord(
94 commit_id=commit_id,
95 repo_id="test-repo",
96 branch=branch,
97 snapshot_id=snap_id,
98 message=message,
99 committed_at=committed_at,
100 parent_commit_id=parent,
101 ))
102 return commit_id
103
104
105 def _set_ref(repo: pathlib.Path, branch: str, commit_id: str) -> None:
106 ref_dir = repo / ".muse" / "refs" / "heads"
107 ref_dir.mkdir(parents=True, exist_ok=True)
108 (ref_dir / branch).write_text(commit_id)
109
110
111 def _push(src: pathlib.Path, remote: str = "local", branch: str = "main", *extra: str) -> InvokeResult:
112 from muse.cli.app import main as cli
113 return runner.invoke(
114 cli,
115 ["push", remote, branch, "--json", *extra],
116 env={"MUSE_REPO_ROOT": str(src)},
117 )
118
119
120 def _pull(dst: pathlib.Path, remote: str = "origin", branch: str = "main", *extra: str) -> InvokeResult:
121 from muse.cli.app import main as cli
122 return runner.invoke(
123 cli,
124 ["pull", remote, branch, *extra],
125 env={"MUSE_REPO_ROOT": str(dst)},
126 )
127
128
129 def _object_path(repo: pathlib.Path, oid: str) -> pathlib.Path:
130 """Return the on-disk path for a content-addressed object."""
131 return object_path(repo, oid)
132
133
134 def _commit_path(repo: pathlib.Path, commit_id: str) -> pathlib.Path:
135 return _store_commit_path(repo, commit_id)
136
137
138 def _branch_ref(repo: pathlib.Path, branch: str) -> str | None:
139 ref = repo / ".muse" / "refs" / "heads" / branch
140 return ref.read_text().strip() if ref.exists() else None
141
142
143 # ---------------------------------------------------------------------------
144 # Basic push round-trip
145 # ---------------------------------------------------------------------------
146
147
148 class TestPushBasic:
149 def test_push_single_commit(self, tmp_path: pathlib.Path) -> None:
150 """A single commit with no objects is pushed; dst has the commit and ref."""
151 src = _make_repo(tmp_path / "src")
152 dst = _make_repo(tmp_path / "dst")
153 _write_config_toml(src, {"local": dst.as_uri()})
154
155 sid = _snap(src)
156 cid = _commit(src, sid, message="initial")
157 _set_ref(src, "main", cid)
158
159 result = _push(src)
160 assert result.exit_code == 0, result.output
161
162 data = _parse_json_output(result)
163 assert data["status"] == "pushed"
164 assert data["branch"] == "main"
165 assert data["commits_sent"] == 1
166
167 assert _commit_path(dst, cid).exists(), "commit file missing from remote"
168 assert _branch_ref(dst, "main") == cid, "remote branch ref not updated"
169
170 def test_push_with_objects(self, tmp_path: pathlib.Path) -> None:
171 """Objects referenced by the commit are transferred to the remote."""
172 src = _make_repo(tmp_path / "src")
173 dst = _make_repo(tmp_path / "dst")
174 _write_config_toml(src, {"local": dst.as_uri()})
175
176 content = b"hello muse e2e push"
177 oid = long_id(hashlib.sha256(content).hexdigest())
178 write_object(src, oid, content)
179
180 sid = _snap(src, {"hello.txt": oid})
181 cid = _commit(src, sid, message="add hello.txt")
182 _set_ref(src, "main", cid)
183
184 result = _push(src)
185 assert result.exit_code == 0, result.output
186
187 data = _parse_json_output(result)
188 assert data["status"] == "pushed"
189 assert data["objects_sent"] == 1
190
191 assert _object_path(dst, oid).exists(), "object file missing from remote"
192
193 def test_push_reports_compression_type(self, tmp_path: pathlib.Path) -> None:
194 """Objects use choose_compression() — zstd when available, zlib fallback."""
195 src = _make_repo(tmp_path / "src")
196 dst = _make_repo(tmp_path / "dst")
197 _write_config_toml(src, {"local": dst.as_uri()})
198
199 content = b"compression selection test " * 100
200 oid = long_id(hashlib.sha256(content).hexdigest())
201 write_object(src, oid, content)
202
203 sid = _snap(src, {"data.bin": oid})
204 cid = _commit(src, sid)
205 _set_ref(src, "main", cid)
206
207 result = _push(src)
208 assert result.exit_code == 0, result.output
209
210 # The remote object should exist regardless of which algorithm was chosen.
211 assert _object_path(dst, oid).exists()
212 # Verify the correct algorithm would have been selected (no assertion on
213 # the actual stored bytes — that is an implementation detail of transport).
214 expected_algo = choose_compression()
215 assert expected_algo in ("zstd", "zlib"), f"unexpected algorithm: {expected_algo}"
216
217 def test_push_up_to_date(self, tmp_path: pathlib.Path) -> None:
218 """Second push of the same HEAD reports up_to_date."""
219 src = _make_repo(tmp_path / "src")
220 dst = _make_repo(tmp_path / "dst")
221 _write_config_toml(src, {"local": dst.as_uri()})
222
223 sid = _snap(src)
224 cid = _commit(src, sid)
225 _set_ref(src, "main", cid)
226
227 r1 = _push(src)
228 assert r1.exit_code == 0
229
230 r2 = _push(src)
231 assert r2.exit_code == 0
232 data = _parse_json_output(r2)
233 assert data["status"] == "up_to_date"
234 assert data["commits_sent"] == 0
235
236 def test_push_multi_commit_chain(self, tmp_path: pathlib.Path) -> None:
237 """A chain of commits is pushed in full on first push."""
238 src = _make_repo(tmp_path / "src")
239 dst = _make_repo(tmp_path / "dst")
240 _write_config_toml(src, {"local": dst.as_uri()})
241
242 sid = _snap(src)
243 c1 = _commit(src, sid, message="first")
244 c2 = _commit(src, sid, parent=c1, message="second")
245 c3 = _commit(src, sid, parent=c2, message="third")
246 _set_ref(src, "main", c3)
247
248 result = _push(src)
249 assert result.exit_code == 0, result.output
250
251 data = _parse_json_output(result)
252 assert data["status"] == "pushed"
253 assert data["commits_sent"] == 3
254
255 for cid in (c1, c2, c3):
256 assert _commit_path(dst, cid).exists(), f"commit {cid[:8]} missing"
257 assert _branch_ref(dst, "main") == c3
258
259 def test_push_incremental_second_commit(self, tmp_path: pathlib.Path) -> None:
260 """Second push sends only the new commit, not the already-transferred one."""
261 src = _make_repo(tmp_path / "src")
262 dst = _make_repo(tmp_path / "dst")
263 _write_config_toml(src, {"local": dst.as_uri()})
264
265 sid = _snap(src)
266 c1 = _commit(src, sid, message="initial")
267 _set_ref(src, "main", c1)
268
269 r1 = _push(src)
270 assert r1.exit_code == 0
271 assert _parse_json_output(r1)["commits_sent"] == 1
272
273 c2 = _commit(src, sid, parent=c1, message="follow-up")
274 _set_ref(src, "main", c2)
275
276 r2 = _push(src)
277 assert r2.exit_code == 0
278 data2 = _parse_json_output(r2)
279 assert data2["status"] == "pushed"
280 assert data2["commits_sent"] == 1 # only the new commit
281
282 assert _branch_ref(dst, "main") == c2
283
284
285 # ---------------------------------------------------------------------------
286 # Push error conditions
287 # ---------------------------------------------------------------------------
288
289
290 class TestPushErrors:
291 def test_push_no_remote_configured(self, tmp_path: pathlib.Path) -> None:
292 """Push to an unconfigured remote exits with error."""
293 src = _make_repo(tmp_path / "src")
294 sid = _snap(src)
295 cid = _commit(src, sid)
296 _set_ref(src, "main", cid)
297
298 result = _push(src, remote="nonexistent")
299 assert result.exit_code != 0
300 data = _parse_json_output(result)
301 assert data["error"] == "remote_not_configured"
302
303 def test_push_no_commits(self, tmp_path: pathlib.Path) -> None:
304 """Push with no commits on the branch exits with error."""
305 src = _make_repo(tmp_path / "src")
306 dst = _make_repo(tmp_path / "dst")
307 _write_config_toml(src, {"local": dst.as_uri()})
308
309 result = _push(src)
310 assert result.exit_code != 0
311 data = _parse_json_output(result)
312 assert "error" in data
313
314 def test_push_dry_run(self, tmp_path: pathlib.Path) -> None:
315 """--dry-run returns status dry_run without writing to remote."""
316 src = _make_repo(tmp_path / "src")
317 dst = _make_repo(tmp_path / "dst")
318 _write_config_toml(src, {"local": dst.as_uri()})
319
320 sid = _snap(src)
321 cid = _commit(src, sid, message="dry run test")
322 _set_ref(src, "main", cid)
323
324 result = _push(src, "local", "main", "--dry-run")
325 assert result.exit_code == 0, result.output
326 data = _parse_json_output(result)
327 assert data["status"] == "dry_run"
328 assert data["dry_run"] is True
329
330 # Remote must not have been modified.
331 assert _branch_ref(dst, "main") is None
332
333 def test_push_diverged_rejected_without_force(self, tmp_path: pathlib.Path) -> None:
334 """Push is rejected when the remote branch has diverged (non-fast-forward)."""
335 src = _make_repo(tmp_path / "src")
336 dst = _make_repo(tmp_path / "dst")
337 _write_config_toml(src, {"local": dst.as_uri()})
338
339 sid = _snap(src)
340 c1 = _commit(src, sid, message="shared root")
341 _set_ref(src, "main", c1)
342
343 # First push establishes c1 on the remote.
344 r1 = _push(src)
345 assert r1.exit_code == 0
346
347 # Advance the remote independently (simulate another push from elsewhere).
348 c_remote = _commit(dst, sid, parent=c1, message="remote advancement")
349 _set_ref(dst, "main", c_remote)
350
351 # Advance local independently from c1.
352 c_local = _commit(src, sid, parent=c1, message="local advancement")
353 _set_ref(src, "main", c_local)
354
355 result = _push(src)
356 assert result.exit_code != 0
357
358 def test_push_force_overwrites_diverged(self, tmp_path: pathlib.Path) -> None:
359 """--force allows pushing over a diverged remote branch."""
360 src = _make_repo(tmp_path / "src")
361 dst = _make_repo(tmp_path / "dst")
362 _write_config_toml(src, {"local": dst.as_uri()})
363
364 sid = _snap(src)
365 c1 = _commit(src, sid, message="shared root")
366 _set_ref(src, "main", c1)
367 _push(src)
368
369 # Diverge remote and local.
370 c_remote = _commit(dst, sid, parent=c1, message="remote diverge")
371 _set_ref(dst, "main", c_remote)
372
373 c_local = _commit(src, sid, parent=c1, message="local diverge")
374 _set_ref(src, "main", c_local)
375
376 result = _push(src, "local", "main", "--force")
377 assert result.exit_code == 0, result.output
378 data = _parse_json_output(result)
379 assert data["status"] == "pushed"
380 assert _branch_ref(dst, "main") == c_local
381
382
383 # ---------------------------------------------------------------------------
384 # Object content integrity
385 # ---------------------------------------------------------------------------
386
387
388 class TestObjectIntegrity:
389 def test_object_content_survives_push(self, tmp_path: pathlib.Path) -> None:
390 """Object bytes at the remote match what was written to src."""
391 src = _make_repo(tmp_path / "src")
392 dst = _make_repo(tmp_path / "dst")
393 _write_config_toml(src, {"local": dst.as_uri()})
394
395 content = b"binary\x00\x01\x02data" * 512
396 oid = long_id(hashlib.sha256(content).hexdigest())
397 write_object(src, oid, content)
398
399 sid = _snap(src, {"bin.dat": oid})
400 cid = _commit(src, sid)
401 _set_ref(src, "main", cid)
402
403 result = _push(src)
404 assert result.exit_code == 0, result.output
405
406 from muse.core.object_store import read_object
407 recovered = read_object(dst, oid)
408 assert recovered == content, "object content mismatch after push"
409
410 def test_multiple_objects_all_transferred(self, tmp_path: pathlib.Path) -> None:
411 """All objects in the manifest are present on the remote after push."""
412 src = _make_repo(tmp_path / "src")
413 dst = _make_repo(tmp_path / "dst")
414 _write_config_toml(src, {"local": dst.as_uri()})
415
416 manifest: dict[str, str] = {}
417 for i in range(5):
418 content = f"file content {i}".encode() * 100
419 oid = long_id(hashlib.sha256(content).hexdigest())
420 write_object(src, oid, content)
421 manifest[f"file{i}.txt"] = oid
422
423 sid = _snap(src, manifest)
424 cid = _commit(src, sid)
425 _set_ref(src, "main", cid)
426
427 result = _push(src)
428 assert result.exit_code == 0, result.output
429
430 data = _parse_json_output(result)
431 assert data["objects_sent"] == 5
432
433 from muse.core.object_store import read_object
434 for oid in manifest.values():
435 assert read_object(dst, oid) is not None, f"object {oid[:16]} missing"
436
437 def test_dedup_objects_not_resent(self, tmp_path: pathlib.Path) -> None:
438 """Objects already on the remote are not re-transferred on subsequent push."""
439 src = _make_repo(tmp_path / "src")
440 dst = _make_repo(tmp_path / "dst")
441 _write_config_toml(src, {"local": dst.as_uri()})
442
443 content = b"shared object content"
444 oid = long_id(hashlib.sha256(content).hexdigest())
445 write_object(src, oid, content)
446
447 sid = _snap(src, {"shared.txt": oid})
448 c1 = _commit(src, sid, message="first")
449 _set_ref(src, "main", c1)
450
451 r1 = _push(src)
452 assert r1.exit_code == 0
453 assert _parse_json_output(r1)["objects_sent"] == 1
454
455 # Add a new commit with an EMPTY snapshot — no new objects to transfer.
456 sid2 = _snap(src)
457 c2 = _commit(src, sid2, parent=c1, message="second (empty snapshot)")
458 _set_ref(src, "main", c2)
459
460 r2 = _push(src)
461 assert r2.exit_code == 0
462 data2 = _parse_json_output(r2)
463 # The second commit's snapshot has no objects — nothing new to send.
464 assert data2["objects_sent"] == 0
465
466
467 # ---------------------------------------------------------------------------
468 # Pull round-trip (fetch_stream via LocalFileTransport)
469 # ---------------------------------------------------------------------------
470
471
472 class TestPullRoundTrip:
473 def test_pull_fetches_commit_and_objects(self, tmp_path: pathlib.Path) -> None:
474 """After push→pull, a third empty repo has the commit and object."""
475 src = _make_repo(tmp_path / "src")
476 remote = _make_repo(tmp_path / "remote")
477 _write_config_toml(src, {"local": remote.as_uri()})
478
479 content = b"pull round-trip content"
480 oid = long_id(hashlib.sha256(content).hexdigest())
481 write_object(src, oid, content)
482
483 sid = _snap(src, {"payload.bin": oid})
484 cid = _commit(src, sid, message="push payload")
485 _set_ref(src, "main", cid)
486
487 push_result = _push(src)
488 assert push_result.exit_code == 0, push_result.output
489
490 # Set up a fresh dst that pulls from the same remote.
491 dst = _make_repo(tmp_path / "dst")
492 _write_config_toml(dst, {"origin": remote.as_uri()})
493
494 pull_result = _pull(dst)
495 assert pull_result.exit_code == 0, pull_result.output
496
497 assert _commit_path(dst, cid).exists(), "commit missing after pull"
498 assert _branch_ref(dst, "main") == cid
499
500 from muse.core.object_store import read_object
501 assert read_object(dst, oid) == content, "object content mismatch after pull"
File History 1 commit
sha256:88ac91129873e6a496e9189515aa690eb893ae25d69c8f72af141a2be5068eb3 docs: docstring sprint contract→find-symbol — idiomatic run… Sonnet 4.6 patch 139 days ago