gabriel / muse public
verify_pack.py python
556 lines 19.1 KB
Raw
sha256:88ac91129873e6a496e9189515aa690eb893ae25d69c8f72af141a2be5068eb3 docs: docstring sprint contract→find-symbol — idiomatic run… Sonnet 4.6 patch 141 days ago
1 """muse verify-pack — verify the integrity of a MPackBundle.
2
3 Reads a MPackBundle msgpack binary from stdin (or ``--file``) and performs
4 three levels of integrity checking:
5
6 1. **Object integrity** — every ``objects`` entry has its SHA-256 recomputed
7 from the raw ``content`` bytes. The digest must match the declared
8 ``object_id``.
9
10 2. **Snapshot consistency** — every snapshot in the bundle references only
11 object IDs that are either in the bundle itself or already present in the
12 local store. Orphaned manifest entries are reported as failures.
13
14 3. **Commit consistency** — every commit in the bundle references a
15 ``snapshot_id`` that is either in the bundle or already in the local store.
16
17 Pipe from ``pack-objects`` to validate before sending to a remote::
18
19 muse pack-objects <sha> | muse verify-pack
20
21 Or verify a saved bundle file::
22
23 muse verify-pack --file bundle.muse
24
25 Quick structural inspection without full hash verification::
26
27 muse verify-pack --stat --file bundle.muse
28
29 Output (JSON, default)::
30
31 {
32 "objects_checked": 42,
33 "snapshots_checked": 5,
34 "commits_checked": 5,
35 "all_ok": true,
36 "failures": [],
37 "promised_objects": 0,
38 "base_objects": 0,
39 "bundle_mode": "full",
40 "base_commits": [],
41 "duration_ms": 1.234,
42 "exit_code": 0
43 }
44
45 With failures::
46
47 {
48 "objects_checked": 42,
49 "snapshots_checked": 5,
50 "commits_checked": 5,
51 "all_ok": false,
52 "failures": [
53 {"kind": "object", "id": "<sha256>", "error": "hash mismatch"},
54 {"kind": "snapshot", "id": "<sha256>", "error": "missing object: <sha256>"}
55 ],
56 "promised_objects": 3,
57 "base_objects": 0,
58 "bundle_mode": "full",
59 "base_commits": [],
60 "duration_ms": 1.234,
61 "exit_code": 1
62 }
63
64 Stat-only output (``--stat``)::
65
66 {"objects": 42, "snapshots": 5, "commits": 5, "duration_ms": 0.456, "exit_code": 0}
67
68 Object availability model
69 -------------------------
70
71 Objects absent from the bundle are resolved against the local store using a
72 three-state model identical to ``muse verify``:
73
74 - **PRESENT** — object exists in the local ``.muse/objects/`` store (hash
75 verified). Not a failure.
76 - **PROMISED** — absent locally but at least one promisor remote is
77 configured (``.muse/config.toml``). Counted in ``promised_objects``; does
78 not affect ``all_ok``. Use ``--strict`` to treat promised objects as
79 failures.
80 - **MISSING** — absent locally with no promisor remote. Always a failure.
81
82 Output contract
83 ---------------
84
85 - Exit 0: bundle is fully intact; or ``--stat`` completed.
86 - Exit 1: one or more integrity failures; malformed msgpack input; bad format.
87 - Exit 3: I/O error reading stdin or the bundle file.
88
89 Agent use
90 ---------
91
92 Verify before pushing::
93
94 muse pack-objects "$TIP" \\
95 | muse verify-pack --json \\
96 | python3 -c "import sys,json; d=json.load(sys.stdin); sys.exit(0 if d['all_ok'] else 1)"
97
98 Inspect bundle structure without hashing (fast)::
99
100 muse verify-pack --stat --file bundle.muse --json
101
102 Quiet CI gate — fails pipeline if bundle is corrupt::
103
104 muse pack-objects "$TIP" --file bundle.muse
105 muse verify-pack --quiet --file bundle.muse
106 """
107
108 from __future__ import annotations
109
110 import argparse
111 import json
112 import logging
113 import pathlib
114 import sys
115 from typing import TypedDict
116
117 from muse.core._types import blob_id, short_id
118 from muse.core.errors import ExitCode
119 from muse.core.object_availability import ObjectState, load_promisor_remotes, object_state
120 from muse.core.object_store import read_object
121 from muse.core.repo import require_repo
122 from muse.core.store import MAX_PACK_MSGPACK_BYTES, read_snapshot, safe_unpackb
123 from muse.core.validation import sanitize_display, validate_object_id
124 from muse.core.timing import start_timer
125
126 logger = logging.getLogger(__name__)
127
128 _FORMAT_CHOICES = ("json", "text")
129
130
131 class _Failure(TypedDict):
132 kind: str
133 id: str
134 error: str
135
136
137 class _VerifyPackResult(TypedDict):
138 objects_checked: int
139 snapshots_checked: int
140 commits_checked: int
141 all_ok: bool
142 failures: list[_Failure]
143 promised_objects: int
144 base_objects: int
145 bundle_mode: str
146 base_commits: list[str]
147
148
149 class _StatResult(TypedDict):
150 objects: int
151 snapshots: int
152 commits: int
153
154
155 def register(subparsers: "argparse._SubParsersAction[argparse.ArgumentParser]") -> None:
156 """Register the verify-pack subcommand."""
157 parser = subparsers.add_parser(
158 "verify-pack",
159 help="Verify the integrity of a MPackBundle.",
160 description=__doc__,
161 formatter_class=argparse.RawDescriptionHelpFormatter,
162 )
163 parser.add_argument(
164 "--file", "-i",
165 default=None,
166 dest="bundle_file",
167 metavar="PATH",
168 help="Path to a MPackBundle file. Reads from stdin when omitted.",
169 )
170 parser.add_argument(
171 "--stat",
172 action="store_true",
173 dest="stat_only",
174 help=(
175 "Fast structural inspection: count objects, snapshots, and commits "
176 "without computing any hashes. Exits 0 on valid msgpack structure."
177 ),
178 )
179 parser.add_argument(
180 "--quiet", "-q",
181 action="store_true",
182 help="No output. Exit 0 if all checks pass, exit 1 otherwise.",
183 )
184 parser.add_argument(
185 "--no-local", "-L",
186 action="store_true",
187 dest="skip_local_check",
188 help="Skip checking the local store for missing snapshot/commit refs.",
189 )
190 parser.add_argument(
191 "--format", "-f",
192 dest="fmt",
193 default="json",
194 metavar="FORMAT",
195 help="Output format: json or text. (default: json)",
196 )
197 parser.add_argument(
198 "--json", action="store_const", const="json", dest="fmt",
199 help="Shorthand for --format json.",
200 )
201 parser.add_argument(
202 "--strict",
203 action="store_true",
204 help=(
205 "Treat promised objects (absent locally but covered by a promisor remote) "
206 "as integrity failures. By default promised objects are counted in "
207 "``promised_objects`` and do not affect ``all_ok``."
208 ),
209 )
210 parser.set_defaults(func=run)
211
212
213 def run(args: argparse.Namespace) -> None:
214 """Verify the integrity of a MPackBundle.
215
216 Reads a MPackBundle from stdin or ``--file`` and checks:
217
218 - Every object's payload re-hashes to its declared SHA-256 ID.
219 - Every snapshot's manifest references objects present in the bundle,
220 the local store, or a configured promisor remote.
221 - Every commit's snapshot ID is present in the bundle or the local store.
222
223 Objects absent from the bundle are resolved with the three-state model:
224 PRESENT (local store, hash-verified), PROMISED (absent locally but a
225 promisor remote is configured — counted in ``promised_objects``, not a
226 failure unless ``--strict``), or MISSING (absent with no promisor — always
227 a failure).
228
229 Use ``--stat`` for a fast structural count without hash verification.
230 Use ``--strict`` to require full self-containment (treats promised objects
231 as failures).
232 """
233 elapsed = start_timer()
234 fmt: str = args.fmt
235 bundle_file: str | None = args.bundle_file
236 quiet: bool = args.quiet
237 skip_local_check: bool = args.skip_local_check
238 stat_only: bool = args.stat_only
239 strict: bool = args.strict
240
241 if fmt not in _FORMAT_CHOICES:
242 print(
243 json.dumps(
244 {"error": f"Unknown format {fmt!r}. Valid: {', '.join(_FORMAT_CHOICES)}"}
245 ),
246 file=sys.stderr,
247 )
248 raise SystemExit(ExitCode.USER_ERROR)
249
250 # Read bundle bytes.
251 if bundle_file is not None:
252 try:
253 raw_bytes = pathlib.Path(bundle_file).read_bytes()
254 except OSError as exc:
255 print(
256 json.dumps({"error": f"Cannot read file: {sanitize_display(str(exc))}"}),
257 file=sys.stderr,
258 )
259 raise SystemExit(ExitCode.INTERNAL_ERROR)
260 else:
261 try:
262 raw_bytes = sys.stdin.buffer.read()
263 except OSError as exc:
264 print(
265 json.dumps({"error": f"Cannot read stdin: {exc}"}),
266 file=sys.stderr,
267 )
268 raise SystemExit(ExitCode.INTERNAL_ERROR)
269
270 try:
271 bundle = safe_unpackb(
272 raw_bytes,
273 context="pack input",
274 max_bytes=MAX_PACK_MSGPACK_BYTES,
275 allow_binary=True,
276 )
277 except (ValueError, TypeError, Exception) as exc:
278 print(json.dumps({"error": f"Invalid msgpack: {exc}"}), file=sys.stderr)
279 raise SystemExit(ExitCode.USER_ERROR)
280
281 if not isinstance(bundle, dict):
282 print(
283 json.dumps({"error": "MPackBundle must be a msgpack map."}),
284 file=sys.stderr,
285 )
286 raise SystemExit(ExitCode.USER_ERROR)
287
288 # --stat: fast structural count — no hash verification.
289 if stat_only:
290 objects_raw = bundle.get("objects", [])
291 snapshots_raw = bundle.get("snapshots", [])
292 commits_raw = bundle.get("commits", [])
293 stat_result: _StatResult = {
294 "objects": len(objects_raw) if isinstance(objects_raw, list) else 0,
295 "snapshots": len(snapshots_raw) if isinstance(snapshots_raw, list) else 0,
296 "commits": len(commits_raw) if isinstance(commits_raw, list) else 0,
297 }
298 if fmt == "text":
299 print(
300 f"objects={stat_result['objects']} "
301 f"snapshots={stat_result['snapshots']} "
302 f"commits={stat_result['commits']} "
303 f"duration_ms={elapsed()}"
304 )
305 else:
306 print(json.dumps({**stat_result, "duration_ms": elapsed(), "exit_code": 0}))
307 return
308
309 # Every bundle must carry a meta field — it is always written by build_mpack.
310 bundle_meta = bundle.get("meta")
311 if not isinstance(bundle_meta, dict):
312 print(json.dumps({"error": "MPackBundle is missing required 'meta' field."}), file=sys.stderr)
313 raise SystemExit(ExitCode.USER_ERROR)
314 bundle_mode: str = bundle_meta.get("mode", "")
315 if bundle_mode not in ("full", "incremental"):
316 print(json.dumps({"error": f"meta.mode must be 'full' or 'incremental', got {bundle_mode!r}."}), file=sys.stderr)
317 raise SystemExit(ExitCode.USER_ERROR)
318 raw_base = bundle_meta.get("base_commits")
319 if not isinstance(raw_base, list):
320 print(json.dumps({"error": "meta.base_commits must be a list."}), file=sys.stderr)
321 raise SystemExit(ExitCode.USER_ERROR)
322 base_commits: list[str] = [str(c) for c in raw_base]
323 # Incremental bundles have unresolved refs expected at the base — track separately.
324 base_objects_count = 0
325
326 # We need the repo root for local-store checks (optional).
327 root: pathlib.Path | None = require_repo() if not skip_local_check else None
328 # Load promisor remotes once — used in snapshot manifest checks to distinguish
329 # PROMISED objects (absent locally but expected on a remote) from MISSING ones.
330 promisor_remotes: list[str] = load_promisor_remotes(root) if root is not None else []
331 promised_count = 0
332
333 failures: list[_Failure] = []
334
335 # -----------------------------------------------------------------------
336 # 1. Object integrity — re-hash each payload.
337 # -----------------------------------------------------------------------
338 bundle_object_ids: set[str] = set()
339 objects_raw = bundle.get("objects", [])
340 if not isinstance(objects_raw, list):
341 print(
342 json.dumps({"error": "'objects' field must be a list."}),
343 file=sys.stderr,
344 )
345 raise SystemExit(ExitCode.USER_ERROR)
346
347 for entry in objects_raw:
348 if not isinstance(entry, dict):
349 failures.append(
350 _Failure(kind="object", id="(unknown)", error="entry is not a dict")
351 )
352 continue
353 oid = entry.get("object_id", "")
354 content = entry.get("content")
355 if not isinstance(oid, str) or not isinstance(content, (bytes, bytearray)):
356 failures.append(
357 _Failure(
358 kind="object",
359 id="(unknown)",
360 error="missing or invalid object_id / content fields",
361 )
362 )
363 continue
364
365 # Validate object ID format before embedding it anywhere.
366 try:
367 validate_object_id(oid)
368 except ValueError:
369 failures.append(
370 _Failure(
371 kind="object",
372 id="(invalid)",
373 error=f"object_id is not a valid sha256-prefixed hex ID: {oid[:24]!r}",
374 )
375 )
376 continue
377
378 # Hash without copying: msgpack returns bytes; hashlib accepts bytes/bytearray.
379 actual = blob_id(content)
380 if actual != oid:
381 failures.append(
382 _Failure(
383 kind="object",
384 id=oid,
385 error=(
386 f"hash mismatch: declared {short_id(oid)}... "
387 f"recomputed {short_id(actual)}..."
388 ),
389 )
390 )
391 else:
392 bundle_object_ids.add(oid)
393
394 objects_checked = len(objects_raw)
395
396 # -----------------------------------------------------------------------
397 # 2. Snapshot consistency — manifest entries must be present.
398 # -----------------------------------------------------------------------
399 bundle_snapshot_ids: set[str] = set()
400 snapshots_raw = bundle.get("snapshots", [])
401 if not isinstance(snapshots_raw, list):
402 print(
403 json.dumps({"error": "'snapshots' field must be a list."}),
404 file=sys.stderr,
405 )
406 raise SystemExit(ExitCode.USER_ERROR)
407
408 for snap_entry in snapshots_raw:
409 if not isinstance(snap_entry, dict):
410 failures.append(
411 _Failure(
412 kind="snapshot", id="(unknown)", error="snapshot entry is not a dict"
413 )
414 )
415 continue
416 snap_id = snap_entry.get("snapshot_id", "")
417 if not isinstance(snap_id, str):
418 failures.append(
419 _Failure(kind="snapshot", id="(unknown)", error="missing snapshot_id")
420 )
421 continue
422
423 bundle_snapshot_ids.add(snap_id)
424 manifest = snap_entry.get("manifest", {})
425 if not isinstance(manifest, dict):
426 continue
427
428 for path, obj_id in manifest.items():
429 if not isinstance(obj_id, str):
430 continue
431 if obj_id in bundle_object_ids:
432 continue
433 # Object not in bundle.
434 if root is not None:
435 # Local store available — use the PRESENT / PROMISED / MISSING tristate.
436 try:
437 local_content = read_object(root, obj_id)
438 except OSError as exc:
439 failures.append(
440 _Failure(
441 kind="object",
442 id=obj_id,
443 error=(
444 f"local store object {short_id(obj_id)}... failed "
445 f"SHA-256 integrity check: {exc}"
446 ),
447 )
448 )
449 continue
450 if local_content is not None:
451 continue # PRESENT and verified
452
453 state = object_state(root, obj_id, promisor_remotes)
454 if state == ObjectState.PROMISED and not strict:
455 promised_count += 1
456 continue
457 else:
458 # No local store (--no-local). For incremental bundles, unresolved
459 # refs are expected to exist at the declared base — not a failure
460 # unless --strict is set.
461 if bundle_mode == "incremental" and not strict:
462 base_objects_count += 1
463 continue
464
465 failures.append(
466 _Failure(
467 kind="snapshot",
468 id=snap_id,
469 error=(
470 f"manifest path {path!r} references "
471 f"missing object {short_id(obj_id)}..."
472 ),
473 )
474 )
475
476 snapshots_checked = len(snapshots_raw)
477
478 # -----------------------------------------------------------------------
479 # 3. Commit consistency — snapshot_id must be resolvable.
480 # -----------------------------------------------------------------------
481 commits_raw = bundle.get("commits", [])
482 if not isinstance(commits_raw, list):
483 print(
484 json.dumps({"error": "'commits' field must be a list."}),
485 file=sys.stderr,
486 )
487 raise SystemExit(ExitCode.USER_ERROR)
488
489 for commit_entry in commits_raw:
490 if not isinstance(commit_entry, dict):
491 failures.append(
492 _Failure(
493 kind="commit", id="(unknown)", error="commit entry is not a dict"
494 )
495 )
496 continue
497 commit_id = commit_entry.get("commit_id", "")
498 snap_id = commit_entry.get("snapshot_id", "")
499 if not isinstance(commit_id, str) or not isinstance(snap_id, str):
500 failures.append(
501 _Failure(
502 kind="commit",
503 id="(unknown)",
504 error="missing commit_id or snapshot_id",
505 )
506 )
507 continue
508
509 if snap_id in bundle_snapshot_ids:
510 continue
511 if root is not None and read_snapshot(root, snap_id) is not None:
512 continue
513 if not skip_local_check:
514 failures.append(
515 _Failure(
516 kind="commit",
517 id=commit_id,
518 error=f"references snapshot {short_id(snap_id)}... not in bundle or local store",
519 )
520 )
521
522 commits_checked = len(commits_raw)
523 all_ok = len(failures) == 0
524
525 if quiet:
526 raise SystemExit(0 if all_ok else ExitCode.USER_ERROR)
527
528 if fmt == "text":
529 print(
530 f"objects={objects_checked} snapshots={snapshots_checked} "
531 f"commits={commits_checked} all_ok={all_ok}"
532 )
533 for f in failures:
534 print(
535 f" FAIL [{sanitize_display(f['kind'])}] "
536 f"{sanitize_display(short_id(f['id']))}... "
537 f"{sanitize_display(f['error'])}"
538 )
539 if not all_ok:
540 raise SystemExit(ExitCode.USER_ERROR)
541 return
542
543 result: _VerifyPackResult = {
544 "objects_checked": objects_checked,
545 "snapshots_checked": snapshots_checked,
546 "commits_checked": commits_checked,
547 "all_ok": all_ok,
548 "failures": failures,
549 "promised_objects": promised_count,
550 "base_objects": base_objects_count,
551 "bundle_mode": bundle_mode,
552 "base_commits": base_commits,
553 }
554 print(json.dumps({**result, "duration_ms": elapsed(), "exit_code": 0 if all_ok else int(ExitCode.USER_ERROR)}))
555 if not all_ok:
556 raise SystemExit(ExitCode.USER_ERROR)
File History 2 commits
sha256:88ac91129873e6a496e9189515aa690eb893ae25d69c8f72af141a2be5068eb3 docs: docstring sprint contract→find-symbol — idiomatic run… Sonnet 4.6 patch 141 days ago
sha256:a09b1b4f6838754495547f200aa0ce88e2f56ffc5b20b900f6f0cff2c3cdede9 fix(cursorignore): remove git-ism (.git/worktrees) Human minor 144 days ago