gabriel / muse public
fetch.py python
609 lines 21.1 KB
Raw
sha256:ec9f1f4725bce6f66bf08946ea05413572841d9ba43e02df2f3d1cf94e14eb68 feat: fetch.prune config option; muse pull now actually pru… Sonnet 5 minor ⚠ breaking 1 day ago
1 """muse fetch — download commits, snapshots, and objects from a remote.
2
3 Fetches the latest state of a remote branch without touching the local branch
4 HEAD or working tree. After a successful fetch:
5
6 - All new commits, snapshots, and objects from the remote are stored locally.
7 - The remote tracking pointer ``.muse/remotes/<remote>/<branch>`` is updated.
8
9 Use ``muse pull`` to fetch *and* merge into the current branch, or run
10 ``muse merge`` after fetching to integrate on your own schedule.
11
12 Flags
13 -----
14 ``--all``
15 Fetch every configured remote instead of just one. When combined with
16 ``--branch``, that branch is fetched from every remote.
17
18 ``--prune / -p``
19 After fetching, delete local remote-tracking refs (pointers under
20 ``.muse/remotes/<remote>/``) for branches that no longer exist on the
21 remote. Mirrors ``git fetch --prune``.
22
23 ``--dry-run / -n``
24 Show what would be fetched without writing anything.
25
26 ``--tags``
27 Also fetch tags from the remote (default behaviour when tags exist).
28
29 ``--no-tags``
30 Do not fetch tags from the remote.
31
32 ``--format {text,json}`` / ``--json``
33 Emit a machine-readable JSON object on stdout instead of human text.
34 Human-readable diagnostics always go to stderr regardless of format.
35
36 JSON output schema
37 ------------------
38 Always emits a single JSON object on stdout::
39
40 {
41 "results": [
42 {
43 "remote": "<name>",
44 "branch": "<branch>",
45 "status": "fetched | up_to_date | dry_run | branch_missing",
46 "commits_received": <N>,
47 "blobs_written": <N>,
48 "head": "<commit-id> | null",
49 "pruned": ["<remote>/<branch>", ...],
50 "dry_run": false
51 }
52 ],
53 "dry_run": false
54 }
55
56 Exit codes::
57
58 0 — success (fetched, up_to_date, dry_run, or branch_missing + prune)
59 1 — remote not configured, network error, or no remotes when using --all
60 """
61
62 import argparse
63 import json
64 import logging
65 import pathlib
66 import sys
67 import time
68 from collections.abc import Callable
69 from typing import TypedDict
70
71 from muse.cli.config import (
72 get_fetch_prune,
73 get_signing_identity,
74 get_remote,
75 get_remote_head,
76 list_remotes,
77 set_remote_head,
78 )
79 from muse.core.envelope import EnvelopeJson, make_envelope
80 from muse.core.errors import ExitCode
81 from muse.core.mpack import apply_mpack
82 from muse.core.repo import require_repo
83 from muse.core.refs import read_current_branch
84 from muse.core.commits import get_all_commits
85 from muse.core.timing import start_timer
86 from muse.core.transport import TransportError, make_transport
87 from muse.core.validation import sanitize_display
88 from muse.core.types import BranchHeads
89 from muse.core.paths import remote_tracking_dir as _remote_tracking_dir
90
91 logger = logging.getLogger(__name__)
92
93 class _RemoteResultJson(TypedDict):
94 """Per-remote/branch fetch result nested inside :class:`_FetchJson`.
95
96 Fields
97 ------
98 remote Remote name (e.g. ``"origin"``, ``"local"``).
99 branch Branch name fetched from the remote.
100 status Outcome string — one of:
101 ``"fetched"`` (new data received),
102 ``"up_to_date"`` (remote matches local),
103 ``"dry_run"`` (no writes performed),
104 ``"branch_missing"`` (branch not found on remote).
105 commits_received Number of new commit records written to local storage.
106 blobs_written Number of new blobs written.
107 head Remote commit ID that the branch tip points to after the
108 fetch, or ``None`` when nothing was fetched.
109 pruned List of ``"<remote>/<branch>"`` strings for each ref that
110 existed locally but is no longer present on the remote.
111 dry_run True when the fetch was simulated — no blobs were written.
112 """
113
114 remote: str
115 branch: str
116 status: str
117 commits_received: int
118 blobs_written: int
119 head: str | None
120 pruned: list[str]
121 dry_run: bool
122
123 class _FetchJson(EnvelopeJson):
124 """JSON output for ``muse fetch --json``.
125
126 Inherits the 6 standard envelope fields from :class:`~muse.core.envelope.EnvelopeJson`.
127
128 Fields
129 ------
130 results One result entry per remote/branch combination fetched;
131 see :class:`_RemoteResultJson`.
132 dry_run True when no objects were written (``--dry-run`` was passed).
133 """
134
135 results: list[_RemoteResultJson]
136 dry_run: bool
137
138 class _FetchErrorJsonBase(EnvelopeJson):
139 """Required fields for all ``muse fetch`` error JSON outputs."""
140
141 error: str
142 message: str
143
144 class _FetchErrorJson(_FetchErrorJsonBase, total=False):
145 """JSON error output for ``muse fetch --json`` on failure.
146
147 Fields
148 ------
149 error Machine-readable error code (e.g. ``"remote_not_configured"``).
150 message Human-readable description of the failure.
151 remote Remote name involved, when applicable.
152 branch Branch name involved, when applicable.
153 available Sorted list of available branch names (for ``branch_not_found``).
154 hint Suggested remediation command.
155 retryable True when error is a 503 that exhausted the retry budget.
156 """
157
158 remote: str
159 branch: str
160 available: list[str]
161 hint: str
162 retryable: bool
163
164 def register(subparsers: "argparse._SubParsersAction[argparse.ArgumentParser]") -> None:
165 """Register the ``muse fetch`` subcommand and all its flags."""
166 parser = subparsers.add_parser(
167 "fetch",
168 help="Download commits, snapshots, and objects from a remote.",
169 description=__doc__,
170 formatter_class=argparse.RawDescriptionHelpFormatter,
171 )
172 parser.add_argument(
173 "remote",
174 nargs="?",
175 default="origin",
176 help="Remote name to fetch from (default: origin). Ignored when --all is set.",
177 )
178 parser.add_argument(
179 "--branch", "-b",
180 default=None,
181 help=(
182 "Remote branch to fetch (default: current branch). "
183 "When combined with --all, this branch is fetched from every remote."
184 ),
185 )
186 parser.add_argument(
187 "--all",
188 action="store_true",
189 default=False,
190 help="Fetch all configured remotes.",
191 )
192 prune_group = parser.add_mutually_exclusive_group()
193 prune_group.add_argument(
194 "--prune", "-p",
195 action="store_true",
196 default=None,
197 dest="prune",
198 help=(
199 "Remove local remote-tracking refs for branches that no longer exist "
200 "on the remote (mirrors git fetch --prune). Overrides fetch.prune "
201 "config for this invocation."
202 ),
203 )
204 prune_group.add_argument(
205 "--no-prune",
206 action="store_false",
207 dest="prune",
208 help="Disable pruning for this invocation, even if fetch.prune is set to true.",
209 )
210 parser.add_argument(
211 "--dry-run", "-n",
212 action="store_true",
213 default=False,
214 dest="dry_run",
215 help="Show what would be fetched without writing any objects or tracking refs.",
216 )
217 # Tag handling flags — reserved for future use when tag storage is added.
218 tag_group = parser.add_mutually_exclusive_group()
219 tag_group.add_argument(
220 "--tags",
221 action="store_true",
222 default=None,
223 dest="tags",
224 help="Fetch tags from the remote (default).",
225 )
226 tag_group.add_argument(
227 "--no-tags",
228 action="store_false",
229 dest="tags",
230 help="Do not fetch tags from the remote.",
231 )
232 parser.add_argument(
233 "--json", "-j",
234 action="store_true",
235 dest="json_out",
236 help="Emit machine-readable JSON.",
237 )
238 parser.add_argument(
239 "--retry-timeout",
240 type=float,
241 default=None,
242 dest="retry_budget_s",
243 metavar="SECONDS",
244 help=(
245 "Maximum seconds to wait for the remote to finish preparing fetch data "
246 "(503 Retry-After loop). Default: 120 s or MUSE_FETCH_RETRY_BUDGET_S env."
247 ),
248 )
249 parser.add_argument(
250 "--no-retry",
251 action="store_const",
252 const=0.0,
253 dest="retry_budget_s",
254 help=(
255 "Disable the 503 retry loop — fail immediately on the first 503 response. "
256 "Useful for deterministic CI. Overrides --retry-timeout."
257 ),
258 )
259 parser.set_defaults(func=run)
260
261 def _stale_ref_names(
262 root: pathlib.Path,
263 remote: str,
264 live_branch_heads: BranchHeads,
265 ) -> list[str]:
266 """Return branch names whose local tracking refs are absent from *live_branch_heads*.
267
268 Branch names may contain slashes (e.g. ``feat/my-thing``), so the refs are
269 stored as nested files under ``.muse/remotes/<remote>/``. We walk the tree
270 recursively and compute the relative path from ``refs_dir`` to get the full
271 branch name to compare against *live_branch_heads*.
272
273 Symlinks inside the refs directory are skipped to prevent path-traversal
274 attacks via a malicious remote name or branch name.
275 """
276 refs_dir = _remote_tracking_dir(root, remote)
277 if not refs_dir.is_dir():
278 return []
279 stale: list[str] = []
280 for ref_file in refs_dir.rglob("*"):
281 if ref_file.is_symlink() or not ref_file.is_file():
282 continue
283 branch_name = str(ref_file.relative_to(refs_dir))
284 if branch_name not in live_branch_heads:
285 stale.append(branch_name)
286 return stale
287
288 def _prune_stale_refs(
289 root: pathlib.Path,
290 remote: str,
291 live_branch_heads: BranchHeads,
292 *,
293 dry_run: bool,
294 ) -> list[str]:
295 """Prune stale remote-tracking refs, returning the list of pruned branch names.
296
297 Walks ``.muse/remotes/<remote>/`` recursively (branch names may contain
298 slashes and are stored as nested paths) and, unless *dry_run* is True,
299 deletes any file whose relative path is not a key in *live_branch_heads*.
300 Prints a ``- [deleted]`` or ``Would prune`` line for each, mirroring
301 ``git fetch --prune`` output. All output goes to stderr so stdout stays
302 clean for structured JSON.
303 """
304 refs_dir = _remote_tracking_dir(root, remote)
305 pruned: list[str] = []
306 for branch_name in sorted(_stale_ref_names(root, remote, live_branch_heads)):
307 safe_ref = f"{sanitize_display(remote)}/{sanitize_display(branch_name)}"
308 if dry_run:
309 print(f" Would prune {safe_ref}", file=sys.stderr)
310 else:
311 ref_file = refs_dir / branch_name
312 ref_file.unlink()
313 # Remove empty parent directories left behind (e.g. feat/ after feat/my-thing).
314 for parent in ref_file.parents:
315 if parent == refs_dir:
316 break
317 try:
318 parent.rmdir()
319 except OSError:
320 break
321 logger.debug("🗑 Pruned stale tracking ref %s/%s", remote, branch_name)
322 print(f" - [deleted] {safe_ref}", file=sys.stderr)
323 pruned.append(f"{remote}/{branch_name}")
324 return pruned
325
326 def _fetch_one(
327 root: pathlib.Path,
328 remote: str,
329 branch: str,
330 *,
331 prune: bool,
332 dry_run: bool,
333 json_out: bool = False,
334 elapsed: Callable[[], float] = lambda: 0.0,
335 retry_budget_s: "float | None" = None,
336 ) -> _RemoteResultJson:
337 """Fetch a single remote/branch pair.
338
339 Returns a :class:`_RemoteResultJson` describing the outcome. Raises
340 ``SystemExit`` on unrecoverable errors (unknown remote, network failure).
341 Writes nothing when *dry_run* is True.
342
343 Full local history is sent as the ``have`` list so the server can compute
344 the minimal delta.
345 """
346 result: _RemoteResultJson = {
347 "remote": remote,
348 "branch": branch,
349 "status": "fetched",
350 "commits_received": 0,
351 "blobs_written": 0,
352 "head": None,
353 "pruned": [],
354 "dry_run": dry_run,
355 }
356
357 url = get_remote(remote, root)
358 if url is None:
359 if json_out:
360 print(json.dumps(_FetchErrorJson(
361 **make_envelope(elapsed, exit_code=ExitCode.USER_ERROR),
362 error="remote_not_configured",
363 remote=remote,
364 message=f"remote '{remote}' is not configured",
365 hint=f"muse remote add {remote} <url>",
366 )))
367 print(
368 f"❌ Remote '{sanitize_display(remote)}' is not configured.",
369 file=sys.stderr,
370 )
371 print(
372 f" Add it with: muse remote add {sanitize_display(remote)} <url>",
373 file=sys.stderr,
374 )
375 raise SystemExit(ExitCode.USER_ERROR)
376
377 token = get_signing_identity(root, remote_url=url)
378 transport = make_transport(url)
379
380 try:
381 info = transport.fetch_remote_info(url, token)
382 except TransportError as exc:
383 if json_out:
384 print(json.dumps(_FetchErrorJson(
385 **make_envelope(elapsed, exit_code=ExitCode.INTERNAL_ERROR),
386 error="remote_unreachable",
387 remote=remote,
388 message=str(exc),
389 )))
390 print(
391 f"❌ Cannot reach remote '{sanitize_display(remote)}': "
392 f"{sanitize_display(str(exc))}",
393 file=sys.stderr,
394 )
395 raise SystemExit(ExitCode.INTERNAL_ERROR)
396
397 remote_commit_id = info["branch_heads"].get(branch)
398 if remote_commit_id is None:
399 if prune:
400 # The branch we were tracking is gone from the remote.
401 # Prune it (and any other stale refs) then return cleanly —
402 # this mirrors `git fetch --prune` behaviour where a deleted
403 # upstream branch produces " - [deleted] remote/branch" rather
404 # than an error.
405 pruned = _prune_stale_refs(root, remote, info["branch_heads"], dry_run=dry_run)
406 result["status"] = "branch_missing"
407 result["pruned"] = pruned
408 return result
409 available = sorted(info["branch_heads"])
410 if json_out:
411 print(json.dumps(_FetchErrorJson(
412 **make_envelope(elapsed, exit_code=ExitCode.USER_ERROR),
413 error="branch_not_found",
414 remote=remote,
415 branch=branch,
416 available=available,
417 message=f"branch '{branch}' does not exist on remote '{remote}'",
418 )))
419 print(
420 f"❌ Branch '{sanitize_display(branch)}' does not exist on "
421 f"remote '{sanitize_display(remote)}'.",
422 file=sys.stderr,
423 )
424 print(f" Available branches: {', '.join(sanitize_display(b) for b in available)}", file=sys.stderr)
425 raise SystemExit(ExitCode.USER_ERROR)
426
427 already_known = get_remote_head(remote, branch, root)
428 if already_known == remote_commit_id:
429 print(
430 f"✅ {sanitize_display(remote)}/{sanitize_display(branch)} "
431 f"is already up to date ({remote_commit_id})",
432 file=sys.stderr,
433 )
434 if prune and not dry_run:
435 pruned = _prune_stale_refs(root, remote, info["branch_heads"], dry_run=False)
436 result["pruned"] = pruned
437 result["status"] = "up_to_date"
438 result["head"] = remote_commit_id
439 return result
440
441 if dry_run:
442 print(
443 f" Would fetch {sanitize_display(remote)}/{sanitize_display(branch)} "
444 f"→ {remote_commit_id}",
445 file=sys.stderr,
446 )
447 if prune:
448 pruned = _prune_stale_refs(root, remote, info["branch_heads"], dry_run=True)
449 result["pruned"] = pruned
450 result["status"] = "dry_run"
451 result["head"] = remote_commit_id
452 return result
453
454 have_for_fetch = [c.commit_id for c in get_all_commits(root)]
455
456 print(f"Fetching {sanitize_display(remote)}/{sanitize_display(branch)} …", file=sys.stderr)
457
458 t0_fetch = time.perf_counter()
459 try:
460 fetch_result = transport.fetch_mpack(
461 url, token,
462 want=[remote_commit_id],
463 have=have_for_fetch,
464 retry_budget_s=retry_budget_s,
465 )
466 except TransportError as exc:
467 _is_503 = exc.status_code == 503
468 if _is_503:
469 from muse.core.transport import _resolve_retry_budget
470 _budget = _resolve_retry_budget(retry_budget_s)
471 _msg = f"Remote still preparing fetch data after {int(_budget)}s — try again shortly."
472 else:
473 _msg = str(exc)
474 if json_out:
475 print(json.dumps(_FetchErrorJson(
476 **make_envelope(elapsed, exit_code=ExitCode.INTERNAL_ERROR),
477 error="fetch_failed",
478 remote=remote,
479 branch=branch,
480 message=_msg,
481 retryable=_is_503,
482 )))
483 print(f"❌ {sanitize_display(_msg)}", file=sys.stderr)
484 raise SystemExit(ExitCode.INTERNAL_ERROR)
485 t_fetch = time.perf_counter() - t0_fetch
486
487 apply_result = apply_mpack(root, {
488 "commits": fetch_result["commits"],
489 "snapshots": fetch_result["snapshots"],
490 "blobs": fetch_result["blobs"],
491 })
492
493 if not apply_result["failed_blobs"]:
494 set_remote_head(remote, branch, remote_commit_id, root)
495
496 commits_received: int = apply_result["commits_written"]
497 blobs_written: int = apply_result["blobs_written"]
498 print(
499 f"[mpack] fetch/mpack: {t_fetch:.2f}s "
500 f"blobs: {fetch_result.get('blobs_received', 0)} "
501 f"commits: {commits_received}",
502 file=sys.stderr,
503 )
504 print(
505 f"✅ Fetched {commits_received} commit(s), "
506 f"{blobs_written} new blob(s) "
507 f"from {sanitize_display(remote)}/{sanitize_display(branch)} "
508 f"({remote_commit_id})",
509 file=sys.stderr,
510 )
511
512 if prune:
513 pruned = _prune_stale_refs(root, remote, info["branch_heads"], dry_run=False)
514 result["pruned"] = pruned
515
516 result["commits_received"] = commits_received
517 result["blobs_written"] = blobs_written
518 result["head"] = remote_commit_id
519 return result
520
521 def run(args: argparse.Namespace) -> None:
522 """Download commits, snapshots, and objects from a remote.
523
524 Updates remote tracking pointers but does NOT change local HEAD or the
525 working tree. Run ``muse pull`` to fetch and merge in one step.
526 ``--all`` fetches every configured remote; ``--prune`` deletes stale
527 remote-tracking refs after a successful fetch.
528
529 Agent quickstart
530 ----------------
531 ::
532
533 muse fetch local --json
534 muse fetch local main --json
535 muse fetch --all --json
536 muse fetch local --prune --json
537
538 JSON fields
539 -----------
540 results List of per-remote result objects: ``remote``, ``branch``,
541 ``new_commits``, ``objects_fetched``, ``ok``, ``error``.
542 dry_run ``true`` if ``--dry-run`` was passed (no writes occurred).
543
544 Exit codes
545 ----------
546 0 Fetch complete (all remotes succeeded).
547 1 One or more remotes failed, no remotes configured, or bad arguments.
548 2 Not inside a Muse repository.
549 """
550 elapsed = start_timer()
551 root = require_repo()
552 current_branch = read_current_branch(root)
553 dry_run: bool = args.dry_run
554 # --prune/--no-prune (explicit) override fetch.prune config; when neither
555 # flag is passed args.prune is None and the config value decides (muse#22).
556 prune: bool = args.prune if args.prune is not None else get_fetch_prune(root)
557 retry_budget_s: float | None = getattr(args, "retry_budget_s", None)
558 json_out: bool = args.json_out
559
560 if dry_run:
561 print("(dry run — no objects or refs will be written)", file=sys.stderr)
562
563 results: list[_RemoteResultJson] = []
564
565 if args.all:
566 remotes = list_remotes(root)
567 if not remotes:
568 if json_out:
569 print(json.dumps(_FetchErrorJson(
570 **make_envelope(elapsed, exit_code=ExitCode.USER_ERROR),
571 error="no_remotes",
572 message="no remotes configured",
573 hint="muse remote add <name> <url>",
574 )))
575 print("❌ No remotes configured.", file=sys.stderr)
576 print(" Add one with: muse remote add <name> <url>", file=sys.stderr)
577 raise SystemExit(ExitCode.USER_ERROR)
578 # --branch with --all: fetch the specified branch from every remote.
579 branch: str = args.branch or current_branch
580 for remote_cfg in remotes:
581 result = _fetch_one(
582 root,
583 remote_cfg["name"],
584 branch,
585 prune=prune,
586 dry_run=dry_run,
587 json_out=json_out,
588 elapsed=elapsed,
589 retry_budget_s=retry_budget_s,
590 )
591 results.append(result)
592 else:
593 remote: str = args.remote
594 # Use the explicitly passed branch, or fall back to the current local branch.
595 # Do NOT use get_upstream() here — that returns the remote *name*, not branch.
596 branch_single: str = args.branch or current_branch
597 result = _fetch_one(
598 root, remote, branch_single,
599 prune=prune, dry_run=dry_run, json_out=json_out, elapsed=elapsed,
600 retry_budget_s=retry_budget_s,
601 )
602 results.append(result)
603
604 if json_out:
605 print(json.dumps(_FetchJson(
606 **make_envelope(elapsed),
607 results=results,
608 dry_run=dry_run,
609 )))
File History 1 commit
sha256:ec9f1f4725bce6f66bf08946ea05413572841d9ba43e02df2f3d1cf94e14eb68 feat: fetch.prune config option; muse pull now actually pru… Sonnet 5 minor ⚠ 1 day ago