gabriel / muse public
git2muse.py python
576 lines 18.3 KB
Raw
sha256:1c4b3e3a9a1f300774c3ee662b572a698d5fd405bf765a71e6011a2e9c3eaaaa feat: Muse — version control for the agent era Human 151 days ago
1 """git2muse — Replay a Git commit graph into a Muse repository.
2
3 Usage
4 -----
5 ::
6
7 python tools/git2muse.py [--repo-root PATH] [--dry-run] [--verbose]
8 [--branch BRANCH [BRANCH ...]]
9
10 Strategy
11 --------
12 1. Walk requested branches oldest-first and create Muse commits preserving the
13 original author, timestamp, and message.
14 2. For multi-branch runs, ``main`` is always replayed first so that other
15 branches can branch from the correct Muse ancestor.
16 3. Skip merge commits (commits with more than one parent) — they carry no
17 unique file-state delta; the Muse DAG is reconstructed faithfully through
18 the parent chain on each branch.
19
20 For each Git commit the tool:
21 - Extracts the commit's file tree into a temp dir using ``git archive``.
22 - Removes files that Muse should not snapshot (build artefacts, caches, IDE
23 files, etc.) according to a hard-coded exclusion list that mirrors
24 ``.museignore``.
25 - Calls the Muse Python API directly (bypassing the CLI) so the original
26 Git author name, e-mail, and committer timestamp are preserved verbatim in
27 the Muse ``CommitRecord``.
28 - Updates the Muse branch HEAD ref so the Muse repo tracks the same history.
29
30 After a successful run the Muse repo under ``.muse/`` contains a full code-
31 domain representation of the project history and is ready to push to MuseHub.
32 """
33
34 from __future__ import annotations
35
36 import argparse
37 import datetime
38 import logging
39 import pathlib
40 import shutil
41 import subprocess
42 import sys
43 import tarfile
44 import tempfile
45
46 # ---------------------------------------------------------------------------
47 # Bootstrap: make sure the project root is on sys.path so we can import muse
48 # even when running from the tools/ directory.
49 # ---------------------------------------------------------------------------
50 _REPO_ROOT = pathlib.Path(__file__).parent.parent
51 if str(_REPO_ROOT) not in sys.path:
52 sys.path.insert(0, str(_REPO_ROOT))
53
54 from muse.core.object_store import write_object
55 from muse.core.reflog import append_reflog
56 from muse.core.store import (
57 CommitRecord,
58 SnapshotRecord,
59 get_head_commit_id,
60 write_commit,
61 write_head_branch,
62 write_snapshot,
63 )
64 from muse.core.snapshot import (
65 compute_commit_id,
66 compute_snapshot_id,
67 walk_workdir_with_dirs,
68 )
69
70 logger = logging.getLogger("git2muse")
71
72 # ---------------------------------------------------------------------------
73 # Files / dirs that should never end up in a Muse snapshot.
74 # These mirror .museignore + the hidden-directory exclusion in walk_workdir.
75 # ---------------------------------------------------------------------------
76
77 _EXCLUDE_PREFIXES: tuple[str, ...] = (
78 ".git/",
79 ".muse/",
80 ".muse",
81 ".venv/",
82 ".tox/",
83 ".mypy_cache/",
84 ".pytest_cache/",
85 ".hypothesis/",
86 "artifacts/",
87 "__pycache__/",
88 )
89
90 _EXCLUDE_SUFFIXES: tuple[str, ...] = (
91 ".pyc",
92 ".pyo",
93 ".egg-info",
94 ".swp",
95 ".swo",
96 ".tmp",
97 "Thumbs.db",
98 ".DS_Store",
99 )
100
101
102 def _should_exclude(rel_path: str) -> bool:
103 """Return True if *rel_path* should be excluded from the Muse snapshot."""
104 for prefix in _EXCLUDE_PREFIXES:
105 if rel_path.startswith(prefix) or rel_path == prefix.rstrip("/"):
106 return True
107 for suffix in _EXCLUDE_SUFFIXES:
108 if rel_path.endswith(suffix):
109 return True
110 return False
111
112
113 # ---------------------------------------------------------------------------
114 # Git helpers
115 # ---------------------------------------------------------------------------
116
117
118 def _git(repo_root: pathlib.Path, *args: str) -> str:
119 """Run a git command and return stdout (stripped)."""
120 result = subprocess.run(
121 ["git", *args],
122 cwd=repo_root,
123 capture_output=True,
124 text=True,
125 check=True,
126 )
127 return result.stdout.strip()
128
129
130 def _git_commits_oldest_first(
131 repo_root: pathlib.Path,
132 branch: str,
133 exclude_branches: list[str] | None = None,
134 ) -> list[str]:
135 """Return SHA1 hashes oldest-first for *branch*.
136
137 When *exclude_branches* is given, commits reachable from any of those
138 branches are excluded (used to extract branch-unique commits).
139 """
140 cmd = ["log", "--topo-order", "--reverse", "--format=%H"]
141 if exclude_branches:
142 cmd.append(branch)
143 for excl in exclude_branches:
144 cmd.append(f"^{excl}")
145 else:
146 cmd.append(branch)
147 raw = _git(repo_root, *cmd)
148 return [line for line in raw.splitlines() if line.strip()]
149
150
151 _META_SEP = "|||GIT2MUSE|||"
152
153
154 def _git_commit_meta(repo_root: pathlib.Path, sha: str) -> dict[str, str]:
155 """Return author name, email, timestamp, and message for *sha*."""
156 fmt = f"%an{_META_SEP}%ae{_META_SEP}%at{_META_SEP}%B"
157 raw = _git(repo_root, "show", "-s", f"--format={fmt}", sha)
158 parts = raw.split(_META_SEP, 3)
159 if len(parts) < 4:
160 return {"name": "unknown", "email": "", "ts": "0", "message": sha[:12]}
161 name, email, ts, message = parts
162 return {
163 "name": name.strip(),
164 "email": email.strip(),
165 "ts": ts.strip(),
166 "message": message.strip(),
167 }
168
169
170 def _git_parent_shas(repo_root: pathlib.Path, sha: str) -> list[str]:
171 """Return parent SHA1s for *sha* (empty list for root commits)."""
172 raw = _git(repo_root, "log", "-1", "--format=%P", sha)
173 return [p for p in raw.split() if p]
174
175
176 def _is_merge_commit(repo_root: pathlib.Path, sha: str) -> bool:
177 return len(_git_parent_shas(repo_root, sha)) > 1
178
179
180 def _git_local_branches(repo_root: pathlib.Path) -> list[str]:
181 """Return all local branch names."""
182 raw = _git(repo_root, "branch", "--format=%(refname:short)")
183 return [b.strip() for b in raw.splitlines() if b.strip()]
184
185
186 def _extract_tree_to(
187 repo_root: pathlib.Path,
188 sha: str,
189 dest: pathlib.Path,
190 ) -> None:
191 """Extract the git tree for *sha* into *dest*, applying exclusions."""
192 if dest.exists():
193 shutil.rmtree(dest)
194 dest.mkdir(parents=True)
195
196 archive = subprocess.run(
197 ["git", "archive", "--format=tar", sha],
198 cwd=repo_root,
199 capture_output=True,
200 check=True,
201 )
202 with tempfile.NamedTemporaryFile(suffix=".tar", delete=False) as tmp:
203 tmp.write(archive.stdout)
204 tmp_path = pathlib.Path(tmp.name)
205
206 try:
207 with tarfile.open(tmp_path) as tf:
208 for member in tf.getmembers():
209 if not member.isfile():
210 continue
211 rel = member.name.removeprefix("./")
212 if _should_exclude(rel):
213 continue
214 target = dest / rel
215 target.parent.mkdir(parents=True, exist_ok=True)
216 f = tf.extractfile(member)
217 if f is not None:
218 target.write_bytes(f.read())
219 finally:
220 tmp_path.unlink(missing_ok=True)
221
222
223 # ---------------------------------------------------------------------------
224 # Muse snapshot helpers (bypass CLI to preserve git metadata)
225 # ---------------------------------------------------------------------------
226
227
228 def _build_manifest_with_dirs(
229 workdir: pathlib.Path,
230 ) -> tuple[dict[str, str], list[str]]:
231 """Walk *workdir* and return ``(manifest, directories)``.
232
233 Uses :func:`muse.core.snapshot.walk_workdir_with_dirs` — the same
234 canonical walker that ``muse commit`` uses — so that the resulting
235 snapshot ID is byte-for-byte identical to what ``muse commit`` would
236 produce on the same tree. Passing both manifest and directories to
237 :func:`compute_snapshot_id` is required; omitting directories produces a
238 different hash (the old pre-directory-tracking format) that will not match
239 any snapshot created by the CLI.
240 """
241 return walk_workdir_with_dirs(workdir)
242
243
244 def _store_objects(
245 repo_root: pathlib.Path,
246 workdir: pathlib.Path,
247 manifest: dict[str, str],
248 ) -> None:
249 """Write all objects referenced in *manifest* to the object store."""
250 for rel, oid in manifest.items():
251 fpath = workdir / rel
252 if not fpath.exists():
253 logger.warning("⚠️ Missing file in workdir: %s", rel)
254 continue
255 content = fpath.read_bytes()
256 write_object(repo_root, oid, content)
257
258
259 # ---------------------------------------------------------------------------
260 # Branch ref helpers (direct file I/O — mirrors store.py internal logic)
261 # ---------------------------------------------------------------------------
262
263
264 def _refs_dir(repo_root: pathlib.Path) -> pathlib.Path:
265 return repo_root / ".muse" / "refs" / "heads"
266
267
268 def _set_branch_head(
269 repo_root: pathlib.Path, branch: str, commit_id: str
270 ) -> None:
271 ref_path = _refs_dir(repo_root) / branch
272 ref_path.parent.mkdir(parents=True, exist_ok=True)
273 ref_path.write_text(commit_id + "\n")
274
275
276 def _get_branch_head(repo_root: pathlib.Path, branch: str) -> str | None:
277 ref_path = _refs_dir(repo_root) / branch
278 if not ref_path.exists():
279 return None
280 return ref_path.read_text().strip() or None
281
282
283 def _set_head_ref(repo_root: pathlib.Path, branch: str) -> None:
284 write_head_branch(repo_root, branch)
285
286
287 def _ensure_branch_exists(repo_root: pathlib.Path, branch: str) -> None:
288 _refs_dir(repo_root).mkdir(parents=True, exist_ok=True)
289 ref_path = _refs_dir(repo_root) / branch
290 if not ref_path.exists():
291 ref_path.write_text("")
292
293
294 # ---------------------------------------------------------------------------
295 # Core replay logic
296 # ---------------------------------------------------------------------------
297
298
299 def _replay_commit(
300 repo_root: pathlib.Path,
301 workdir: pathlib.Path,
302 git_sha: str,
303 muse_branch: str,
304 parent_muse_id: str | None,
305 meta: dict[str, str],
306 repo_id: str,
307 dry_run: bool,
308 ) -> str:
309 """Replay one Git commit into the Muse object store.
310
311 Returns the new Muse commit ID.
312 """
313 manifest, directories = _build_manifest_with_dirs(workdir)
314
315 # compute_snapshot_id requires directories for hash parity with muse commit.
316 snapshot_id = compute_snapshot_id(manifest, directories=directories)
317
318 committed_at = datetime.datetime.fromtimestamp(
319 int(meta["ts"]), tz=datetime.timezone.utc
320 )
321 author = f"{meta['name']} <{meta['email']}>"
322 message = meta["message"] or git_sha[:12]
323
324 committed_at_iso = committed_at.isoformat()
325 parent_ids = [parent_muse_id] if parent_muse_id else []
326 commit_id = compute_commit_id(
327 parent_ids=parent_ids,
328 snapshot_id=snapshot_id,
329 message=message,
330 committed_at_iso=committed_at_iso,
331 )
332
333 if dry_run:
334 logger.info(
335 "[dry-run] Would create commit %s (git: %s) on %s | %s",
336 commit_id[:12],
337 git_sha[:12],
338 muse_branch,
339 message[:60],
340 )
341 return commit_id
342
343 _store_objects(repo_root, workdir, manifest)
344
345 snap = SnapshotRecord(snapshot_id=snapshot_id, manifest=manifest)
346 write_snapshot(repo_root, snap)
347
348 record = CommitRecord(
349 commit_id=commit_id,
350 repo_id=repo_id,
351 branch=muse_branch,
352 snapshot_id=snapshot_id,
353 message=message,
354 committed_at=committed_at,
355 parent_commit_id=parent_muse_id,
356 author=author,
357 )
358 write_commit(repo_root, record)
359
360 _set_branch_head(repo_root, muse_branch, commit_id)
361 append_reflog(
362 repo_root,
363 muse_branch,
364 old_id=parent_muse_id,
365 new_id=commit_id,
366 author=author,
367 operation=f"git2muse: {message[:60]}",
368 )
369
370 return commit_id
371
372
373 def _replay_branch(
374 repo_root: pathlib.Path,
375 workdir: pathlib.Path,
376 git_shas: list[str],
377 muse_branch: str,
378 start_parent_muse_id: str | None,
379 repo_id: str,
380 dry_run: bool,
381 verbose: bool,
382 ) -> dict[str, str]:
383 """Replay a list of git SHAs (oldest first) onto *muse_branch*.
384
385 Returns a mapping of git_sha → muse_commit_id for every replayed commit.
386 """
387 _ensure_branch_exists(repo_root, muse_branch)
388
389 git_to_muse: dict[str, str] = {}
390 parent_muse_id = start_parent_muse_id
391 total = len(git_shas)
392
393 for i, git_sha in enumerate(git_shas, 1):
394 meta = _git_commit_meta(repo_root, git_sha)
395
396 if verbose or i % 10 == 0 or i == 1 or i == total:
397 logger.info(
398 "[%s] %d/%d git:%s '%s'",
399 muse_branch,
400 i,
401 total,
402 git_sha[:12],
403 meta["message"][:60],
404 )
405
406 if not dry_run:
407 _extract_tree_to(repo_root, git_sha, workdir)
408
409 muse_id = _replay_commit(
410 repo_root=repo_root,
411 workdir=workdir,
412 git_sha=git_sha,
413 muse_branch=muse_branch,
414 parent_muse_id=parent_muse_id,
415 meta=meta,
416 repo_id=repo_id,
417 dry_run=dry_run,
418 )
419
420 git_to_muse[git_sha] = muse_id
421 parent_muse_id = muse_id
422
423 return git_to_muse
424
425
426 # ---------------------------------------------------------------------------
427 # Entry point
428 # ---------------------------------------------------------------------------
429
430
431 def _load_repo_id(repo_root: pathlib.Path) -> str:
432 import json
433 repo_json = repo_root / ".muse" / "repo.json"
434 data: dict[str, str] = json.loads(repo_json.read_text())
435 return data["repo_id"]
436
437
438 def main(argv: list[str] | None = None) -> int:
439 parser = argparse.ArgumentParser(
440 description="Replay a Git commit graph into a Muse repository."
441 )
442 parser.add_argument(
443 "--repo-root",
444 type=pathlib.Path,
445 default=_REPO_ROOT,
446 help="Path to the repository root (default: parent of this script).",
447 )
448 parser.add_argument(
449 "--dry-run",
450 action="store_true",
451 help="Log what would happen without writing anything.",
452 )
453 parser.add_argument(
454 "--verbose",
455 "-v",
456 action="store_true",
457 help="Log every commit (default: log every 10 + first/last).",
458 )
459 parser.add_argument(
460 "--branch",
461 nargs="+",
462 default=["main"],
463 metavar="BRANCH",
464 help=(
465 "Git branch(es) to replay, in order (default: main). "
466 "Pass multiple names: --branch main dev feat/x. "
467 "Use 'all' to replay every local branch (main first, then rest "
468 "alphabetically). The first branch listed is treated as the "
469 "primary branch and replayed in full; subsequent branches replay "
470 "only commits not reachable from any previously replayed branch."
471 ),
472 )
473 args = parser.parse_args(argv)
474
475 logging.basicConfig(
476 level=logging.INFO,
477 format="%(levelname)s %(message)s",
478 )
479
480 repo_root: pathlib.Path = args.repo_root.resolve()
481 dry_run: bool = args.dry_run
482 verbose: bool = args.verbose
483
484 # Resolve branch list.
485 requested: list[str] = args.branch
486 if requested == ["all"]:
487 all_branches = _git_local_branches(repo_root)
488 # main first, then alphabetically sorted remainder.
489 primaries = [b for b in all_branches if b == "main"]
490 others = sorted(b for b in all_branches if b != "main")
491 requested = primaries + others
492 logger.info("Branches to replay: %s", requested)
493
494 if not requested:
495 logger.error("❌ No branches to replay.")
496 return 1
497
498 # Auto-initialise if .muse/ doesn't exist yet.
499 if not (repo_root / ".muse" / "repo.json").exists():
500 logger.info("No .muse/repo.json found — running 'muse init --domain code' …")
501 if not dry_run:
502 result = subprocess.run(
503 ["muse", "init", "--domain", "code"],
504 cwd=repo_root,
505 capture_output=True,
506 text=True,
507 )
508 if result.returncode != 0:
509 logger.error("❌ muse init failed:\n%s", result.stderr)
510 return 1
511 logger.info("✅ muse init --domain code succeeded")
512
513 if (repo_root / ".muse" / "repo.json").exists():
514 repo_id = _load_repo_id(repo_root)
515 logger.info("✅ Muse repo ID: %s", repo_id)
516 else:
517 repo_id = "dry-run-placeholder"
518 logger.info("ℹ️ No repo.json yet (dry-run) — using placeholder repo ID")
519
520 with tempfile.TemporaryDirectory(prefix="git2muse-") as _tmpdir:
521 workdir = pathlib.Path(_tmpdir)
522
523 all_git_to_muse: dict[str, str] = {}
524 replayed_branches: list[str] = []
525
526 for branch in requested:
527 logger.info("━━━ Replaying branch: %s ━━━", branch)
528
529 # Commits on this branch not reachable from any already-replayed branch.
530 exclude = replayed_branches if replayed_branches else None
531 git_shas = _git_commits_oldest_first(repo_root, branch, exclude_branches=exclude)
532 git_shas = [s for s in git_shas if not _is_merge_commit(repo_root, s)]
533 logger.info(" %d non-merge commits unique to %s", len(git_shas), branch)
534
535 if not git_shas:
536 logger.info(" %s has no unique commits — skipping", branch)
537 replayed_branches.append(branch)
538 continue
539
540 # Find the Muse parent to branch from.
541 # The oldest unique commit's git parent should already be in our map.
542 start_parent_muse_id: str | None = None
543 oldest_sha = git_shas[0]
544 for gp in _git_parent_shas(repo_root, oldest_sha):
545 if gp in all_git_to_muse:
546 start_parent_muse_id = all_git_to_muse[gp]
547 break
548 if start_parent_muse_id is None and replayed_branches:
549 # Fall back to the HEAD of the primary (first) replayed branch.
550 start_parent_muse_id = _get_branch_head(repo_root, replayed_branches[0])
551
552 _set_head_ref(repo_root, branch)
553 mapping = _replay_branch(
554 repo_root=repo_root,
555 workdir=workdir,
556 git_shas=git_shas,
557 muse_branch=branch,
558 start_parent_muse_id=start_parent_muse_id,
559 repo_id=repo_id,
560 dry_run=dry_run,
561 verbose=verbose,
562 )
563 all_git_to_muse.update(mapping)
564 replayed_branches.append(branch)
565 logger.info("✅ %s: %d commits written", branch, len(mapping))
566
567 # Leave HEAD pointing at the primary branch.
568 if not dry_run and replayed_branches:
569 _set_head_ref(repo_root, replayed_branches[0])
570
571 logger.info("━━━ Done ━━━ total Muse commits written: %d", len(all_git_to_muse))
572 return 0
573
574
575 if __name__ == "__main__":
576 sys.exit(main())
File History 1 commit
sha256:1c4b3e3a9a1f300774c3ee662b572a698d5fd405bf765a71e6011a2e9c3eaaaa feat: Muse — version control for the agent era Human 151 days ago