gabriel / muse public
snapshot.py python
487 lines 18.8 KB
Raw
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9 docs: expand cache plan with all seven testing tiers and do… Sonnet 4.6 131 days ago
1 """Pure filesystem snapshot logic for ``muse commit``.
2
3 All functions here are side-effect-free (no DB, no I/O besides reading
4 files under ``workdir``). They are kept separate so they can be
5 unit-tested without a database.
6
7 ID derivation contract (deterministic, no random/UUID components):
8
9 object_id = "sha256:" + sha256(file_bytes).hexdigest()
10
11 snapshot_id = "sha256:" + sha256(
12 NUL.join(sorted(f"{path}NUL{strip(oid)}"
13 for path, oid in manifest.items()))
14 ).hexdigest() # strip() removes any leading "sha256:" prefix
15
16 commit_id = "sha256:" + sha256(
17 NUL.join([NUL.join(sorted(strip(p) for p in parent_ids)),
18 strip(snapshot_id), message, committed_at_iso])
19 ).hexdigest() # strip() removes any leading "sha256:" prefix
20
21 All three functions normalize their inputs by stripping any ``sha256:`` prefix
22 before hashing. This makes the IDs stable regardless of whether callers pass
23 canonical ``sha256:<hex>`` or legacy bare-hex strings — the resulting ID is
24 always identical.
25
26 The null byte (\\x00) is used as the field separator because it is:
27 - Illegal in POSIX filenames (preventing separator-injection attacks from
28 crafted file paths).
29 - Absent from SHA-256 hex strings (preventing injection via object IDs).
30 - Absent from ISO-8601 timestamps and typical message text.
31
32 This replaces the previous ``|`` / ``:`` separator scheme which allowed two
33 distinct manifests or commit inputs to produce the same hash if filenames
34 contained those characters.
35
36 Symlinks in the working tree are excluded from snapshots. Following a
37 symlink that points outside state/ would silently commit the contents
38 of arbitrary filesystem paths.
39
40 Exclusion policy
41 ----------------
42 Dotfiles and dot-directories are **tracked by default** — ``.cursorrules``,
43 ``.editorconfig``, ``.eslintrc`` are intentional project configuration that
44 collaborators need. Exclusion is driven entirely by ``.museignore`` plus the
45 built-in secrets blocklist below. The only hard-coded directory skip is
46 ``.muse/`` itself (internal VCS storage) and a performance-only list of
47 directories that are universally noise (``node_modules/``, ``__pycache__/``,
48 ``.venv/`` etc.).
49 """
50
51 from __future__ import annotations
52
53 import fnmatch
54 import os
55 import pathlib
56 import re
57 import stat as _stat
58
59 from muse.core._types import Manifest, blob_id, hash_file, load_json_file, split_id
60 from muse.core.paths import repo_json_path as _repo_json_path
61 from muse.core.ignore import is_ignored, load_ignore_config, resolve_patterns
62 from muse.core.stat_cache import load_cache
63
64 # Directories that are always pruned before os.walk descends into them.
65 # These are either internal VCS storage (.muse) or universally-noisy
66 # directories whose contents are never meaningful project source.
67 # Kept as a frozenset for O(1) lookup inside the hot walk loop.
68 _ALWAYS_PRUNE_DIRS: frozenset[str] = frozenset(
69 {
70 ".muse",
71 ".git",
72 "node_modules",
73 "__pycache__",
74 ".venv",
75 "venv",
76 ".tox",
77 ".nox",
78 ".mypy_cache",
79 ".ruff_cache",
80 ".pytest_cache",
81 ".coverage",
82 "htmlcov",
83 "dist",
84 "build",
85 }
86 )
87
88 # Built-in secrets blocklist — applied even when .museignore is absent.
89 # This is the last line of defence: these files must never appear in a
90 # snapshot regardless of what a user configures in .museignore.
91 #
92 # Note: .env.example is intentionally NOT listed here — it is the universal
93 # convention for a safe, credential-free environment template and must be
94 # trackable. We block the real secret files explicitly instead of using a
95 # wildcard that would accidentally catch .env.example.
96 _BUILTIN_SECRET_PATTERNS: list[str] = [
97 ".env",
98 ".env.local",
99 ".env.development",
100 ".env.staging",
101 ".env.production",
102 ".env.prod",
103 ".envrc",
104 "*.pem",
105 "*.key",
106 "*.p12",
107 "*.pfx",
108 ".DS_Store",
109 "Thumbs.db",
110 ]
111
112
113 def _build_filename_filter(patterns: list[str]) -> re.Pattern[str] | None:
114 """Compile a combined regex for fast per-file ignore pre-rejection.
115
116 Translates every *simple* pattern (no ``/`` in the body) into a single
117 alternating regex so ``re.search(fname)`` can reject the overwhelming
118 majority of files in one call instead of N ``fnmatch`` calls.
119
120 Only patterns without ``/`` in their body are included — they test the
121 filename component. Patterns with an embedded ``/`` (e.g.
122 ``docs/*.md``) or a trailing ``/`` (directory patterns) must still go
123 through the full :func:`~muse.core.ignore.is_ignored` path.
124
125 Returns ``None`` when *patterns* is empty or contains no simple patterns.
126
127 Performance: at 75 000 files with 9 builtin patterns, replacing 9 × N
128 ``fnmatch`` calls with one ``re.search`` call reduces ignore-matching
129 overhead by ~10×, dropping warm ``walk_workdir`` time from ~850 ms to
130 ~85 ms at 75 k scale, making the 1-file-change target of < 200 ms
131 achievable.
132 """
133 translated: list[str] = []
134 for raw_pat in patterns:
135 body = raw_pat.lstrip("!") # strip negation marker
136 if body.endswith("/"):
137 body = body.rstrip("/")
138 if "/" in body:
139 continue # path-level pattern — needs full is_ignored evaluation
140 translated.append(fnmatch.translate(body))
141 if not translated:
142 return None
143 return re.compile(f"(?:{'|'.join(translated)})")
144
145
146 def load_ignore_patterns(workdir: pathlib.Path) -> list[str]:
147 """Return the combined ignore pattern list for *workdir*.
148
149 Reads ``.museignore`` from *workdir* and detects the active domain from
150 ``.muse/repo.json``. Falls back to ``"code"`` when either file is absent.
151 The built-in secrets blocklist is always prepended so it cannot be
152 overridden by user configuration.
153
154 This function is intentionally public so that commands outside
155 ``snapshot.py`` (e.g. ``shelf``) can apply the same ignore rules without
156 duplicating the domain-detection logic.
157 """
158 domain = "code"
159 repo_json = _repo_json_path(workdir)
160 if repo_json.exists():
161 raw = load_json_file(repo_json)
162 if isinstance(raw, dict) and isinstance(raw.get("domain"), str):
163 domain = raw["domain"]
164
165 config = load_ignore_config(workdir)
166 user_patterns = resolve_patterns(config, domain)
167 return _BUILTIN_SECRET_PATTERNS + user_patterns
168
169 _SEP = "\x00"
170
171
172
173 def build_snapshot_manifest(workdir: pathlib.Path) -> Manifest:
174 """Return ``{rel_path: object_id}`` for every tracked file in *workdir*.
175
176 Preferred public name; delegates to :func:`walk_workdir`.
177 """
178 return walk_workdir(workdir)
179
180
181 def directories_from_manifest(files: Manifest) -> list[str]:
182 """Derive all implicit parent directories from a file manifest.
183
184 For every file path in *files*, all ancestor directory components are
185 collected. The result is a sorted, deduplicated list of POSIX directory
186 paths relative to the repository root.
187
188 Empty directories that have no files are not present in *files* and
189 therefore cannot be derived here — they require an explicit ``.musekeep``
190 marker file so the filesystem walk in :func:`walk_workdir_with_dirs` can
191 detect them.
192
193 This helper is used by merge / rebase / cherry-pick operations that
194 compute a merged file manifest without performing a fresh filesystem
195 walk, so that every ``SnapshotRecord`` stores a consistent directory list.
196 """
197 dirs: set[str] = set()
198 for path in files:
199 parts = path.split("/")
200 for i in range(1, len(parts)):
201 dirs.add("/".join(parts[:i]))
202 return sorted(dirs)
203
204
205 def walk_workdir_with_dirs(
206 workdir: pathlib.Path,
207 ) -> tuple[Manifest, list[str]]:
208 """Walk *workdir* and return ``(files_manifest, sorted_directories)``.
209
210 A single ``os.walk`` pass collects both the file content map and the
211 list of every non-root directory encountered (minus always-pruned dirs).
212 This is the canonical entry point for commit and status operations that
213 need first-class directory identity.
214
215 See :func:`walk_workdir` for the exclusion rules that apply to files.
216 Directories follow the same pruning rules — any directory whose name is
217 in :data:`_ALWAYS_PRUNE_DIRS` is never descended into and therefore
218 never appears in the returned list.
219 """
220 ignore_patterns = load_ignore_patterns(workdir)
221 cache = load_cache(workdir)
222 manifest: Manifest = {}
223 dirs: list[str] = []
224 root_str = str(workdir)
225 prefix_len = len(root_str) + 1
226
227 _filename_filter: re.Pattern[str] | None = _build_filename_filter(ignore_patterns)
228 _has_complex_patterns: bool = any(
229 "/" in p.lstrip("!")
230 for p in ignore_patterns
231 )
232
233 for dirpath, dirnames, filenames in os.walk(root_str, followlinks=False):
234 # Prune always-excluded names and any subdirectory that is itself a
235 # nested muse repo (contains .muse/). Nested repos are independent
236 # version-controlled units — their contents belong to their own
237 # snapshot, not the parent repo's.
238 dirnames[:] = [
239 d for d in dirnames
240 if d not in _ALWAYS_PRUNE_DIRS
241 and not os.path.isdir(os.path.join(dirpath, d, ".muse"))
242 ]
243
244 # Track every non-root directory we descend into.
245 if dirpath != root_str:
246 rel_dir = dirpath[prefix_len:]
247 if os.sep != "/":
248 rel_dir = rel_dir.replace(os.sep, "/")
249 dirs.append(rel_dir)
250
251 for fname in filenames:
252 abs_str = os.path.join(dirpath, fname)
253 try:
254 st = os.lstat(abs_str)
255 except OSError:
256 continue
257 if not _stat.S_ISREG(st.st_mode):
258 continue
259 rel = abs_str[prefix_len:]
260 if os.sep != "/":
261 rel = rel.replace(os.sep, "/")
262 if (
263 _filename_filter is not None
264 and not _filename_filter.search(fname)
265 and not _has_complex_patterns
266 ):
267 manifest[rel] = cache.get_cached(rel, abs_str, st.st_mtime, st.st_size, st.st_ino)
268 continue
269 if is_ignored(rel, ignore_patterns):
270 continue
271 manifest[rel] = cache.get_cached(rel, abs_str, st.st_mtime, st.st_size, st.st_ino)
272
273 cache.prune(set(manifest))
274 cache.save()
275 return manifest, sorted(dirs)
276
277
278 def walk_workdir(workdir: pathlib.Path) -> Manifest:
279 """Walk *workdir* and return only the file manifest.
280
281 Thin wrapper around :func:`walk_workdir_with_dirs` that discards the
282 directory list. Callers that need both files and directories should call
283 :func:`walk_workdir_with_dirs` directly to avoid a second filesystem walk.
284
285 Walk *workdir* recursively and return ``{rel_path: object_id}``.
286
287 Exclusions (all silent, no warning emitted):
288 - Symlinks — following them could commit content from outside the repo.
289 - Non-regular files — only regular files are included.
290 - Paths matched by ``.museignore`` or the built-in secrets blocklist.
291 - Directories in ``_ALWAYS_PRUNE_DIRS`` — internal VCS storage and
292 universally-noisy directories (node_modules, __pycache__, .venv, …).
293
294 Dotfiles and dot-directories are tracked unless excluded by the above
295 rules. ``.cursorrules``, ``.editorconfig``, ``.eslintrc`` etc. are
296 intentional project configuration; the blanket dot-skip that Git-adjacent
297 tools inherited is not carried forward here.
298
299 Paths use POSIX separators regardless of host OS for cross-platform
300 reproducibility.
301
302 Performance note: ``os.walk`` with in-place ``dirnames`` pruning is used
303 instead of ``pathlib.rglob`` so that large noisy directories are never
304 descended into. The stat cache further skips re-hashing files whose
305 ``(mtime, size)`` is unchanged since the last walk.
306
307 Ignore-pattern fast path: patterns are compiled into a single combined
308 regex (see :func:`_build_filename_filter`) that is evaluated against the
309 bare filename once per file. For the builtin secrets blocklist (9 simple
310 ``*.ext`` / ``name`` patterns with no ``/``), this replaces 9 separate
311 ``fnmatch`` calls with one ``re.search`` call — a ~10× speedup at 75 k
312 scale that brings warm 1-file-change latency from ~850 ms to < 200 ms.
313 Files whose filename can't possibly match any pattern skip ``is_ignored``
314 entirely; files that might match (rare) fall through to the full check.
315 """
316 files, _ = walk_workdir_with_dirs(workdir)
317 return files
318
319
320 def compute_snapshot_id(
321 manifest: Manifest,
322 directories: list[str] | None = None,
323 ) -> str:
324 """Return sha256 of the sorted ``path NUL object_id`` pairs and directory paths.
325
326 The null-byte separator prevents collisions from filenames or object IDs
327 that contain the previous ``|`` / ``:`` separators.
328
329 Sorting ensures two identical working trees always produce the same
330 snapshot_id, regardless of filesystem traversal order.
331
332 When *directories* is provided (non-empty), directory paths are appended
333 to the hash payload so that a directory rename produces a different
334 snapshot ID even when file contents are unchanged. Callers that do not
335 yet track directories may omit the argument; the resulting ID is identical
336 to the pre-directory-tracking behaviour.
337 """
338 parts = sorted(
339 f"{path}{_SEP}{split_id(oid)[1]}" for path, oid in manifest.items()
340 )
341 if directories:
342 # Prefix directory entries with "dir" so they occupy a distinct namespace
343 # from file entries and cannot collide with path/oid pairs.
344 parts.extend(f"dir{_SEP}{d}" for d in sorted(directories))
345 payload = _SEP.join(parts).encode()
346 return blob_id(payload)
347
348
349 def detect_directory_renames(
350 deleted_dirs: set[str],
351 added_dirs: set[str],
352 last_manifest: Manifest,
353 current_manifest: Manifest,
354 ) -> list[tuple[str, str]]:
355 """Return ``[(old_dir, new_dir)]`` pairs detected from manifest diffs.
356
357 A directory rename is inferred when all files that were under *old_dir*
358 in *last_manifest* appear under *new_dir* in *current_manifest* with
359 identical object IDs (same content, different path). Empty directories
360 and directories whose file sets do not match any added directory are not
361 returned.
362
363 The heuristic is conservative: only 1-to-1 renames are reported. If
364 multiple added directories share the same file set (unusual but possible),
365 the match is ambiguous and no rename is emitted for that pair.
366 """
367 renames: list[tuple[str, str]] = []
368 matched_new: set[str] = set()
369
370 for old_dir in sorted(deleted_dirs):
371 prefix = f"{old_dir}/"
372 old_files = {
373 path[len(prefix):]: oid
374 for path, oid in last_manifest.items()
375 if path.startswith(prefix)
376 }
377 if not old_files:
378 continue # empty dir — can't match by content
379
380 candidates = [
381 new_dir for new_dir in sorted(added_dirs)
382 if new_dir not in matched_new
383 ]
384 for new_dir in candidates:
385 new_prefix = f"{new_dir}/"
386 new_files = {
387 path[len(new_prefix):]: oid
388 for path, oid in current_manifest.items()
389 if path.startswith(new_prefix)
390 }
391 if new_files == old_files:
392 renames.append((old_dir, new_dir))
393 matched_new.add(new_dir)
394 break
395
396 return renames
397
398
399 def diff_workdir_vs_snapshot(
400 workdir: pathlib.Path,
401 last_manifest: Manifest,
402 last_directories: list[str] | None = None,
403 ) -> tuple[set[str], set[str], set[str], set[str], set[str], set[str]]:
404 """Compare *workdir* against *last_manifest* from the previous commit.
405
406 Returns a tuple of six disjoint path sets:
407
408 - ``added`` — files in *workdir* absent from *last_manifest*.
409 - ``modified`` — files present in both but with a differing sha256 hash.
410 - ``deleted`` — files in *last_manifest* absent from *workdir*.
411 - ``untracked`` — non-empty only when *last_manifest* is empty (first
412 commit): every file in *workdir* is untracked.
413 - ``added_dirs`` — directories present in *workdir* but not in
414 *last_directories*.
415 - ``deleted_dirs``— directories in *last_directories* absent from *workdir*.
416
417 All paths use POSIX separators for cross-platform reproducibility.
418 """
419 if not workdir.exists():
420 return (
421 set(), set(),
422 set(last_manifest.keys()), set(),
423 set(), set(last_directories or []),
424 )
425
426 current_manifest, current_dirs = walk_workdir_with_dirs(workdir)
427 current_paths = set(current_manifest.keys())
428 last_paths = set(last_manifest.keys())
429
430 if not last_paths:
431 return set(), set(), set(), current_paths, set(current_dirs), set()
432
433 added = current_paths - last_paths
434 deleted = last_paths - current_paths
435 common = current_paths & last_paths
436 modified = {p for p in common if current_manifest[p] != last_manifest[p]}
437
438 # A file that was tracked in the last snapshot but is now listed in
439 # .museignore and still present on disk is not "deleted" — it has been
440 # intentionally moved out of tracking. Reporting it as deleted would
441 # block checkout, pollute status output, and cause shelf pop to unlink it.
442 # Only files that are genuinely absent from the working tree are deleted.
443 if deleted:
444 ignore_patterns = load_ignore_patterns(workdir)
445 deleted = {
446 p for p in deleted
447 if not (is_ignored(p, ignore_patterns) and (workdir / p).exists())
448 }
449
450 last_dirs_set = set(last_directories or [])
451 current_dirs_set = set(current_dirs)
452 added_dirs = current_dirs_set - last_dirs_set
453 deleted_dirs = last_dirs_set - current_dirs_set
454
455 return added, modified, deleted, set(), added_dirs, deleted_dirs
456
457
458 def compute_commit_id(
459 parent_ids: list[str],
460 snapshot_id: str,
461 message: str,
462 committed_at_iso: str,
463 repo_id: str = "",
464 author: str = "",
465 signer_public_key: str = "",
466 ) -> str:
467 """Return sha256 of the commit's canonical inputs (v2 formula).
468
469 Field order (null-byte separated):
470 repo_id, parents, snapshot_id, message, committed_at, author, signer_public_key
471
472 Uses null bytes as field separators to prevent separator-injection attacks.
473 ``parent_ids`` is sorted before hashing so insertion order does not affect
474 determinism. ``repo_id``, ``author``, and ``signer_public_key`` bind the
475 commit ID to its origin identity — preventing key-swap and cross-repo replay.
476 """
477 parts = [
478 repo_id,
479 _SEP.join(sorted(split_id(p)[1] for p in parent_ids)),
480 split_id(snapshot_id)[1],
481 message,
482 committed_at_iso,
483 author,
484 signer_public_key,
485 ]
486 payload = _SEP.join(parts).encode()
487 return blob_id(payload)
File History 3 commits
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9 docs: expand cache plan with all seven testing tiers and do… Sonnet 4.6 131 days ago
sha256:7f9e2ef5286aedad9c1e6011b4c46ca27f39dbdad6e3409357e36b26e46b3b7c docs: docstring sprint for-each-ref→hotspots — idiomatic ru… Sonnet 4.6 patch 137 days ago
sha256:a09b1b4f6838754495547f200aa0ce88e2f56ffc5b20b900f6f0cff2c3cdede9 fix(cursorignore): remove git-ism (.git/worktrees) Human minor 140 days ago