gabriel / muse public
content_grep.py python
581 lines 20.1 KB
Raw
sha256:1c4b3e3a9a1f300774c3ee662b572a698d5fd405bf765a71e6011a2e9c3eaaaa feat: Muse — version control for the agent era Human 151 days ago
1 """``muse content-grep`` — full-text search across tracked files.
2
3 Searches the content of every tracked file for a pattern. By default the
4 search runs against the committed HEAD snapshot (reading from the object
5 store). Pass ``--working-tree`` to search the actual files on disk,
6 including uncommitted edits — essential for agents verifying their own
7 changes before committing.
8
9 ``--working-tree`` and ``--ref`` are mutually exclusive.
10
11 Binary files and non-UTF-8 files are silently skipped. Regex safety:
12 patterns are compiled with a 500-character length limit to prevent
13 catastrophic backtracking (ReDoS).
14
15 Performance (snapshot mode): object reads run in parallel using a bounded
16 ``ThreadPoolExecutor`` (``min(8, cpu_count())`` workers).
17
18 File filtering: ``--include`` and ``--exclude`` accept ``fnmatch``-style
19 glob patterns applied to relative file paths.
20
21 Usage::
22
23 muse content-grep --pattern "Cm7" # literal substring (HEAD)
24 muse content-grep --pattern "TODO" --working-tree # search working tree (disk)
25 muse content-grep --pattern "tempo:\\s+\\d+" # regex
26 muse content-grep --pattern "TODO" --ignore-case # case-insensitive
27 muse content-grep --pattern "chorus" --files-only # only file paths
28 muse content-grep --pattern "bass" --ref feat/audio # search a branch tip
29 muse content-grep --pattern "note" --include "*.txt" # only .txt files
30 muse content-grep --pattern "debug" --exclude "*.min.js"
31 muse content-grep --pattern "TODO" --max-matches 20 # cap results
32 muse content-grep --pattern "verse" --context 2 # 2 lines of context
33 muse content-grep --pattern "chord" --json # machine-readable
34
35 JSON output schema (``--json``)::
36
37 {
38 "source": "commit" | "working-tree",
39 "commit_id": "<64-char hex>" | null,
40 "snapshot_id": "<64-char hex>" | null,
41 "pattern": "<pattern string>",
42 "total_files_matched": <int>,
43 "total_matches": <int>,
44 "results": [
45 {
46 "path": "<relative path>",
47 "object_id": "<64-char hex>" | null,
48 "match_count": <int>,
49 "matches": [
50 {
51 "line_number": <int>,
52 "text": "<matched line>",
53 "context_before": ["<line>", ...],
54 "context_after": ["<line>", ...]
55 },
56 ...
57 ]
58 },
59 ...
60 ]
61 }
62
63 Exit codes::
64
65 0 — pattern found in at least one file
66 1 — no matches (or no commits)
67 3 — I/O error
68 """
69
70 from __future__ import annotations
71
72 import argparse
73 import concurrent.futures
74 import fnmatch
75 import json
76 import logging
77 import os
78 import pathlib
79 import re
80 import sys
81 from typing import TypedDict
82
83 from muse.core.errors import ExitCode
84 from muse.core.object_store import read_object
85 from muse.core.repo import read_repo_id, require_repo
86 from muse.core.store import (
87 get_head_commit_id,
88 read_commit,
89 read_current_branch,
90 read_snapshot,
91 resolve_commit_ref,
92 )
93 from muse.core.validation import sanitize_display
94
95 logger = logging.getLogger(__name__)
96
97 _BINARY_CHUNK = 8192
98 _MAX_PATTERN_LEN = 500 # reject patterns that could cause catastrophic backtracking
99 _DEFAULT_MAX_WORKERS = min(8, (os.cpu_count() or 1))
100
101 # Directories to skip when walking the working tree.
102 _SKIP_DIRS: frozenset[str] = frozenset({
103 ".muse",
104 ".git",
105 "__pycache__",
106 ".mypy_cache",
107 ".pytest_cache",
108 ".tox",
109 "node_modules",
110 ".venv",
111 "venv",
112 ".env",
113 })
114
115
116 # ---------------------------------------------------------------------------
117 # TypedDicts for structured output
118 # ---------------------------------------------------------------------------
119
120
121 class GrepMatch(TypedDict):
122 """A single matching line within a file, with optional surrounding context."""
123
124 line_number: int
125 text: str
126 context_before: list[str]
127 context_after: list[str]
128
129
130 class GrepFileResult(TypedDict):
131 """All matches within a single file."""
132
133 path: str
134 object_id: str | None # None when source is working-tree
135 match_count: int
136 matches: list[GrepMatch]
137
138
139 class _ContentGrepJson(TypedDict):
140 """Top-level JSON envelope for ``muse content-grep --json``."""
141
142 source: str # "commit" | "working-tree"
143 commit_id: str | None
144 snapshot_id: str | None
145 pattern: str
146 total_files_matched: int
147 total_matches: int
148 results: list[GrepFileResult]
149
150
151 # ---------------------------------------------------------------------------
152 # Internal helpers
153 # ---------------------------------------------------------------------------
154
155
156 def _is_binary(data: bytes) -> bool:
157 """Return ``True`` if *data* (the first chunk) contains null bytes."""
158 return b"\x00" in data
159
160
161 def _path_matches_globs(rel_path: str, include: str | None, exclude: str | None) -> bool:
162 """Return ``True`` if *rel_path* passes the include/exclude glob filters.
163
164 ``--include`` and ``--exclude`` use ``fnmatch`` on the basename **and** on
165 the full relative path so that patterns like ``*.py`` and ``src/*.py`` both
166 work intuitively.
167 """
168 basename = pathlib.PurePosixPath(rel_path).name
169 if include is not None:
170 if not (fnmatch.fnmatch(basename, include) or fnmatch.fnmatch(rel_path, include)):
171 return False
172 if exclude is not None:
173 if fnmatch.fnmatch(basename, exclude) or fnmatch.fnmatch(rel_path, exclude):
174 return False
175 return True
176
177
178 def _search_lines(
179 raw: bytes,
180 pattern: re.Pattern[str],
181 files_only: bool,
182 count_only: bool,
183 context_lines: int,
184 ) -> tuple[int, list[GrepMatch]]:
185 """Search *raw* bytes for *pattern*; return ``(match_count, matches)``.
186
187 Binary content and non-UTF-8 content return ``(0, [])``.
188 """
189 probe = raw[:_BINARY_CHUNK]
190 if _is_binary(probe):
191 return 0, []
192
193 text = raw.decode("utf-8", errors="replace")
194 all_lines = text.splitlines()
195
196 matches: list[GrepMatch] = []
197 total = 0
198 for lineno, line in enumerate(all_lines, start=1):
199 if pattern.search(line):
200 total += 1
201 if not files_only and not count_only:
202 before: list[str] = []
203 after: list[str] = []
204 if context_lines > 0:
205 idx = lineno - 1 # 0-based index
206 before = [
207 l.rstrip("\r")
208 for l in all_lines[max(0, idx - context_lines) : idx]
209 ]
210 after = [
211 l.rstrip("\r")
212 for l in all_lines[idx + 1 : idx + 1 + context_lines]
213 ]
214 matches.append(
215 GrepMatch(
216 line_number=lineno,
217 text=line.rstrip("\r"),
218 context_before=before,
219 context_after=after,
220 )
221 )
222
223 return total, matches
224
225
226 def _search_object(
227 root_path: pathlib.Path,
228 object_id: str,
229 pattern: re.Pattern[str],
230 files_only: bool,
231 count_only: bool,
232 context_lines: int,
233 ) -> tuple[int, list[GrepMatch]]:
234 """Search a committed object for *pattern*; return ``(match_count, matches)``."""
235 try:
236 raw = read_object(root_path, object_id)
237 except OSError as exc:
238 logger.warning("⚠️ grep: could not read object %s: %s", object_id[:12], exc)
239 return 0, []
240
241 if raw is None:
242 return 0, []
243
244 return _search_lines(raw, pattern, files_only, count_only, context_lines)
245
246
247 def _search_disk_file(
248 abs_path: pathlib.Path,
249 pattern: re.Pattern[str],
250 files_only: bool,
251 count_only: bool,
252 context_lines: int,
253 ) -> tuple[int, list[GrepMatch]]:
254 """Search a file on disk for *pattern*; return ``(match_count, matches)``."""
255 try:
256 raw = abs_path.read_bytes()
257 except OSError as exc:
258 logger.warning("⚠️ grep: could not read %s: %s", abs_path, exc)
259 return 0, []
260
261 return _search_lines(raw, pattern, files_only, count_only, context_lines)
262
263
264 def _walk_working_tree(
265 root: pathlib.Path,
266 include_glob: str | None,
267 exclude_glob: str | None,
268 ) -> list[tuple[str, pathlib.Path]]:
269 """Walk *root* recursively, skipping VCS/cache dirs, return ``(rel_path, abs_path)`` pairs."""
270 results: list[tuple[str, pathlib.Path]] = []
271 for dirpath, dirnames, filenames in os.walk(root):
272 # Prune directories in-place so os.walk skips them entirely.
273 dirnames[:] = sorted(
274 d for d in dirnames
275 if d not in _SKIP_DIRS and not d.startswith(".")
276 )
277 for filename in sorted(filenames):
278 abs_path = pathlib.Path(dirpath) / filename
279 try:
280 rel_path = abs_path.relative_to(root).as_posix()
281 except ValueError:
282 continue
283 if _path_matches_globs(rel_path, include_glob, exclude_glob):
284 results.append((rel_path, abs_path))
285 return results
286
287
288 # ---------------------------------------------------------------------------
289 # Registration
290 # ---------------------------------------------------------------------------
291
292
293 def register(subparsers: "argparse._SubParsersAction[argparse.ArgumentParser]") -> None:
294 """Register the content-grep subcommand."""
295 parser = subparsers.add_parser(
296 "content-grep",
297 help="Search tracked file content for a pattern.",
298 description=__doc__,
299 formatter_class=argparse.RawDescriptionHelpFormatter,
300 )
301 parser.add_argument(
302 "--pattern", "-p", required=True,
303 help="Regular expression pattern to search for.",
304 )
305 parser.add_argument(
306 "--working-tree", "-w", action="store_true", dest="working_tree",
307 help=(
308 "Search files on disk (working tree) instead of the committed HEAD snapshot. "
309 "Finds matches in uncommitted edits. Mutually exclusive with --ref."
310 ),
311 )
312 parser.add_argument(
313 "--ref", default=None,
314 help="Branch, tag, or commit SHA to search (default: HEAD). Mutually exclusive with --working-tree.",
315 )
316 parser.add_argument(
317 "--ignore-case", "-i", action="store_true", dest="ignore_case",
318 help="Case-insensitive matching.",
319 )
320 parser.add_argument(
321 "--files-only", "-l", action="store_true", dest="files_only",
322 help="Print only file paths with matches.",
323 )
324 parser.add_argument(
325 "--count", "-c", action="store_true", dest="count_mode",
326 help="Print only match counts per file.",
327 )
328 parser.add_argument(
329 "--include", default=None, dest="include_glob",
330 help="Only search files whose path matches this fnmatch glob (e.g. '*.py').",
331 )
332 parser.add_argument(
333 "--exclude", default=None, dest="exclude_glob",
334 help="Skip files whose path matches this fnmatch glob (e.g. '*.min.js').",
335 )
336 parser.add_argument(
337 "--max-matches", "-m", type=int, default=None, dest="max_matches",
338 help="Stop after this many total matches across all files.",
339 )
340 parser.add_argument(
341 "--context", "-C", type=int, default=0, dest="context_lines",
342 help="Number of surrounding lines to include with each match (like grep -C).",
343 )
344 parser.add_argument(
345 "--json", action="store_true", dest="json_out",
346 help="Emit machine-readable JSON.",
347 )
348 parser.set_defaults(func=run)
349
350
351 # ---------------------------------------------------------------------------
352 # Subcommand handler
353 # ---------------------------------------------------------------------------
354
355
356 def run(args: argparse.Namespace) -> None:
357 """Search tracked file content for a pattern.
358
359 Pattern validation (length + ``re.compile``) is performed *before* any I/O
360 so that invalid patterns are rejected cheaply.
361
362 When ``--working-tree`` is set, files are read directly from disk so that
363 uncommitted edits are visible. This is the correct mode for agents
364 verifying changes before committing. Object reads (snapshot mode) are
365 parallelised with a ``ThreadPoolExecutor`` for I/O efficiency.
366
367 Binary files and non-UTF-8 files are silently skipped. Exit code 0 means
368 at least one match was found; exit code 1 means no matches.
369
370 Examples::
371
372 muse content-grep --pattern "chorus"
373 muse content-grep --pattern "TODO" --working-tree
374 muse content-grep --pattern "TODO|FIXME" --files-only
375 muse content-grep --pattern "tempo" --ignore-case --json
376 muse content-grep --pattern "chord" --ref feat/harmony
377 muse content-grep --pattern "note" --include "*.txt" --max-matches 10
378 muse content-grep --pattern "verse" --context 2
379 """
380 pattern: str = args.pattern
381 working_tree: bool = args.working_tree
382 ref: str | None = args.ref
383 ignore_case: bool = args.ignore_case
384 files_only: bool = args.files_only
385 count_mode: bool = args.count_mode
386 include_glob: str | None = args.include_glob
387 exclude_glob: str | None = args.exclude_glob
388 max_matches: int | None = args.max_matches
389 context_lines: int = max(0, args.context_lines)
390 json_out: bool = args.json_out
391
392 if working_tree and ref is not None:
393 print("❌ --working-tree and --ref are mutually exclusive.", file=sys.stderr)
394 raise SystemExit(ExitCode.USER_ERROR)
395
396 # Validate pattern BEFORE any I/O — cheap rejection of bad inputs.
397 if len(pattern) > _MAX_PATTERN_LEN:
398 print(
399 f"❌ Pattern too long ({len(pattern)} chars, max {_MAX_PATTERN_LEN}). "
400 "Use a shorter pattern or re.escape() for literal matches.",
401 file=sys.stderr,
402 )
403 raise SystemExit(ExitCode.USER_ERROR)
404
405 flags = re.IGNORECASE if ignore_case else 0
406 try:
407 compiled: re.Pattern[str] = re.compile(pattern, flags)
408 except re.error as exc:
409 print(f"❌ Invalid regex: {exc}", file=sys.stderr)
410 raise SystemExit(ExitCode.USER_ERROR) from exc
411
412 root = require_repo()
413
414 # ── Working-tree mode ────────────────────────────────────────────────────
415 if working_tree:
416 disk_files = _walk_working_tree(root, include_glob, exclude_glob)
417
418 file_results: list[GrepFileResult] = []
419 total_matches = 0
420
421 for rel_path, abs_path in disk_files:
422 if max_matches is not None and total_matches >= max_matches:
423 break
424 match_count, matches = _search_disk_file(
425 abs_path, compiled, files_only, count_mode, context_lines
426 )
427 if match_count > 0:
428 if max_matches is not None:
429 remaining = max_matches - total_matches
430 if match_count > remaining:
431 matches = matches[:remaining]
432 match_count = len(matches)
433 file_results.append(
434 GrepFileResult(
435 path=rel_path,
436 object_id=None,
437 match_count=match_count,
438 matches=matches,
439 )
440 )
441 total_matches += match_count
442
443 _emit(
444 file_results=file_results,
445 total_matches=total_matches,
446 source="working-tree",
447 commit_id=None,
448 snapshot_id=None,
449 pattern=pattern,
450 json_out=json_out,
451 files_only=files_only,
452 count_mode=count_mode,
453 context_lines=context_lines,
454 )
455 if not file_results:
456 raise SystemExit(ExitCode.USER_ERROR) # exit 1 = no matches
457 return
458
459 # ── Snapshot mode (default) ──────────────────────────────────────────────
460 repo_id = read_repo_id(root)
461 branch = read_current_branch(root)
462
463 if ref is None:
464 commit_id = get_head_commit_id(root, branch)
465 if commit_id is None:
466 print("❌ No commits on current branch.", file=sys.stderr)
467 raise SystemExit(ExitCode.USER_ERROR)
468 else:
469 commit_rec = resolve_commit_ref(root, repo_id, branch, ref)
470 if commit_rec is None:
471 print(f"❌ Ref '{sanitize_display(ref)}' not found.", file=sys.stderr)
472 raise SystemExit(ExitCode.USER_ERROR)
473 commit_id = commit_rec.commit_id
474
475 commit = read_commit(root, commit_id)
476 if commit is None:
477 print(f"❌ Commit {commit_id[:12]} not found.", file=sys.stderr)
478 raise SystemExit(ExitCode.INTERNAL_ERROR)
479
480 snap = read_snapshot(root, commit.snapshot_id)
481 if snap is None:
482 print(f"❌ Snapshot {commit.snapshot_id[:12]} not found.", file=sys.stderr)
483 raise SystemExit(ExitCode.INTERNAL_ERROR)
484
485 filtered: list[tuple[str, str]] = [
486 (rel_path, object_id)
487 for rel_path, object_id in sorted(snap.manifest.items())
488 if _path_matches_globs(rel_path, include_glob, exclude_glob)
489 ]
490
491 def _search(item: tuple[str, str]) -> tuple[str, str, int, list[GrepMatch]]:
492 rel_path, object_id = item
493 cnt, ms = _search_object(root, object_id, compiled, files_only, count_mode, context_lines)
494 return rel_path, object_id, cnt, ms
495
496 snap_results: list[GrepFileResult] = []
497 snap_total = 0
498
499 with concurrent.futures.ThreadPoolExecutor(max_workers=_DEFAULT_MAX_WORKERS) as pool:
500 for rel_path, object_id, match_count, matches in pool.map(_search, filtered):
501 if match_count > 0:
502 if max_matches is not None:
503 remaining = max_matches - snap_total
504 if remaining <= 0:
505 break
506 if match_count > remaining:
507 matches = matches[:remaining]
508 match_count = len(matches)
509 snap_results.append(
510 GrepFileResult(
511 path=rel_path,
512 object_id=object_id,
513 match_count=match_count,
514 matches=matches,
515 )
516 )
517 snap_total += match_count
518
519 _emit(
520 file_results=snap_results,
521 total_matches=snap_total,
522 source="commit",
523 commit_id=commit_id,
524 snapshot_id=commit.snapshot_id,
525 pattern=pattern,
526 json_out=json_out,
527 files_only=files_only,
528 count_mode=count_mode,
529 context_lines=context_lines,
530 )
531 if not snap_results:
532 raise SystemExit(ExitCode.USER_ERROR) # exit 1 = no matches
533
534
535 # ---------------------------------------------------------------------------
536 # Output helper (shared between modes)
537 # ---------------------------------------------------------------------------
538
539
540 def _emit(
541 *,
542 file_results: list[GrepFileResult],
543 total_matches: int,
544 source: str,
545 commit_id: str | None,
546 snapshot_id: str | None,
547 pattern: str,
548 json_out: bool,
549 files_only: bool,
550 count_mode: bool,
551 context_lines: int,
552 ) -> None:
553 """Render search results to stdout in text or JSON format."""
554 if json_out:
555 payload: _ContentGrepJson = {
556 "source": source,
557 "commit_id": commit_id,
558 "snapshot_id": snapshot_id,
559 "pattern": pattern,
560 "total_files_matched": len(file_results),
561 "total_matches": total_matches,
562 "results": file_results,
563 }
564 print(json.dumps(payload, indent=2))
565 else:
566 for fr in file_results:
567 safe_path = sanitize_display(fr["path"])
568 if files_only:
569 print(safe_path)
570 elif count_mode:
571 print(f"{safe_path}:{fr['match_count']}")
572 else:
573 for m in fr["matches"]:
574 if context_lines > 0:
575 for ctx in m["context_before"]:
576 print(f"{safe_path}:{m['line_number']}-{sanitize_display(ctx)}")
577 print(f"{safe_path}:{m['line_number']}:{sanitize_display(m['text'])}")
578 for ctx in m["context_after"]:
579 print(f"{safe_path}:{m['line_number']}+{sanitize_display(ctx)}")
580 else:
581 print(f"{safe_path}:{m['line_number']}:{sanitize_display(m['text'])}")
File History 1 commit
sha256:1c4b3e3a9a1f300774c3ee662b572a698d5fd405bf765a71e6011a2e9c3eaaaa feat: Muse — version control for the agent era Human 151 days ago