test_query_stat_cache.py
python
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9
docs: expand cache plan with all seven testing tiers and do…
Sonnet 4.6
132 days ago
| 1 | """TDD tests for StatCache integration into symbols_for_snapshot. |
| 2 | |
| 3 | Root cause |
| 4 | ---------- |
| 5 | ``symbols_for_snapshot(workdir=root)`` always calls ``disk_path.read_bytes()`` |
| 6 | for every Python file to compute the SHA-256 cache key, even when the file |
| 7 | hasn't changed since the last run. On the muse repo (~400 files) this costs |
| 8 | ~9,700 ms of pure disk I/O every single invocation. |
| 9 | |
| 10 | Fix |
| 11 | --- |
| 12 | Accept a ``stat_cache: StatCache | None`` parameter. On a stat-cache hit |
| 13 | (``ino + mtime + size`` match) the SHA-256 is already known — skip |
| 14 | ``read_bytes()`` entirely. Only when the SymbolCache also misses do we |
| 15 | actually read the file. |
| 16 | |
| 17 | Coverage |
| 18 | -------- |
| 19 | - ``symbols_for_snapshot`` accepts ``stat_cache=`` keyword argument. |
| 20 | - On stat-cache hit + symbol-cache hit: ``read_bytes()`` is never called. |
| 21 | - On stat-cache hit + symbol-cache miss: file is read once (to parse). |
| 22 | - On stat-cache miss: file is read (to hash + parse if needed). |
| 23 | - Stat cache is populated after a workdir call. |
| 24 | - Results are identical whether stat_cache is supplied or not. |
| 25 | - ``stat_cache`` is ignored when ``workdir=None`` (committed-blob path). |
| 26 | """ |
| 27 | |
| 28 | from __future__ import annotations |
| 29 | |
| 30 | import hashlib |
| 31 | import pathlib |
| 32 | from unittest.mock import patch, MagicMock |
| 33 | |
| 34 | import pytest |
| 35 | |
| 36 | from muse.core._types import blob_id |
| 37 | from muse.core.object_store import write_object |
| 38 | from muse.core.stat_cache import StatCache |
| 39 | from muse.core.symbol_cache import SymbolCache |
| 40 | from muse.plugins.code._query import symbols_for_snapshot |
| 41 | |
| 42 | |
| 43 | # --------------------------------------------------------------------------- |
| 44 | # Helpers |
| 45 | # --------------------------------------------------------------------------- |
| 46 | |
| 47 | |
| 48 | _PY_SRC = b"""\ |
| 49 | def compute(x: int) -> int: |
| 50 | return x * 2 |
| 51 | |
| 52 | def helper() -> int: |
| 53 | return 42 |
| 54 | """ |
| 55 | |
| 56 | _PY_SRC_V2 = b"""\ |
| 57 | def compute(x: int, y: int = 0) -> int: |
| 58 | return x * 2 + y |
| 59 | |
| 60 | def helper() -> int: |
| 61 | return 99 |
| 62 | """ |
| 63 | |
| 64 | |
| 65 | def _make_repo(tmp_path: pathlib.Path, content: bytes = _PY_SRC) -> tuple[pathlib.Path, dict]: |
| 66 | """Write a .muse repo with one Python file; return (root, manifest).""" |
| 67 | muse_dir = tmp_path / ".muse" |
| 68 | muse_dir.mkdir() |
| 69 | oid = blob_id(content) |
| 70 | write_object(tmp_path, oid, content) |
| 71 | (tmp_path / "billing.py").write_bytes(content) |
| 72 | return tmp_path, {"billing.py": oid} |
| 73 | |
| 74 | |
| 75 | # --------------------------------------------------------------------------- |
| 76 | # 1. symbols_for_snapshot accepts stat_cache= keyword |
| 77 | # --------------------------------------------------------------------------- |
| 78 | |
| 79 | |
| 80 | class TestAcceptsStatCache: |
| 81 | def test_accepts_stat_cache_none(self, tmp_path: pathlib.Path) -> None: |
| 82 | root, manifest = _make_repo(tmp_path) |
| 83 | result = symbols_for_snapshot(root, manifest, workdir=root, stat_cache=None) |
| 84 | assert "billing.py" in result |
| 85 | |
| 86 | def test_accepts_stat_cache_instance(self, tmp_path: pathlib.Path) -> None: |
| 87 | root, manifest = _make_repo(tmp_path) |
| 88 | sc = StatCache.empty() |
| 89 | result = symbols_for_snapshot(root, manifest, workdir=root, stat_cache=sc) |
| 90 | assert "billing.py" in result |
| 91 | |
| 92 | def test_result_unchanged_with_or_without_stat_cache( |
| 93 | self, tmp_path: pathlib.Path |
| 94 | ) -> None: |
| 95 | root, manifest = _make_repo(tmp_path) |
| 96 | r1 = symbols_for_snapshot(root, manifest, workdir=root) |
| 97 | r2 = symbols_for_snapshot(root, manifest, workdir=root, stat_cache=StatCache.empty()) |
| 98 | assert set(r1.get("billing.py", {})) == set(r2.get("billing.py", {})) |
| 99 | |
| 100 | |
| 101 | # --------------------------------------------------------------------------- |
| 102 | # 2. Stat-cache hit + symbol-cache hit → read_bytes never called |
| 103 | # --------------------------------------------------------------------------- |
| 104 | |
| 105 | |
| 106 | class TestStatCacheHitSkipsRead: |
| 107 | def test_warm_stat_and_symbol_cache_skips_read_bytes( |
| 108 | self, tmp_path: pathlib.Path |
| 109 | ) -> None: |
| 110 | """Both caches warm → file bytes never read.""" |
| 111 | root, manifest = _make_repo(tmp_path) |
| 112 | |
| 113 | # Warm both caches with a cold run. |
| 114 | sym_cache = SymbolCache.load(root / ".muse") |
| 115 | stat_cache = StatCache.load(root / ".muse") |
| 116 | symbols_for_snapshot( |
| 117 | root, manifest, workdir=root, cache=sym_cache, stat_cache=stat_cache |
| 118 | ) |
| 119 | sym_cache.save() |
| 120 | stat_cache.save() |
| 121 | |
| 122 | # Reload from disk — fully warm. |
| 123 | sym_cache2 = SymbolCache.load(root / ".muse") |
| 124 | stat_cache2 = StatCache.load(root / ".muse") |
| 125 | |
| 126 | read_call_count = [] |
| 127 | original_read_bytes = pathlib.Path.read_bytes |
| 128 | |
| 129 | def counting_read_bytes(self_path: pathlib.Path) -> bytes: |
| 130 | if self_path.suffix == ".py": |
| 131 | read_call_count.append(str(self_path)) |
| 132 | return original_read_bytes(self_path) |
| 133 | |
| 134 | with patch.object(pathlib.Path, "read_bytes", counting_read_bytes): |
| 135 | symbols_for_snapshot( |
| 136 | root, manifest, workdir=root, cache=sym_cache2, stat_cache=stat_cache2 |
| 137 | ) |
| 138 | |
| 139 | assert read_call_count == [], ( |
| 140 | f"read_bytes called on warm cache for: {read_call_count}" |
| 141 | ) |
| 142 | |
| 143 | def test_stat_cache_hit_symbol_cache_miss_reads_once( |
| 144 | self, tmp_path: pathlib.Path |
| 145 | ) -> None: |
| 146 | """Stat-cache hit but cold symbol cache → file read exactly once.""" |
| 147 | root, manifest = _make_repo(tmp_path) |
| 148 | |
| 149 | # Warm only the stat cache. |
| 150 | stat_cache = StatCache.load(root / ".muse") |
| 151 | symbols_for_snapshot(root, manifest, workdir=root, stat_cache=stat_cache) |
| 152 | stat_cache.save() |
| 153 | |
| 154 | stat_cache2 = StatCache.load(root / ".muse") |
| 155 | cold_sym_cache = SymbolCache.empty() |
| 156 | |
| 157 | read_call_count = [] |
| 158 | original_read_bytes = pathlib.Path.read_bytes |
| 159 | |
| 160 | def counting_read_bytes(self_path: pathlib.Path) -> bytes: |
| 161 | if self_path.suffix == ".py": |
| 162 | read_call_count.append(str(self_path)) |
| 163 | return original_read_bytes(self_path) |
| 164 | |
| 165 | with patch.object(pathlib.Path, "read_bytes", counting_read_bytes): |
| 166 | symbols_for_snapshot( |
| 167 | root, manifest, workdir=root, |
| 168 | cache=cold_sym_cache, stat_cache=stat_cache2, |
| 169 | ) |
| 170 | |
| 171 | assert len(read_call_count) == 1, ( |
| 172 | f"Expected exactly 1 read on stat-hit/sym-miss, got {read_call_count}" |
| 173 | ) |
| 174 | |
| 175 | |
| 176 | # --------------------------------------------------------------------------- |
| 177 | # 3. Stat cache is populated after a workdir call |
| 178 | # --------------------------------------------------------------------------- |
| 179 | |
| 180 | |
| 181 | class TestStatCachePopulated: |
| 182 | def test_stat_cache_has_entry_after_workdir_call( |
| 183 | self, tmp_path: pathlib.Path |
| 184 | ) -> None: |
| 185 | root, manifest = _make_repo(tmp_path) |
| 186 | sc = StatCache.load(root / ".muse") |
| 187 | symbols_for_snapshot(root, manifest, workdir=root, stat_cache=sc) |
| 188 | sc.save() |
| 189 | |
| 190 | sc2 = StatCache.load(root / ".muse") |
| 191 | # billing.py must be in the cache after the workdir call. |
| 192 | obj_hash = sc2.get_object_hash(root, root / "billing.py") |
| 193 | assert obj_hash == blob_id(_PY_SRC), ( |
| 194 | f"Stat cache returned wrong hash: {obj_hash}" |
| 195 | ) |
| 196 | |
| 197 | def test_stat_cache_file_created_on_disk(self, tmp_path: pathlib.Path) -> None: |
| 198 | root, manifest = _make_repo(tmp_path) |
| 199 | sc = StatCache.load(root / ".muse") |
| 200 | symbols_for_snapshot(root, manifest, workdir=root, stat_cache=sc) |
| 201 | sc.save() |
| 202 | assert (root / ".muse" / "cache" / "stat.msgpack").exists() |
| 203 | |
| 204 | |
| 205 | # --------------------------------------------------------------------------- |
| 206 | # 4. stat_cache= ignored when workdir=None (committed-blob path unchanged) |
| 207 | # --------------------------------------------------------------------------- |
| 208 | |
| 209 | |
| 210 | class TestStatCacheIgnoredWithoutWorkdir: |
| 211 | def test_no_read_bytes_called_for_committed_blobs( |
| 212 | self, tmp_path: pathlib.Path |
| 213 | ) -> None: |
| 214 | """Committed path reads from object store, not disk — stat_cache irrelevant.""" |
| 215 | root, manifest = _make_repo(tmp_path) |
| 216 | sc = StatCache.empty() |
| 217 | # Should not raise and should return symbols. |
| 218 | result = symbols_for_snapshot(root, manifest, stat_cache=sc) |
| 219 | assert "billing.py" in result |
| 220 | |
| 221 | |
| 222 | # --------------------------------------------------------------------------- |
| 223 | # 5. Changed file invalidates stat cache → re-read |
| 224 | # --------------------------------------------------------------------------- |
| 225 | |
| 226 | |
| 227 | class TestStatCacheInvalidation: |
| 228 | def test_edited_file_triggers_reread(self, tmp_path: pathlib.Path) -> None: |
| 229 | """After editing a file, stat cache miss → file is re-read.""" |
| 230 | root, manifest = _make_repo(tmp_path) |
| 231 | |
| 232 | # Warm stat cache with v1. |
| 233 | sc = StatCache.load(root / ".muse") |
| 234 | r1 = symbols_for_snapshot(root, manifest, workdir=root, stat_cache=sc) |
| 235 | sc.save() |
| 236 | |
| 237 | # Edit file on disk (v2 — different content, new mtime). |
| 238 | (root / "billing.py").write_bytes(_PY_SRC_V2) |
| 239 | |
| 240 | sc2 = StatCache.load(root / ".muse") |
| 241 | r2 = symbols_for_snapshot(root, manifest, workdir=root, stat_cache=sc2) |
| 242 | |
| 243 | # v2 has different signatures → symbol set differs. |
| 244 | syms1 = set(r1.get("billing.py", {})) |
| 245 | syms2 = set(r2.get("billing.py", {})) |
| 246 | # Both have 'compute' and 'helper' but content_id differs — result |
| 247 | # should still be parseable (regression: must not crash or return stale). |
| 248 | assert "billing.py" in r2 |
| 249 | assert any("compute" in addr for addr in syms2) |
File History
2 commits
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9
docs: expand cache plan with all seven testing tiers and do…
Sonnet 4.6
132 days ago
sha256:7f9e2ef5286aedad9c1e6011b4c46ca27f39dbdad6e3409357e36b26e46b3b7c
docs: docstring sprint for-each-ref→hotspots — idiomatic ru…
Sonnet 4.6
patch
138 days ago