implicit_edge_cache.py
python
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9
docs: expand cache plan with all seven testing tiers and do…
Sonnet 4.6
131 days ago
| 1 | """Persistent implicit-edge cache — eliminates re-running framework plugins. |
| 2 | |
| 3 | Architecture |
| 4 | ------------ |
| 5 | ``build_implicit_edge_graph`` re-reads every Python blob and re-runs all |
| 6 | framework plugins (FastAPI, Flask, Celery) on every invocation: ~10 s for |
| 7 | a 779-file repo. The result is fully determined by file content — the same |
| 8 | bytes always produce the same list of ``ImplicitEntryEdge`` objects. |
| 9 | |
| 10 | ``ImplicitEdgeCache`` exploits this by persisting a **per-file edge list** |
| 11 | keyed by the SHA-256 of the file bytes (``object_id`` from the manifest): |
| 12 | |
| 13 | key: SHA-256 hex digest of raw Python file bytes (``object_id``) |
| 14 | value: list of ``ImplicitEntryEdge`` objects for the file |
| 15 | |
| 16 | Storing the complete per-file result means the warm path skips every |
| 17 | expensive operation: ``read_object``, ``parse_symbols``, and all plugin |
| 18 | ``detect_entry_points`` calls. |
| 19 | |
| 20 | This mirrors ``CallGraphCache`` exactly: |
| 21 | |
| 22 | * Same content-addressed key. |
| 23 | * Same storage format (``.muse/implicit_edge_cache.msgpack``). |
| 24 | * Same atomic write (temp-file-then-rename). |
| 25 | * Same graceful-load contract (never raises; corrupt → empty). |
| 26 | |
| 27 | Each ``ImplicitEntryEdge`` is serialised as a plain dict (four string fields). |
| 28 | On load, dicts are validated and converted back to dataclass instances. |
| 29 | |
| 30 | Storage |
| 31 | ------- |
| 32 | ``.muse/implicit_edge_cache.msgpack``:: |
| 33 | |
| 34 | { |
| 35 | "version": 1, |
| 36 | "entries": { |
| 37 | "<sha256-hex>": [ |
| 38 | {"framework_id": "fastapi", "symbol_address": "…", |
| 39 | "kind": "http-route", "metadata": {"method": "GET", "path": "/"}}, |
| 40 | … |
| 41 | ] |
| 42 | } |
| 43 | } |
| 44 | |
| 45 | Typical lifecycle inside ``build_implicit_edge_graph``:: |
| 46 | |
| 47 | cache = load_implicit_edge_cache(root) |
| 48 | for file_path, obj_id in manifest.items(): |
| 49 | edges = cache.get(obj_id) |
| 50 | if edges is None: |
| 51 | raw = read_object(root, obj_id) |
| 52 | edges = _run_plugins(file_path, raw) |
| 53 | cache.put(obj_id, edges) |
| 54 | for edge in edges: |
| 55 | graph.setdefault(edge.symbol_address, []).append(edge) |
| 56 | cache.save() |
| 57 | """ |
| 58 | |
| 59 | from __future__ import annotations |
| 60 | |
| 61 | import logging |
| 62 | import pathlib |
| 63 | from typing import TYPE_CHECKING |
| 64 | |
| 65 | import msgpack |
| 66 | |
| 67 | from muse.core.paths import cache_dir as _cache_dir_path, muse_dir as _muse_dir |
| 68 | from muse.core._types import MsgpackValue |
| 69 | from muse.core.store import read_msgpack_file |
| 70 | |
| 71 | type _EdgeDict = dict[str, str] |
| 72 | |
| 73 | if TYPE_CHECKING: |
| 74 | from muse.plugins.code._framework import ImplicitEntryEdge |
| 75 | |
| 76 | logger = logging.getLogger(__name__) |
| 77 | |
| 78 | _CACHE_VERSION = 1 |
| 79 | _CACHE_FILENAME = "implicit_edges.msgpack" |
| 80 | |
| 81 | # Required string fields in a serialised ImplicitEntryEdge dict. |
| 82 | _REQUIRED_STR_FIELDS = ("framework_id", "symbol_address", "kind") |
| 83 | |
| 84 | |
| 85 | def _edge_from_dict(raw: MsgpackValue) -> ImplicitEntryEdge | None: |
| 86 | """Deserialise a raw msgpack dict to an ``ImplicitEntryEdge``, or None on error. |
| 87 | |
| 88 | Imports ``ImplicitEntryEdge`` lazily to avoid a circular import with |
| 89 | ``muse.plugins.code._framework``. |
| 90 | """ |
| 91 | if not isinstance(raw, dict): |
| 92 | return None |
| 93 | for field in _REQUIRED_STR_FIELDS: |
| 94 | if not isinstance(raw.get(field), str): |
| 95 | return None |
| 96 | metadata = raw.get("metadata", {}) |
| 97 | if not isinstance(metadata, dict): |
| 98 | metadata = {} |
| 99 | from muse.plugins.code._framework import ImplicitEntryEdge # lazy |
| 100 | return ImplicitEntryEdge( |
| 101 | framework_id=raw["framework_id"], |
| 102 | symbol_address=raw["symbol_address"], |
| 103 | kind=raw["kind"], |
| 104 | metadata={str(k): str(v) for k, v in metadata.items()}, |
| 105 | ) |
| 106 | |
| 107 | |
| 108 | def _edge_to_dict(edge: ImplicitEntryEdge) -> _EdgeDict: |
| 109 | """Serialise an ``ImplicitEntryEdge`` to a plain msgpack-compatible dict.""" |
| 110 | return { |
| 111 | "framework_id": edge.framework_id, |
| 112 | "symbol_address": edge.symbol_address, |
| 113 | "kind": edge.kind, |
| 114 | "metadata": dict(edge.metadata), |
| 115 | } |
| 116 | |
| 117 | |
| 118 | class ImplicitEdgeCache: |
| 119 | """Persistent msgpack cache mapping object_id → list[ImplicitEntryEdge]. |
| 120 | |
| 121 | ``ImplicitEntryEdge`` is imported lazily to avoid a circular import with |
| 122 | ``muse.plugins.code._framework``. |
| 123 | |
| 124 | Typical lifecycle inside ``build_implicit_edge_graph``:: |
| 125 | |
| 126 | cache = ImplicitEdgeCache.loadmuse_dir(root) |
| 127 | for file_path, object_id in manifest.items(): |
| 128 | edges = cache.get(object_id) |
| 129 | if edges is None: |
| 130 | raw = read_object(root, object_id) |
| 131 | edges = _run_plugins(file_path, raw) |
| 132 | cache.put(object_id, edges) |
| 133 | for edge in edges: |
| 134 | graph.setdefault(edge.symbol_address, []).append(edge) |
| 135 | cache.save() |
| 136 | """ |
| 137 | |
| 138 | def __init__( |
| 139 | self, |
| 140 | cache_dir: pathlib.Path | None, |
| 141 | entries: dict[str, list[ImplicitEntryEdge]], |
| 142 | ) -> None: |
| 143 | self._cache_dir = cache_dir |
| 144 | self._entries: dict[str, list[ImplicitEntryEdge]] = entries |
| 145 | self._dirty = False |
| 146 | |
| 147 | # ------------------------------------------------------------------ |
| 148 | # Construction |
| 149 | # ------------------------------------------------------------------ |
| 150 | |
| 151 | @classmethod |
| 152 | def load(cls, muse_dir: pathlib.Path) -> ImplicitEdgeCache: |
| 153 | """Load the cache from *muse_dir*/cache/implicit_edges.msgpack. |
| 154 | |
| 155 | Returns a fresh empty cache if the file is absent, unreadable, or |
| 156 | version-mismatched — never raises. Invalid entries are skipped so |
| 157 | a partially corrupt file does not poison the entire cache. |
| 158 | """ |
| 159 | cache_dir = muse_dir / "cache" |
| 160 | cache_file = cache_dir / _CACHE_FILENAME |
| 161 | if not cache_file.is_file(): |
| 162 | return cls(cache_dir, {}) |
| 163 | try: |
| 164 | doc = read_msgpack_file(cache_file) |
| 165 | if not isinstance(doc, dict) or doc.get("version") != _CACHE_VERSION: |
| 166 | logger.debug("⚠️ implicit_edge_cache version mismatch — starting fresh") |
| 167 | return cls(cache_dir, {}) |
| 168 | raw_entries = doc.get("entries") |
| 169 | if not isinstance(raw_entries, dict): |
| 170 | return cls(cache_dir, {}) |
| 171 | entries: dict[str, list[ImplicitEntryEdge]] = {} |
| 172 | for obj_id, edge_list_raw in raw_entries.items(): |
| 173 | if not isinstance(obj_id, str): |
| 174 | continue |
| 175 | if not isinstance(edge_list_raw, list): |
| 176 | continue |
| 177 | edge_list: list[ImplicitEntryEdge] = [] |
| 178 | valid = True |
| 179 | for raw_edge in edge_list_raw: |
| 180 | edge = _edge_from_dict(raw_edge) |
| 181 | if edge is None: |
| 182 | valid = False |
| 183 | break |
| 184 | edge_list.append(edge) |
| 185 | if valid: |
| 186 | entries[obj_id] = edge_list |
| 187 | return cls(cache_dir, entries) |
| 188 | except Exception as exc: # noqa: BLE001 |
| 189 | logger.debug("⚠️ implicit_edge_cache unreadable (%s) — starting fresh", exc) |
| 190 | return cls(cache_dir, {}) |
| 191 | |
| 192 | @classmethod |
| 193 | def empty(cls) -> ImplicitEdgeCache: |
| 194 | """Return a no-op cache for contexts without a ``.muse`` directory.""" |
| 195 | return cls(None, {}) |
| 196 | |
| 197 | # ------------------------------------------------------------------ |
| 198 | # Data access |
| 199 | # ------------------------------------------------------------------ |
| 200 | |
| 201 | def get(self, object_id: str) -> list[ImplicitEntryEdge] | None: |
| 202 | """Return the cached edge list for *object_id*, or ``None`` on miss.""" |
| 203 | return self._entries.get(object_id) |
| 204 | |
| 205 | def put(self, object_id: str, edges: list[ImplicitEntryEdge]) -> None: |
| 206 | """Store *edges* under *object_id* and mark the cache dirty.""" |
| 207 | self._entries[object_id] = edges |
| 208 | self._dirty = True |
| 209 | |
| 210 | def prune(self, live_ids: set[str]) -> None: |
| 211 | """Remove entries whose object IDs are not in *live_ids*.""" |
| 212 | stale = set(self._entries) - live_ids |
| 213 | if stale: |
| 214 | for k in stale: |
| 215 | del self._entries[k] |
| 216 | self._dirty = True |
| 217 | logger.debug("🗑️ implicit_edge_cache pruned %d stale entries", len(stale)) |
| 218 | |
| 219 | @property |
| 220 | def size(self) -> int: |
| 221 | """Number of cached object IDs.""" |
| 222 | return len(self._entries) |
| 223 | |
| 224 | # ------------------------------------------------------------------ |
| 225 | # Persistence |
| 226 | # ------------------------------------------------------------------ |
| 227 | |
| 228 | def save(self) -> None: |
| 229 | """Atomically persist the cache to disk if it has changed. |
| 230 | |
| 231 | Uses a temp-file-then-rename pattern so a crash mid-write never |
| 232 | leaves a corrupt cache file. Silently skips when there is no |
| 233 | ``.muse`` directory (e.g. in-memory unit tests). |
| 234 | """ |
| 235 | if not self._dirty or self._cache_dir is None: |
| 236 | return |
| 237 | self._cache_dir.mkdir(parents=True, exist_ok=True) |
| 238 | serializable = { |
| 239 | obj_id: [_edge_to_dict(e) for e in edges] |
| 240 | for obj_id, edges in self._entries.items() |
| 241 | } |
| 242 | doc = {"version": _CACHE_VERSION, "entries": serializable} |
| 243 | cache_file = self._cache_dir / _CACHE_FILENAME |
| 244 | tmp = self._cache_dir / f"{_CACHE_FILENAME}.tmp" |
| 245 | try: |
| 246 | tmp.write_bytes(msgpack.packb(doc, use_bin_type=True)) |
| 247 | tmp.replace(cache_file) |
| 248 | self._dirty = False |
| 249 | logger.debug("✅ implicit_edge_cache saved (%d entries)", len(self._entries)) |
| 250 | except OSError as exc: |
| 251 | logger.warning("⚠️ implicit_edge_cache save failed: %s", exc) |
| 252 | |
| 253 | |
| 254 | def load_implicit_edge_cache(root: pathlib.Path) -> ImplicitEdgeCache: |
| 255 | """Convenience loader: return an ``ImplicitEdgeCache`` for a repository root. |
| 256 | |
| 257 | Returns ``ImplicitEdgeCache.empty()`` when *root* has no ``.muse`` directory. |
| 258 | """ |
| 259 | _dir = _muse_dir(root) |
| 260 | if _dir.is_dir(): |
| 261 | return ImplicitEdgeCache.load(_dir) |
| 262 | return ImplicitEdgeCache.empty() |
File History
2 commits
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9
docs: expand cache plan with all seven testing tiers and do…
Sonnet 4.6
131 days ago
sha256:7f9e2ef5286aedad9c1e6011b4c46ca27f39dbdad6e3409357e36b26e46b3b7c
docs: docstring sprint for-each-ref→hotspots — idiomatic ru…
Sonnet 4.6
patch
137 days ago