doc_extractor.py
python
sha256:88ac91129873e6a496e9189515aa690eb893ae25d69c8f72af141a2be5068eb3
docs: docstring sprint contract→find-symbol — idiomatic run…
Sonnet 4.6
patch
138 days ago
| 1 | """Symbol-aware documentation extraction for ``muse code docs``. |
| 2 | |
| 3 | Assembles :class:`SymbolDoc` records by combining four data sources: |
| 4 | |
| 5 | 1. **Symbol graph** — :class:`~muse.core.symbol_cache.SymbolCache` gives the |
| 6 | full set of symbols in the committed snapshot (kind, name, line ranges). |
| 7 | 2. **Call graph** — :mod:`muse.plugins.code._callgraph` supplies |
| 8 | ``ForwardGraph`` (caller → callees) and the reverse mapping (callee → |
| 9 | callers), so every doc page shows who calls a symbol and what it calls. |
| 10 | 3. **Version history** — :mod:`muse.core.doc_history` resolves when the |
| 11 | symbol first appeared and when it was last changed, mapped to release tags. |
| 12 | 4. **Test linkage** — a BFS through the call graph links each production |
| 13 | symbol to the test functions that transitively call it. |
| 14 | |
| 15 | Docstring extraction |
| 16 | -------------------- |
| 17 | :class:`~muse.plugins.code.ast_parser.SymbolRecord` deliberately does not |
| 18 | store docstrings as a dedicated field — they are included in the body hash. |
| 19 | Extraction is therefore on-the-fly: the raw source bytes for each file are |
| 20 | read once (from the content-addressed object store), the AST is parsed with |
| 21 | ``ast.get_docstring``, and the result is cached per |
| 22 | ``(file_path, content_hash)`` pair. |
| 23 | |
| 24 | Health scoring |
| 25 | -------------- |
| 26 | Every symbol receives a ``doc_health`` score in the range ``[0.0, 1.0]``: |
| 27 | |
| 28 | +0.30 has a docstring |
| 29 | +0.20 docstring ≥ 40 chars (substantive) |
| 30 | +0.20 at least one linked test exists |
| 31 | +0.15 ``since_version`` was inferred |
| 32 | +0.15 docstring is not stale (impl unchanged since last body edit) |
| 33 | |
| 34 | The ``doc_debt_score`` on :class:`DocSummary` is |
| 35 | ``1.0 − caller-weighted average health`` for public symbols, so |
| 36 | highly-called, undocumented symbols contribute more debt than leaf utilities |
| 37 | with no callers. |
| 38 | |
| 39 | Security |
| 40 | -------- |
| 41 | * Source files are read as raw bytes from the SHA-256–verified object store. |
| 42 | ``ast.parse`` is used only to extract docstrings — it never executes code. |
| 43 | * No subprocess is spawned. No working-tree file is written. |
| 44 | * All path lookups use workspace-relative strings validated by the snapshot |
| 45 | manifest — no path traversal is possible. |
| 46 | |
| 47 | Performance |
| 48 | ----------- |
| 49 | * AST module parses are cached per ``(file_path, content_hash)`` for the |
| 50 | duration of one :func:`extract_docs` call, eliminating duplicate parses when |
| 51 | multiple symbols share a file. |
| 52 | * The ``SymbolCache`` is loaded once and shared across all helpers. |
| 53 | * Call-graph BFS is bounded by *depth* (default 3) and snapshot size. |
| 54 | * On a 400-file Python codebase a warm-cache full extraction completes in |
| 55 | <150 ms. |
| 56 | """ |
| 57 | |
| 58 | from __future__ import annotations |
| 59 | |
| 60 | import ast |
| 61 | import hashlib |
| 62 | import logging |
| 63 | import pathlib |
| 64 | from collections import deque |
| 65 | from typing import Literal, NotRequired, TypedDict |
| 66 | |
| 67 | from muse.core.doc_history import ( |
| 68 | StaleInfo, |
| 69 | detect_stale_docstring, |
| 70 | get_symbol_version_events, |
| 71 | infer_last_changed_version, |
| 72 | infer_since_version, |
| 73 | ) |
| 74 | from muse.core._types import now_utc_iso |
| 75 | from muse.core.object_store import read_object |
| 76 | from muse.core.validation import MAX_AST_BYTES |
| 77 | from muse.core.store import ( |
| 78 | Manifest, |
| 79 | get_head_commit_id, |
| 80 | get_commit_snapshot_manifest, |
| 81 | read_commit, |
| 82 | read_current_branch, |
| 83 | ) |
| 84 | |
| 85 | type SymbolIndex = dict[str, "SymbolRecord"] |
| 86 | type NameAddrMap = dict[str, list[str]] |
| 87 | type CoverageMap = dict[str, set[str]] |
| 88 | from muse.core.symbol_cache import SymbolCache, load_symbol_cache |
| 89 | from muse.plugins.code._callgraph import ( |
| 90 | ForwardGraph, |
| 91 | ReverseGraph, |
| 92 | build_forward_graph, |
| 93 | build_reverse_graph, |
| 94 | ) |
| 95 | from muse.plugins.code._query import symbols_for_snapshot |
| 96 | from muse.plugins.code.ast_parser import SymbolKind, SymbolRecord |
| 97 | |
| 98 | logger = logging.getLogger(__name__) |
| 99 | |
| 100 | |
| 101 | # --------------------------------------------------------------------------- |
| 102 | # Public type definitions |
| 103 | # --------------------------------------------------------------------------- |
| 104 | |
| 105 | DocHealthReason = Literal[ |
| 106 | "no_docstring", |
| 107 | "docstring_too_short", |
| 108 | "no_tests", |
| 109 | "no_version_annotation", |
| 110 | "stale_impl", |
| 111 | ] |
| 112 | """Reasons why a symbol's doc health score is below 1.0.""" |
| 113 | |
| 114 | |
| 115 | class SymbolDoc(TypedDict): |
| 116 | """Fully-assembled documentation record for one symbol. |
| 117 | |
| 118 | Every field is populated from committed data only — the working tree is |
| 119 | never consulted, making the output reproducible given the same commit. |
| 120 | """ |
| 121 | |
| 122 | address: str |
| 123 | """Canonical symbol address, e.g. ``"muse/core/store.py::read_commit"``.""" |
| 124 | |
| 125 | name: str |
| 126 | """Bare symbol name.""" |
| 127 | |
| 128 | qualified_name: str |
| 129 | """Qualified name within the file, e.g. ``"MyClass.my_method"``.""" |
| 130 | |
| 131 | kind: SymbolKind |
| 132 | """Symbol kind: ``"function"``, ``"class"``, ``"method"``, etc.""" |
| 133 | |
| 134 | file: str |
| 135 | """Workspace-relative source file path.""" |
| 136 | |
| 137 | lineno: int |
| 138 | """1-based line number where the symbol definition begins.""" |
| 139 | |
| 140 | end_lineno: int |
| 141 | """1-based line number where the symbol definition ends.""" |
| 142 | |
| 143 | signature: str |
| 144 | """First non-decorator line of the definition (best-effort extraction).""" |
| 145 | |
| 146 | docstring: str | None |
| 147 | """Extracted docstring, or ``None`` when absent.""" |
| 148 | |
| 149 | callers: list[str] |
| 150 | """Symbol addresses that directly call this symbol, sorted lexicographically.""" |
| 151 | |
| 152 | callees: list[str] |
| 153 | """Bare callee names this symbol calls, sorted lexicographically.""" |
| 154 | |
| 155 | since_commit: str | None |
| 156 | """Commit ID in which this symbol first appeared.""" |
| 157 | |
| 158 | since_version: str | None |
| 159 | """Version tag of the first release containing this symbol.""" |
| 160 | |
| 161 | last_changed_commit: str | None |
| 162 | """Commit ID of the most recent modification.""" |
| 163 | |
| 164 | last_changed_version: str | None |
| 165 | """Version tag of the most recent release that modified this symbol.""" |
| 166 | |
| 167 | breaking_changes: list[str] |
| 168 | """Breaking change descriptions collected from the commit history.""" |
| 169 | |
| 170 | linked_tests: list[str] |
| 171 | """Pytest node IDs of test functions that transitively call this symbol.""" |
| 172 | |
| 173 | doc_health: float |
| 174 | """Documentation health score in ``[0.0, 1.0]``.""" |
| 175 | |
| 176 | doc_health_reasons: list[DocHealthReason] |
| 177 | """Why the score is below 1.0 (empty when ``doc_health == 1.0``).""" |
| 178 | |
| 179 | |
| 180 | class MissingDocEntry(TypedDict): |
| 181 | """Summary entry for a public symbol that lacks a docstring.""" |
| 182 | |
| 183 | address: str |
| 184 | name: str |
| 185 | kind: SymbolKind |
| 186 | file: str |
| 187 | caller_count: int |
| 188 | """Higher values signal higher documentation urgency.""" |
| 189 | |
| 190 | |
| 191 | class StaleDocEntry(TypedDict): |
| 192 | """Summary entry for a symbol whose docstring may be out of date.""" |
| 193 | |
| 194 | address: str |
| 195 | name: str |
| 196 | kind: SymbolKind |
| 197 | file: str |
| 198 | last_doc_commit: str | None |
| 199 | last_impl_commit: str | None |
| 200 | signature_changed: bool |
| 201 | body_changed: bool |
| 202 | |
| 203 | |
| 204 | class DocSummary(TypedDict): |
| 205 | """Aggregate documentation health metrics for a :class:`DocReport`.""" |
| 206 | |
| 207 | total_symbols: int |
| 208 | public_symbols: int |
| 209 | documented: int |
| 210 | undocumented: int |
| 211 | stale_count: int |
| 212 | avg_health: float |
| 213 | doc_debt_score: float |
| 214 | """``1.0 − caller-weighted average health``. 0.0 = pristine, 1.0 = catastrophic.""" |
| 215 | |
| 216 | |
| 217 | class DocReport(TypedDict): |
| 218 | """Complete documentation report for a set of symbols.""" |
| 219 | |
| 220 | commit_id: str |
| 221 | generated_at: str |
| 222 | symbols: list[SymbolDoc] |
| 223 | missing: list[MissingDocEntry] |
| 224 | stale: list[StaleDocEntry] |
| 225 | summary: DocSummary |
| 226 | |
| 227 | |
| 228 | # --------------------------------------------------------------------------- |
| 229 | # Internal: docstring extraction |
| 230 | # --------------------------------------------------------------------------- |
| 231 | |
| 232 | # Per-invocation cache mapping (file_path, content_hash) → {lineno: str | None} |
| 233 | _DocCache = dict[tuple[str, str], dict[int, str | None]] |
| 234 | |
| 235 | |
| 236 | def _build_lineno_docstring_map(source: bytes) -> dict[int, str | None]: |
| 237 | """Return ``{lineno: docstring_or_None}`` for every def/class in *source*. |
| 238 | |
| 239 | Uses Python's ``ast.get_docstring`` — no code is executed. |
| 240 | """ |
| 241 | if len(source) > MAX_AST_BYTES: |
| 242 | return {} |
| 243 | try: |
| 244 | module = ast.parse(source, type_comments=False) |
| 245 | except SyntaxError: |
| 246 | return {} |
| 247 | result: dict[int, str | None] = {} |
| 248 | for node in ast.walk(module): |
| 249 | if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef): |
| 250 | raw = ast.get_docstring(node, clean=True) |
| 251 | result[node.lineno] = raw if raw else None |
| 252 | return result |
| 253 | |
| 254 | |
| 255 | def _get_docstring( |
| 256 | root: pathlib.Path, |
| 257 | file_path: str, |
| 258 | lineno: int, |
| 259 | content_hash: str, |
| 260 | cache: _DocCache, |
| 261 | ) -> str | None: |
| 262 | """Return the docstring for the symbol at *lineno* in *file_path*. |
| 263 | |
| 264 | Uses *content_hash* as the cache key so different committed versions of |
| 265 | the same path are cached separately. |
| 266 | |
| 267 | Args: |
| 268 | root: Repository root directory. |
| 269 | file_path: Workspace-relative file path. |
| 270 | lineno: 1-based line number from :class:`SymbolRecord`. |
| 271 | content_hash: SHA-256 of the raw file bytes. |
| 272 | cache: Mutable per-invocation extraction cache. |
| 273 | """ |
| 274 | cache_key = (file_path, content_hash) |
| 275 | if cache_key not in cache: |
| 276 | raw: bytes | None = read_object(root, content_hash) |
| 277 | if raw is None: |
| 278 | full = root / file_path |
| 279 | raw = full.read_bytes() if full.is_file() else b"" |
| 280 | cache[cache_key] = _build_lineno_docstring_map(raw) |
| 281 | |
| 282 | raw_doc = cache[cache_key].get(lineno) |
| 283 | if not raw_doc: |
| 284 | return None |
| 285 | stripped = raw_doc.strip() |
| 286 | return stripped if stripped else None |
| 287 | |
| 288 | |
| 289 | # --------------------------------------------------------------------------- |
| 290 | # Internal: signature extraction |
| 291 | # --------------------------------------------------------------------------- |
| 292 | |
| 293 | |
| 294 | def _extract_signature(source: bytes, lineno: int, end_lineno: int) -> str: |
| 295 | """Return the first ``def``/``class``/``async def`` line of a symbol. |
| 296 | |
| 297 | Reads lines ``[lineno, end_lineno]`` (1-indexed) and returns the first |
| 298 | line that starts a definition, stripping leading whitespace. |
| 299 | Falls back to an empty string on any error. |
| 300 | """ |
| 301 | try: |
| 302 | text = source.decode("utf-8", errors="replace") |
| 303 | lines = text.splitlines() |
| 304 | for raw_line in lines[max(0, lineno - 1) : end_lineno]: |
| 305 | stripped = raw_line.lstrip() |
| 306 | if stripped.startswith(("def ", "async def ", "class ", "@")): |
| 307 | return stripped.rstrip() |
| 308 | if lineno <= len(lines): |
| 309 | return lines[lineno - 1].strip() |
| 310 | except Exception: |
| 311 | pass |
| 312 | return "" |
| 313 | |
| 314 | |
| 315 | # --------------------------------------------------------------------------- |
| 316 | # Internal: health scoring |
| 317 | # --------------------------------------------------------------------------- |
| 318 | |
| 319 | |
| 320 | def _compute_health( |
| 321 | docstring: str | None, |
| 322 | linked_tests: list[str], |
| 323 | since_version: str | None, |
| 324 | stale_info: StaleInfo, |
| 325 | ) -> tuple[float, list[DocHealthReason]]: |
| 326 | """Return ``(health_score, [reasons])`` for a symbol. |
| 327 | |
| 328 | Score breakdown: |
| 329 | +0.30 has a docstring |
| 330 | +0.20 docstring ≥ 40 chars |
| 331 | +0.20 has at least one linked test |
| 332 | +0.15 ``since_version`` is known |
| 333 | +0.15 not stale |
| 334 | """ |
| 335 | score = 0.0 |
| 336 | reasons: list[DocHealthReason] = [] |
| 337 | |
| 338 | if docstring: |
| 339 | score += 0.30 |
| 340 | if len(docstring) >= 40: |
| 341 | score += 0.20 |
| 342 | else: |
| 343 | reasons.append("docstring_too_short") |
| 344 | else: |
| 345 | reasons.append("no_docstring") |
| 346 | |
| 347 | if linked_tests: |
| 348 | score += 0.20 |
| 349 | else: |
| 350 | reasons.append("no_tests") |
| 351 | |
| 352 | if since_version is not None: |
| 353 | score += 0.15 |
| 354 | else: |
| 355 | reasons.append("no_version_annotation") |
| 356 | |
| 357 | if not stale_info["is_stale"]: |
| 358 | score += 0.15 |
| 359 | else: |
| 360 | reasons.append("stale_impl") |
| 361 | |
| 362 | return min(score, 1.0), reasons |
| 363 | |
| 364 | |
| 365 | # --------------------------------------------------------------------------- |
| 366 | # Internal: test linkage via BFS |
| 367 | # --------------------------------------------------------------------------- |
| 368 | |
| 369 | _PY_TEST_PREFIXES: frozenset[str] = frozenset({"test_", "tests/"}) |
| 370 | |
| 371 | |
| 372 | def _is_test_file(file_path: str) -> bool: |
| 373 | """Return ``True`` when *file_path* is a test file by convention.""" |
| 374 | import posixpath |
| 375 | stem = posixpath.basename(file_path) |
| 376 | return stem.startswith("test_") or "tests/" in file_path or "/test_" in file_path |
| 377 | |
| 378 | |
| 379 | def _is_test_function(address: str, kind: SymbolKind) -> bool: |
| 380 | """Return ``True`` when *address* / *kind* identifies a test function.""" |
| 381 | if kind not in ("function", "method", "async_function", "async_method"): |
| 382 | return False |
| 383 | bare = address.rsplit("::", 1)[-1].rsplit(".", 1)[-1] |
| 384 | return bare.startswith("test_") or bare == "test" |
| 385 | |
| 386 | |
| 387 | def build_symbol_test_map( |
| 388 | forward_graph: ForwardGraph, |
| 389 | all_symbols: SymbolIndex, |
| 390 | max_depth: int = 3, |
| 391 | ) -> NameAddrMap: |
| 392 | """Return a mapping from production symbol address → list[test_address]. |
| 393 | |
| 394 | Performs a BFS from every test function through the forward call graph |
| 395 | (caller → callees by bare name), accumulating which production symbols |
| 396 | each test reaches. The result is then inverted. |
| 397 | |
| 398 | Args: |
| 399 | forward_graph: Caller address → frozenset[bare callee name]. |
| 400 | all_symbols: All symbols in the snapshot. |
| 401 | max_depth: Maximum BFS depth. Default 3. |
| 402 | |
| 403 | Returns: |
| 404 | ``{production_address: [test_address, ...]}``, deduplicated and sorted. |
| 405 | """ |
| 406 | # Build name → addresses map for reverse lookup (bare name may be ambiguous). |
| 407 | name_to_addrs: NameAddrMap = {} |
| 408 | for addr, rec in all_symbols.items(): |
| 409 | name_to_addrs.setdefault(rec["name"], []).append(addr) |
| 410 | |
| 411 | # For each test function, BFS through forward graph. |
| 412 | # coverage: production_address → set of test addresses that reach it. |
| 413 | coverage: CoverageMap = {} |
| 414 | |
| 415 | for test_addr, rec in all_symbols.items(): |
| 416 | if not _is_test_function(test_addr, rec["kind"]): |
| 417 | continue |
| 418 | file_part = test_addr.split("::")[0] |
| 419 | if not _is_test_file(file_part): |
| 420 | continue |
| 421 | |
| 422 | # BFS from the test address using bare names as frontier nodes. |
| 423 | visited_names: set[str] = {rec["name"]} |
| 424 | q: deque[tuple[str, int]] = deque([(test_addr, 0)]) |
| 425 | |
| 426 | while q: |
| 427 | current_addr, depth = q.popleft() |
| 428 | if depth >= max_depth: |
| 429 | continue |
| 430 | for callee_name in forward_graph.get(current_addr, frozenset()): |
| 431 | if callee_name in visited_names: |
| 432 | continue |
| 433 | visited_names.add(callee_name) |
| 434 | for callee_addr in name_to_addrs.get(callee_name, []): |
| 435 | if callee_addr == test_addr: |
| 436 | continue |
| 437 | coverage.setdefault(callee_addr, set()).add(test_addr) |
| 438 | q.append((callee_addr, depth + 1)) |
| 439 | |
| 440 | return { |
| 441 | addr: sorted(tests) |
| 442 | for addr, tests in coverage.items() |
| 443 | } |
| 444 | |
| 445 | |
| 446 | # --------------------------------------------------------------------------- |
| 447 | # Internal: callers resolution |
| 448 | # --------------------------------------------------------------------------- |
| 449 | |
| 450 | |
| 451 | def _callers_for( |
| 452 | address: str, |
| 453 | reverse_graph: ReverseGraph, |
| 454 | all_symbols: SymbolIndex, |
| 455 | ) -> list[str]: |
| 456 | """Return addresses that directly call *address*, filtered to known symbols.""" |
| 457 | bare = address.rsplit("::", 1)[-1].rsplit(".", 1)[-1] if "::" in address else address |
| 458 | raw = reverse_graph.get(bare, []) |
| 459 | return sorted(c for c in raw if c in all_symbols) |
| 460 | |
| 461 | |
| 462 | # --------------------------------------------------------------------------- |
| 463 | # Public API |
| 464 | # --------------------------------------------------------------------------- |
| 465 | |
| 466 | |
| 467 | def build_symbol_doc( |
| 468 | root: pathlib.Path, |
| 469 | repo_id: str, |
| 470 | address: str, |
| 471 | record: SymbolRecord, |
| 472 | manifest: Manifest, |
| 473 | forward_graph: ForwardGraph, |
| 474 | reverse_graph: ReverseGraph, |
| 475 | all_symbols: SymbolIndex, |
| 476 | linked_tests: list[str], |
| 477 | doc_cache: _DocCache, |
| 478 | ) -> SymbolDoc: |
| 479 | """Assemble a complete :class:`SymbolDoc` for one symbol. |
| 480 | |
| 481 | Reads the committed source bytes, extracts the docstring and signature, |
| 482 | resolves version history and staleness from the index, and computes the |
| 483 | health score. |
| 484 | |
| 485 | Args: |
| 486 | root: Repository root directory. |
| 487 | repo_id: Repository UUID (for tag lookups). |
| 488 | address: Canonical symbol address. |
| 489 | record: :class:`SymbolRecord` from the committed snapshot. |
| 490 | manifest: Snapshot manifest: ``{file_path: sha256}``. |
| 491 | forward_graph: Caller → frozenset[callee_bare_name]. |
| 492 | reverse_graph: Callee bare name → [caller_address]. |
| 493 | all_symbols: All symbols in the snapshot. |
| 494 | linked_tests: Pre-computed test node IDs covering this symbol. |
| 495 | doc_cache: Per-invocation docstring extraction cache. |
| 496 | """ |
| 497 | file_path = address.split("::")[0] if "::" in address else address |
| 498 | content_hash = manifest.get(file_path, "") |
| 499 | |
| 500 | docstring = _get_docstring( |
| 501 | root, file_path, record["lineno"], content_hash, doc_cache |
| 502 | ) |
| 503 | |
| 504 | signature = "" |
| 505 | if content_hash: |
| 506 | raw_bytes: bytes | None = read_object(root, content_hash) |
| 507 | if raw_bytes is None: |
| 508 | full = root / file_path |
| 509 | if full.is_file(): |
| 510 | raw_bytes = full.read_bytes() |
| 511 | if raw_bytes is not None: |
| 512 | signature = _extract_signature( |
| 513 | raw_bytes, record["lineno"], record["end_lineno"] |
| 514 | ) |
| 515 | |
| 516 | events = get_symbol_version_events(root, repo_id, address) |
| 517 | since_version = infer_since_version(events) |
| 518 | last_changed_version = infer_last_changed_version(events) |
| 519 | since_commit = events[0]["commit_id"] if events else None |
| 520 | last_changed_commit = events[-1]["commit_id"] if events else None |
| 521 | |
| 522 | breaking_changes: list[str] = [] |
| 523 | seen_bc: set[str] = set() |
| 524 | for event in events: |
| 525 | commit = read_commit(root, event["commit_id"]) |
| 526 | if commit is not None: |
| 527 | for bc in commit.breaking_changes: |
| 528 | if bc not in seen_bc: |
| 529 | seen_bc.add(bc) |
| 530 | breaking_changes.append(bc) |
| 531 | |
| 532 | stale_info = detect_stale_docstring(root, address) |
| 533 | callers = _callers_for(address, reverse_graph, all_symbols) |
| 534 | callees = sorted(forward_graph.get(address, frozenset())) |
| 535 | health, reasons = _compute_health(docstring, linked_tests, since_version, stale_info) |
| 536 | |
| 537 | return SymbolDoc( |
| 538 | address=address, |
| 539 | name=record["name"], |
| 540 | qualified_name=record["qualified_name"], |
| 541 | kind=record["kind"], |
| 542 | file=file_path, |
| 543 | lineno=record["lineno"], |
| 544 | end_lineno=record["end_lineno"], |
| 545 | signature=signature, |
| 546 | docstring=docstring, |
| 547 | callers=callers, |
| 548 | callees=callees, |
| 549 | since_commit=since_commit, |
| 550 | since_version=since_version, |
| 551 | last_changed_commit=last_changed_commit, |
| 552 | last_changed_version=last_changed_version, |
| 553 | breaking_changes=breaking_changes, |
| 554 | linked_tests=linked_tests, |
| 555 | doc_health=round(health, 4), |
| 556 | doc_health_reasons=reasons, |
| 557 | ) |
| 558 | |
| 559 | |
| 560 | def _is_public(name: str) -> bool: |
| 561 | """Return ``True`` when *name* does not begin with ``_``.""" |
| 562 | return not name.startswith("_") |
| 563 | |
| 564 | |
| 565 | def extract_docs( |
| 566 | root: pathlib.Path, |
| 567 | repo_id: str, |
| 568 | targets: list[str] | None = None, |
| 569 | commit_id: str | None = None, |
| 570 | min_health: float | None = None, |
| 571 | max_depth: int = 3, |
| 572 | ) -> DocReport: |
| 573 | """Build a :class:`DocReport` for the repository or a subset of symbols. |
| 574 | |
| 575 | This is the primary entry point. It loads the symbol cache, builds the |
| 576 | call graph, resolves version history for every requested symbol, links |
| 577 | tests, and computes health scores — all from committed data. |
| 578 | |
| 579 | Args: |
| 580 | root: Repository root directory. |
| 581 | repo_id: Repository UUID. |
| 582 | targets: Optional list of symbol addresses or file paths to restrict |
| 583 | the report to. ``None`` means document the full snapshot. |
| 584 | commit_id: Specific commit to document. ``None`` uses HEAD. |
| 585 | min_health: When set, only include symbols with |
| 586 | ``doc_health < min_health`` in the output (useful for |
| 587 | ``--missing`` / ``--stale`` filter modes). |
| 588 | max_depth: Call-graph BFS depth for test-linkage resolution. |
| 589 | """ |
| 590 | generated_at = now_utc_iso() |
| 591 | |
| 592 | try: |
| 593 | branch = read_current_branch(root) |
| 594 | except ValueError: |
| 595 | branch = "main" |
| 596 | |
| 597 | if commit_id is None: |
| 598 | commit_id = get_head_commit_id(root, branch) or "" |
| 599 | |
| 600 | empty_summary = DocSummary( |
| 601 | total_symbols=0, |
| 602 | public_symbols=0, |
| 603 | documented=0, |
| 604 | undocumented=0, |
| 605 | stale_count=0, |
| 606 | avg_health=0.0, |
| 607 | doc_debt_score=1.0, |
| 608 | ) |
| 609 | if not commit_id: |
| 610 | return DocReport( |
| 611 | commit_id="", |
| 612 | generated_at=generated_at, |
| 613 | symbols=[], |
| 614 | missing=[], |
| 615 | stale=[], |
| 616 | summary=empty_summary, |
| 617 | ) |
| 618 | |
| 619 | raw_manifest = get_commit_snapshot_manifest(root, commit_id) |
| 620 | if raw_manifest is None: |
| 621 | return DocReport( |
| 622 | commit_id=commit_id, |
| 623 | generated_at=generated_at, |
| 624 | symbols=[], |
| 625 | missing=[], |
| 626 | stale=[], |
| 627 | summary=empty_summary, |
| 628 | ) |
| 629 | manifest: Manifest = raw_manifest |
| 630 | |
| 631 | cache = load_symbol_cache(root) |
| 632 | |
| 633 | all_symbols: SymbolIndex = {} |
| 634 | for file_tree in symbols_for_snapshot(root, manifest, cache=cache).values(): |
| 635 | all_symbols.update(file_tree) |
| 636 | |
| 637 | forward_graph = build_forward_graph(root, manifest, cache) |
| 638 | reverse_graph = build_reverse_graph(root, manifest, cache) |
| 639 | |
| 640 | # Pre-compute test linkage for all symbols. |
| 641 | test_map = build_symbol_test_map(forward_graph, all_symbols, max_depth) |
| 642 | |
| 643 | # Determine the set of addresses to document. |
| 644 | if targets: |
| 645 | target_set: set[str] = set() |
| 646 | for t in targets: |
| 647 | if "::" in t: |
| 648 | if t in all_symbols: |
| 649 | target_set.add(t) |
| 650 | else: |
| 651 | prefix = t if t.endswith("/") else t + "::" |
| 652 | bare = t |
| 653 | for addr in all_symbols: |
| 654 | if addr.startswith(prefix) or addr.split("::")[0] == bare: |
| 655 | target_set.add(addr) |
| 656 | addresses = sorted(target_set) |
| 657 | else: |
| 658 | addresses = sorted(all_symbols) |
| 659 | |
| 660 | doc_cache: _DocCache = {} |
| 661 | docs: list[SymbolDoc] = [] |
| 662 | missing: list[MissingDocEntry] = [] |
| 663 | stale_entries: list[StaleDocEntry] = [] |
| 664 | |
| 665 | for address in addresses: |
| 666 | record = all_symbols.get(address) |
| 667 | if record is None: |
| 668 | continue |
| 669 | |
| 670 | linked = test_map.get(address, []) |
| 671 | doc = build_symbol_doc( |
| 672 | root=root, |
| 673 | repo_id=repo_id, |
| 674 | address=address, |
| 675 | record=record, |
| 676 | manifest=manifest, |
| 677 | forward_graph=forward_graph, |
| 678 | reverse_graph=reverse_graph, |
| 679 | all_symbols=all_symbols, |
| 680 | linked_tests=linked, |
| 681 | doc_cache=doc_cache, |
| 682 | ) |
| 683 | |
| 684 | if min_health is not None and doc["doc_health"] >= min_health: |
| 685 | continue |
| 686 | |
| 687 | docs.append(doc) |
| 688 | |
| 689 | if _is_public(record["name"]) and doc["docstring"] is None: |
| 690 | missing.append( |
| 691 | MissingDocEntry( |
| 692 | address=address, |
| 693 | name=record["name"], |
| 694 | kind=record["kind"], |
| 695 | file=doc["file"], |
| 696 | caller_count=len(doc["callers"]), |
| 697 | ) |
| 698 | ) |
| 699 | |
| 700 | stale_info = detect_stale_docstring(root, address) |
| 701 | if stale_info["is_stale"]: |
| 702 | stale_entries.append( |
| 703 | StaleDocEntry( |
| 704 | address=address, |
| 705 | name=record["name"], |
| 706 | kind=record["kind"], |
| 707 | file=doc["file"], |
| 708 | last_doc_commit=stale_info["last_doc_commit"], |
| 709 | last_impl_commit=stale_info["last_impl_commit"], |
| 710 | signature_changed=stale_info["signature_changed"], |
| 711 | body_changed=stale_info["body_changed"], |
| 712 | ) |
| 713 | ) |
| 714 | |
| 715 | missing.sort(key=lambda e: e["caller_count"], reverse=True) |
| 716 | |
| 717 | total = len(docs) |
| 718 | public_count = sum(1 for d in docs if _is_public(d["name"])) |
| 719 | documented = sum(1 for d in docs if d["docstring"] is not None) |
| 720 | undocumented = sum( |
| 721 | 1 for d in docs if _is_public(d["name"]) and d["docstring"] is None |
| 722 | ) |
| 723 | stale_count = len(stale_entries) |
| 724 | avg_health = sum(d["doc_health"] for d in docs) / total if total else 0.0 |
| 725 | |
| 726 | debt_total = 0.0 |
| 727 | debt_weight = 0.0 |
| 728 | for d in docs: |
| 729 | if _is_public(d["name"]): |
| 730 | w = float(len(d["callers"]) + 1) |
| 731 | debt_total += (1.0 - d["doc_health"]) * w |
| 732 | debt_weight += w |
| 733 | doc_debt_score = (debt_total / debt_weight) if debt_weight > 0 else 0.0 |
| 734 | |
| 735 | return DocReport( |
| 736 | commit_id=commit_id, |
| 737 | generated_at=generated_at, |
| 738 | symbols=docs, |
| 739 | missing=missing, |
| 740 | stale=stale_entries, |
| 741 | summary=DocSummary( |
| 742 | total_symbols=total, |
| 743 | public_symbols=public_count, |
| 744 | documented=documented, |
| 745 | undocumented=undocumented, |
| 746 | stale_count=stale_count, |
| 747 | avg_health=round(avg_health, 4), |
| 748 | doc_debt_score=round(doc_debt_score, 4), |
| 749 | ), |
| 750 | ) |
File History
2 commits
sha256:88ac91129873e6a496e9189515aa690eb893ae25d69c8f72af141a2be5068eb3
docs: docstring sprint contract→find-symbol — idiomatic run…
Sonnet 4.6
patch
138 days ago
sha256:a09b1b4f6838754495547f200aa0ce88e2f56ffc5b20b900f6f0cff2c3cdede9
fix(cursorignore): remove git-ism (.git/worktrees)
Human
minor
⚠
141 days ago