gabriel / musehub public
test_intel_pacing.py python
189 lines 8.2 KB
Raw
sha256:71cfef07c390e977ebb8066cbd052c3c5de869eb5395522f6768fd32c016f8ae feat(intel): small write batches and a stricter yield rule … Sonnet 5.5 patch 3 days ago
1 """Database pacing for code-intelligence writes (production incident 2026-11-13: a cold index starved the web on a small database).
2
3 P1 Off by default: no queries, no sleeps, zero overhead.
4 P2 Duty cycle: sleeps ``t * (1/d - 1)`` after a unit that took ``t`` (capped), never after a failed unit.
5 P3 Back-off while the web is busy: waits while other queries are active, bounded per unit and per job; a failed look never fails the analysis.
6 P4 The build is paced end to end through the real code path, and an unpaced build is unchanged.
7 """
8 from __future__ import annotations
9
10 import pytest
11
12 from musehub.services.musehub_intel_pacing import MAX_DUTY_SLEEP_SECONDS, DbPacer
13
14
15 class Clock:
16 def __init__(self) -> None:
17 self.now = 0.0
18 self.sleeps: list[float] = []
19
20 def __call__(self) -> float:
21 return self.now
22
23 async def sleep(self, seconds: float) -> None:
24 self.sleeps.append(seconds)
25 self.now += seconds
26
27
28 class FakeSession:
29 """Only what the pacer touches: ``begin_nested`` (a savepoint) and ``execute`` returning the number of other active queries."""
30
31 def __init__(self, counts: list[int] | None = None, fail: bool = False) -> None:
32 self.counts, self.fail, self.looks = list(counts or []), fail, 0
33
34 def begin_nested(self):
35 class _Ctx:
36 async def __aenter__(self_inner): return self_inner
37 async def __aexit__(self_inner, *a): return False
38 return _Ctx()
39
40 async def execute(self, _stmt):
41 self.looks += 1
42 if self.fail:
43 raise RuntimeError("no access to pg_stat_activity")
44
45 class _R:
46 def __init__(self, n): self.n = n
47 def scalar_one(self): return self.n
48 return _R(self.counts.pop(0) if self.counts else 0)
49
50
51 def test_p1_off_by_default_and_from_empty_environment() -> None:
52 assert DbPacer.from_environ({}).enabled is False
53 assert DbPacer().enabled is False
54 assert DbPacer.from_environ({"MUSEHUB_INTEL_DB_DUTY": "garbage"}).enabled is False
55 assert DbPacer.from_environ({"MUSEHUB_INTEL_DB_DUTY": "1"}).enabled is False
56 assert DbPacer.from_environ({"MUSEHUB_INTEL_DB_DUTY": "0"}).enabled is False # nonsense (would sleep forever): ignored
57 assert DbPacer.from_environ({"MUSEHUB_INTEL_DB_DUTY": "0.4"}).enabled is True
58 assert DbPacer.from_environ({"MUSEHUB_INTEL_DB_BUSY_THRESHOLD": "2"}).enabled is True
59
60
61 @pytest.mark.asyncio
62 async def test_p1_a_disabled_pacer_touches_nothing() -> None:
63 clock = Clock()
64 p = DbPacer(sleep=clock.sleep, clock=clock)
65 async with p.unit(None): # type: ignore[arg-type] # a disabled pacer never uses the session
66 clock.now += 5
67 assert clock.sleeps == [] and p.units == 0
68
69
70 @pytest.mark.asyncio
71 async def test_p2_duty_cycle_sleeps_the_complement_and_is_capped() -> None:
72 clock = Clock()
73 p = DbPacer(duty=0.25, sleep=clock.sleep, clock=clock)
74 async with p.unit(FakeSession()): # type: ignore[arg-type]
75 clock.now += 2.0
76 assert clock.sleeps == [pytest.approx(6.0)] # busy 2 s of every 8 s = 25 %
77 async with p.unit(FakeSession()): # type: ignore[arg-type]
78 clock.now += 100.0
79 assert clock.sleeps[-1] == MAX_DUTY_SLEEP_SECONDS
80
81
82 @pytest.mark.asyncio
83 async def test_p2_a_failed_unit_does_not_nap() -> None:
84 clock = Clock()
85 p = DbPacer(duty=0.25, sleep=clock.sleep, clock=clock)
86 with pytest.raises(ValueError):
87 async with p.unit(FakeSession()): # type: ignore[arg-type]
88 clock.now += 2.0
89 raise ValueError("batch failed")
90 assert clock.sleeps == []
91
92
93 @pytest.mark.asyncio
94 async def test_p3_it_waits_while_other_queries_are_active_then_proceeds() -> None:
95 clock = Clock()
96 s = FakeSession([5, 4, 3, 0])
97 p = DbPacer(busy_threshold=2, poll_seconds=2.0, sleep=clock.sleep, clock=clock)
98 async with p.unit(s): # type: ignore[arg-type]
99 pass
100 assert s.looks == 4 and clock.sleeps == [2.0, 2.0, 2.0] and p.waited_busy == 6.0
101
102
103 @pytest.mark.asyncio
104 async def test_p3_a_permanently_busy_database_delays_but_never_stops_the_job() -> None:
105 clock = Clock()
106 p = DbPacer(busy_threshold=2, poll_seconds=2.0, max_wait_seconds=10.0, total_wait_seconds=14.0, sleep=clock.sleep, clock=clock)
107 for _ in range(4):
108 async with p.unit(FakeSession([9] * 100)): # type: ignore[arg-type]
109 pass
110 assert p.waited_busy <= 14.0 + 2.0 # the per-job budget is spent, then every unit runs without waiting
111 assert sum(clock.sleeps) == p.waited_busy and p.units == 4
112
113
114 @pytest.mark.asyncio
115 async def test_p3_a_failed_look_never_fails_the_analysis() -> None:
116 clock = Clock()
117 p = DbPacer(busy_threshold=2, sleep=clock.sleep, clock=clock)
118 async with p.unit(FakeSession(fail=True)): # type: ignore[arg-type]
119 pass
120 assert clock.sleeps == [] and p.units == 1
121
122
123 # ---------------------------------------------------------------------------
124 # P4 — through the real code path (private test database)
125 # ---------------------------------------------------------------------------
126
127 @pytest.mark.asyncio
128 async def test_p4_the_build_is_paced_unit_by_unit_and_unpaced_is_unchanged(db_session, monkeypatch) -> None:
129 from musehub.services import musehub_symbol_indexer as ix
130 from tests.test_intel_code_correctness import standard_dag
131 from tests.factories import create_repo
132
133 dag = standard_dag()
134 repo = await create_repo(db_session, owner="gabriel", slug="p4-paced")
135 await dag.install(db_session, repo.repo_id, ["c0", "c1", "c2", "m", "c3"])
136 await db_session.commit()
137 rid, head = repo.repo_id, dag.commits_id("c3")
138
139 monkeypatch.setattr(ix, "_COMMIT_BATCH", 2) # 5 commits -> 3 ingest units
140 units: list[int] = []
141 real_unit = DbPacer.unit
142
143 def counting_unit(self, session):
144 units.append(1)
145 return real_unit(self, session)
146
147 monkeypatch.setattr(DbPacer, "unit", counting_unit)
148 monkeypatch.setenv("MUSEHUB_INTEL_DB_DUTY", "0.9") # tiny real sleeps
149 monkeypatch.setenv("MUSEHUB_INTEL_DB_BUSY_THRESHOLD", "100") # exercise the real pg_stat_activity look; never busy
150 monkeypatch.setenv("MUSEHUB_INTEL_DB_POLL_SECONDS", "0.05")
151 paced = await ix.build_symbol_index(db_session, rid, head)
152 assert paced.outcome == "computed" and len(units) >= 3 + 3 # ingest units + at least one each for symbol intel, vitals and hash rows
153 await db_session.rollback()
154
155 units.clear()
156 monkeypatch.delenv("MUSEHUB_INTEL_DB_DUTY")
157 monkeypatch.delenv("MUSEHUB_INTEL_DB_BUSY_THRESHOLD")
158 plain = await ix.build_symbol_index(db_session, rid, head)
159 assert plain.outcome == "computed" and sorted(plain) == sorted(paced) # identical results, paced or not
160
161
162 def test_p5_batch_caps_only_ever_shrink_a_batch() -> None:
163 from musehub.services.musehub_intel_pacing import commit_batch, row_batch
164
165 assert row_batch(3000, {}) == 3000 and commit_batch(25, {}) == 25
166 assert row_batch(3000, {"MUSEHUB_INTEL_DB_ROW_BATCH": "250"}) == 250
167 assert row_batch(100, {"MUSEHUB_INTEL_DB_ROW_BATCH": "250"}) == 100 # never grows a batch past what the driver's parameter limit allows
168 assert row_batch(3000, {"MUSEHUB_INTEL_DB_ROW_BATCH": "garbage"}) == 3000
169 assert row_batch(3000, {"MUSEHUB_INTEL_DB_ROW_BATCH": "0"}) == 3000 and row_batch(3000, {"MUSEHUB_INTEL_DB_ROW_BATCH": "-5"}) == 3000
170 assert commit_batch(25, {"MUSEHUB_INTEL_DB_COMMIT_BATCH": "5"}) == 5
171
172
173 @pytest.mark.asyncio
174 async def test_p5_small_batches_give_identical_results(db_session, monkeypatch) -> None:
175 from musehub.services import musehub_symbol_indexer as ix
176 from tests.factories import create_repo
177 from tests.test_intel_code_correctness import standard_dag
178
179 dag = standard_dag()
180 repo = await create_repo(db_session, owner="gabriel", slug="p5-small")
181 await dag.install(db_session, repo.repo_id, ["c0", "c1", "c2", "m", "c3"])
182 await db_session.commit()
183 rid, head = repo.repo_id, dag.commits_id("c3")
184 plain = await ix.build_symbol_index(db_session, rid, head)
185 await db_session.rollback()
186 monkeypatch.setenv("MUSEHUB_INTEL_DB_ROW_BATCH", "1")
187 monkeypatch.setenv("MUSEHUB_INTEL_DB_COMMIT_BATCH", "1")
188 small = await ix.build_symbol_index(db_session, rid, head)
189 assert small.outcome == plain.outcome == "computed" and sorted(small) == sorted(plain)
File History 2 commits
sha256:71cfef07c390e977ebb8066cbd052c3c5de869eb5395522f6768fd32c016f8ae feat(intel): small write batches and a stricter yield rule … Sonnet 5.5 patch 3 days ago
sha256:238a7da4341ade2c8d41ea9834d8e75e250647aefec02327f1e25fac55acd036 feat(intel): pace analysis database writes so a cold index … Sonnet 5.5 patch 3 days ago