test_intel_pacing.py
python
sha256:71cfef07c390e977ebb8066cbd052c3c5de869eb5395522f6768fd32c016f8ae
feat(intel): small write batches and a stricter yield rule …
Sonnet 5.5
patch
3 days ago
| 1 | """Database pacing for code-intelligence writes (production incident 2026-11-13: a cold index starved the web on a small database). |
| 2 | |
| 3 | P1 Off by default: no queries, no sleeps, zero overhead. |
| 4 | P2 Duty cycle: sleeps ``t * (1/d - 1)`` after a unit that took ``t`` (capped), never after a failed unit. |
| 5 | P3 Back-off while the web is busy: waits while other queries are active, bounded per unit and per job; a failed look never fails the analysis. |
| 6 | P4 The build is paced end to end through the real code path, and an unpaced build is unchanged. |
| 7 | """ |
| 8 | from __future__ import annotations |
| 9 | |
| 10 | import pytest |
| 11 | |
| 12 | from musehub.services.musehub_intel_pacing import MAX_DUTY_SLEEP_SECONDS, DbPacer |
| 13 | |
| 14 | |
| 15 | class Clock: |
| 16 | def __init__(self) -> None: |
| 17 | self.now = 0.0 |
| 18 | self.sleeps: list[float] = [] |
| 19 | |
| 20 | def __call__(self) -> float: |
| 21 | return self.now |
| 22 | |
| 23 | async def sleep(self, seconds: float) -> None: |
| 24 | self.sleeps.append(seconds) |
| 25 | self.now += seconds |
| 26 | |
| 27 | |
| 28 | class FakeSession: |
| 29 | """Only what the pacer touches: ``begin_nested`` (a savepoint) and ``execute`` returning the number of other active queries.""" |
| 30 | |
| 31 | def __init__(self, counts: list[int] | None = None, fail: bool = False) -> None: |
| 32 | self.counts, self.fail, self.looks = list(counts or []), fail, 0 |
| 33 | |
| 34 | def begin_nested(self): |
| 35 | class _Ctx: |
| 36 | async def __aenter__(self_inner): return self_inner |
| 37 | async def __aexit__(self_inner, *a): return False |
| 38 | return _Ctx() |
| 39 | |
| 40 | async def execute(self, _stmt): |
| 41 | self.looks += 1 |
| 42 | if self.fail: |
| 43 | raise RuntimeError("no access to pg_stat_activity") |
| 44 | |
| 45 | class _R: |
| 46 | def __init__(self, n): self.n = n |
| 47 | def scalar_one(self): return self.n |
| 48 | return _R(self.counts.pop(0) if self.counts else 0) |
| 49 | |
| 50 | |
| 51 | def test_p1_off_by_default_and_from_empty_environment() -> None: |
| 52 | assert DbPacer.from_environ({}).enabled is False |
| 53 | assert DbPacer().enabled is False |
| 54 | assert DbPacer.from_environ({"MUSEHUB_INTEL_DB_DUTY": "garbage"}).enabled is False |
| 55 | assert DbPacer.from_environ({"MUSEHUB_INTEL_DB_DUTY": "1"}).enabled is False |
| 56 | assert DbPacer.from_environ({"MUSEHUB_INTEL_DB_DUTY": "0"}).enabled is False # nonsense (would sleep forever): ignored |
| 57 | assert DbPacer.from_environ({"MUSEHUB_INTEL_DB_DUTY": "0.4"}).enabled is True |
| 58 | assert DbPacer.from_environ({"MUSEHUB_INTEL_DB_BUSY_THRESHOLD": "2"}).enabled is True |
| 59 | |
| 60 | |
| 61 | @pytest.mark.asyncio |
| 62 | async def test_p1_a_disabled_pacer_touches_nothing() -> None: |
| 63 | clock = Clock() |
| 64 | p = DbPacer(sleep=clock.sleep, clock=clock) |
| 65 | async with p.unit(None): # type: ignore[arg-type] # a disabled pacer never uses the session |
| 66 | clock.now += 5 |
| 67 | assert clock.sleeps == [] and p.units == 0 |
| 68 | |
| 69 | |
| 70 | @pytest.mark.asyncio |
| 71 | async def test_p2_duty_cycle_sleeps_the_complement_and_is_capped() -> None: |
| 72 | clock = Clock() |
| 73 | p = DbPacer(duty=0.25, sleep=clock.sleep, clock=clock) |
| 74 | async with p.unit(FakeSession()): # type: ignore[arg-type] |
| 75 | clock.now += 2.0 |
| 76 | assert clock.sleeps == [pytest.approx(6.0)] # busy 2 s of every 8 s = 25 % |
| 77 | async with p.unit(FakeSession()): # type: ignore[arg-type] |
| 78 | clock.now += 100.0 |
| 79 | assert clock.sleeps[-1] == MAX_DUTY_SLEEP_SECONDS |
| 80 | |
| 81 | |
| 82 | @pytest.mark.asyncio |
| 83 | async def test_p2_a_failed_unit_does_not_nap() -> None: |
| 84 | clock = Clock() |
| 85 | p = DbPacer(duty=0.25, sleep=clock.sleep, clock=clock) |
| 86 | with pytest.raises(ValueError): |
| 87 | async with p.unit(FakeSession()): # type: ignore[arg-type] |
| 88 | clock.now += 2.0 |
| 89 | raise ValueError("batch failed") |
| 90 | assert clock.sleeps == [] |
| 91 | |
| 92 | |
| 93 | @pytest.mark.asyncio |
| 94 | async def test_p3_it_waits_while_other_queries_are_active_then_proceeds() -> None: |
| 95 | clock = Clock() |
| 96 | s = FakeSession([5, 4, 3, 0]) |
| 97 | p = DbPacer(busy_threshold=2, poll_seconds=2.0, sleep=clock.sleep, clock=clock) |
| 98 | async with p.unit(s): # type: ignore[arg-type] |
| 99 | pass |
| 100 | assert s.looks == 4 and clock.sleeps == [2.0, 2.0, 2.0] and p.waited_busy == 6.0 |
| 101 | |
| 102 | |
| 103 | @pytest.mark.asyncio |
| 104 | async def test_p3_a_permanently_busy_database_delays_but_never_stops_the_job() -> None: |
| 105 | clock = Clock() |
| 106 | p = DbPacer(busy_threshold=2, poll_seconds=2.0, max_wait_seconds=10.0, total_wait_seconds=14.0, sleep=clock.sleep, clock=clock) |
| 107 | for _ in range(4): |
| 108 | async with p.unit(FakeSession([9] * 100)): # type: ignore[arg-type] |
| 109 | pass |
| 110 | assert p.waited_busy <= 14.0 + 2.0 # the per-job budget is spent, then every unit runs without waiting |
| 111 | assert sum(clock.sleeps) == p.waited_busy and p.units == 4 |
| 112 | |
| 113 | |
| 114 | @pytest.mark.asyncio |
| 115 | async def test_p3_a_failed_look_never_fails_the_analysis() -> None: |
| 116 | clock = Clock() |
| 117 | p = DbPacer(busy_threshold=2, sleep=clock.sleep, clock=clock) |
| 118 | async with p.unit(FakeSession(fail=True)): # type: ignore[arg-type] |
| 119 | pass |
| 120 | assert clock.sleeps == [] and p.units == 1 |
| 121 | |
| 122 | |
| 123 | # --------------------------------------------------------------------------- |
| 124 | # P4 — through the real code path (private test database) |
| 125 | # --------------------------------------------------------------------------- |
| 126 | |
| 127 | @pytest.mark.asyncio |
| 128 | async def test_p4_the_build_is_paced_unit_by_unit_and_unpaced_is_unchanged(db_session, monkeypatch) -> None: |
| 129 | from musehub.services import musehub_symbol_indexer as ix |
| 130 | from tests.test_intel_code_correctness import standard_dag |
| 131 | from tests.factories import create_repo |
| 132 | |
| 133 | dag = standard_dag() |
| 134 | repo = await create_repo(db_session, owner="gabriel", slug="p4-paced") |
| 135 | await dag.install(db_session, repo.repo_id, ["c0", "c1", "c2", "m", "c3"]) |
| 136 | await db_session.commit() |
| 137 | rid, head = repo.repo_id, dag.commits_id("c3") |
| 138 | |
| 139 | monkeypatch.setattr(ix, "_COMMIT_BATCH", 2) # 5 commits -> 3 ingest units |
| 140 | units: list[int] = [] |
| 141 | real_unit = DbPacer.unit |
| 142 | |
| 143 | def counting_unit(self, session): |
| 144 | units.append(1) |
| 145 | return real_unit(self, session) |
| 146 | |
| 147 | monkeypatch.setattr(DbPacer, "unit", counting_unit) |
| 148 | monkeypatch.setenv("MUSEHUB_INTEL_DB_DUTY", "0.9") # tiny real sleeps |
| 149 | monkeypatch.setenv("MUSEHUB_INTEL_DB_BUSY_THRESHOLD", "100") # exercise the real pg_stat_activity look; never busy |
| 150 | monkeypatch.setenv("MUSEHUB_INTEL_DB_POLL_SECONDS", "0.05") |
| 151 | paced = await ix.build_symbol_index(db_session, rid, head) |
| 152 | assert paced.outcome == "computed" and len(units) >= 3 + 3 # ingest units + at least one each for symbol intel, vitals and hash rows |
| 153 | await db_session.rollback() |
| 154 | |
| 155 | units.clear() |
| 156 | monkeypatch.delenv("MUSEHUB_INTEL_DB_DUTY") |
| 157 | monkeypatch.delenv("MUSEHUB_INTEL_DB_BUSY_THRESHOLD") |
| 158 | plain = await ix.build_symbol_index(db_session, rid, head) |
| 159 | assert plain.outcome == "computed" and sorted(plain) == sorted(paced) # identical results, paced or not |
| 160 | |
| 161 | |
| 162 | def test_p5_batch_caps_only_ever_shrink_a_batch() -> None: |
| 163 | from musehub.services.musehub_intel_pacing import commit_batch, row_batch |
| 164 | |
| 165 | assert row_batch(3000, {}) == 3000 and commit_batch(25, {}) == 25 |
| 166 | assert row_batch(3000, {"MUSEHUB_INTEL_DB_ROW_BATCH": "250"}) == 250 |
| 167 | assert row_batch(100, {"MUSEHUB_INTEL_DB_ROW_BATCH": "250"}) == 100 # never grows a batch past what the driver's parameter limit allows |
| 168 | assert row_batch(3000, {"MUSEHUB_INTEL_DB_ROW_BATCH": "garbage"}) == 3000 |
| 169 | assert row_batch(3000, {"MUSEHUB_INTEL_DB_ROW_BATCH": "0"}) == 3000 and row_batch(3000, {"MUSEHUB_INTEL_DB_ROW_BATCH": "-5"}) == 3000 |
| 170 | assert commit_batch(25, {"MUSEHUB_INTEL_DB_COMMIT_BATCH": "5"}) == 5 |
| 171 | |
| 172 | |
| 173 | @pytest.mark.asyncio |
| 174 | async def test_p5_small_batches_give_identical_results(db_session, monkeypatch) -> None: |
| 175 | from musehub.services import musehub_symbol_indexer as ix |
| 176 | from tests.factories import create_repo |
| 177 | from tests.test_intel_code_correctness import standard_dag |
| 178 | |
| 179 | dag = standard_dag() |
| 180 | repo = await create_repo(db_session, owner="gabriel", slug="p5-small") |
| 181 | await dag.install(db_session, repo.repo_id, ["c0", "c1", "c2", "m", "c3"]) |
| 182 | await db_session.commit() |
| 183 | rid, head = repo.repo_id, dag.commits_id("c3") |
| 184 | plain = await ix.build_symbol_index(db_session, rid, head) |
| 185 | await db_session.rollback() |
| 186 | monkeypatch.setenv("MUSEHUB_INTEL_DB_ROW_BATCH", "1") |
| 187 | monkeypatch.setenv("MUSEHUB_INTEL_DB_COMMIT_BATCH", "1") |
| 188 | small = await ix.build_symbol_index(db_session, rid, head) |
| 189 | assert small.outcome == plain.outcome == "computed" and sorted(small) == sorted(plain) |
File History
2 commits
sha256:71cfef07c390e977ebb8066cbd052c3c5de869eb5395522f6768fd32c016f8ae
feat(intel): small write batches and a stricter yield rule …
Sonnet 5.5
patch
3 days ago
sha256:238a7da4341ade2c8d41ea9834d8e75e250647aefec02327f1e25fac55acd036
feat(intel): pace analysis database writes so a cold index …
Sonnet 5.5
patch
3 days ago