gabriel / musehub public
test_intel_universal.py python
767 lines 40.5 KB
Raw
sha256:71cfef07c390e977ebb8066cbd052c3c5de869eb5395522f6768fd32c016f8ae feat(intel): small write batches and a stricter yield rule … Sonnet 5.5 patch 3 days ago
1 """Always-on code intelligence for EVERY repository: universal coverage gate, prebuild priority, reconciler, status.
2
3 Run (private test database, see tests/conftest.py):: python -P -m pytest tests/test_intel_universal.py
4
5 U1 ``MUSEHUB_INTEL_ALL_REPOS=1`` makes every repository eligible — but only with a known deployment identity matching ``MUSE_ENV``; ``local`` still needs a local database;
6 any other flag value, an unknown/mismatched identity, or a wildcard in the allowlist refuses.
7 S1 Prebuilds are claimed before intel jobs, whatever their age; an aged intel job is claimed ahead at most once per slice (no starvation).
8 S2 ``prebuild_waiting`` mirrors the claim eligibility; ``yield_job`` is fenced, is not an attempt, and counts yields.
9 S3 A running intel execution ends with ``reason == "yield"`` as soon as the yield check says so (the body is cancelled, nothing else is written).
10 R1 The reconciler enqueues exactly the missing providers (from the registry), once; current, queued, not-applicable and terminally failed work is left alone.
11 R2 Failures are retried a bounded number of times with back-off; dependents are not enqueued behind a dead index; exhausted pending jobs are failed.
12 R3 Bounds: backlog hold, per-pass repository cap, resumable cursor; allowlist mode still only touches listed repositories.
13 T1 Status: pending-only is ``queued``, running is ``analyzing``; a non-code domain is ``not_applicable``.
14 """
15 from __future__ import annotations
16
17 import asyncio
18 import secrets
19 from datetime import datetime, timedelta, timezone
20
21 import pytest
22 from sqlalchemy import select, update
23 from sqlalchemy.ext.asyncio import AsyncSession
24
25 from musehub.db.musehub_jobs_models import MusehubBackgroundJob
26 from musehub.db.musehub_repo_models import MusehubBranch
27 from musehub.services import musehub_intel_gate as gate
28 from musehub.services import musehub_intel_reconcile as rec
29 from musehub.services import musehub_jobs as jobs
30 from musehub.services.musehub_intel_providers import job_types_for_push
31 from musehub.services.musehub_intel_status import REQUIRED_JOB_TYPES, IntelStatus, load_intel_status, required_job_types
32 from tests.factories import create_repo
33 from tests.test_intel_job_state_machine import worker_db # noqa: F401 - fixture: production engine on the private test database
34
35 REPO = "sha256:" + "a" * 64
36 LOCAL_DB = "postgresql+asyncpg://musehub:pw@postgres:5432/musehub"
37 REMOTE_DB = "postgresql+asyncpg://musehub:[email protected]:5432/musehub"
38 H = "sha256:" + "1" * 64
39
40
41 # ---------------------------------------------------------------------------
42 # U1 — the universal switch
43 # ---------------------------------------------------------------------------
44
45 @pytest.fixture
46 def env(monkeypatch):
47 from musehub.config import settings
48
49 def setup(*, deployment, muse_env="UNSET", all_repos="1", allow=None, database_url=LOCAL_DB):
50 for name, value in ((gate.DEPLOYMENT_ENV, deployment), (gate.MUSE_ENV_ENV, None if muse_env == "UNSET" else muse_env),
51 (gate.ALL_REPOS_ENV, all_repos), (gate.ALLOWLIST_ENV, allow)):
52 if value is None:
53 monkeypatch.delenv(name, raising=False)
54 else:
55 monkeypatch.setenv(name, value)
56 monkeypatch.setattr(settings, "database_url", database_url)
57 return setup
58
59
60 @pytest.mark.parametrize("deployment,muse_env", [("local", "UNSET"), ("local", "local"), ("staging", "staging"), ("production", "production")])
61 @pytest.mark.parametrize("database_url", [LOCAL_DB, REMOTE_DB])
62 def test_u1_every_repository_is_eligible_in_a_correctly_identified_deployment(env, deployment, muse_env, database_url) -> None:
63 env(deployment=deployment, muse_env=muse_env, database_url=database_url)
64 expected = deployment != "local" or database_url == LOCAL_DB # local also needs a local database
65 for repo in (REPO, "sha256:" + "b" * 64, "anything-at-all"):
66 assert gate.intel_execution_allowed(repo) is expected
67 assert gate.intel_all_repos() is True
68
69
70 @pytest.mark.parametrize("flag", [None, "", "0", "true", "yes", "on", "TRUE", " 1", "1 ", "11", "*"])
71 def test_u1_only_exactly_one_turns_it_on(env, flag) -> None:
72 env(deployment="production", muse_env="production", all_repos=flag)
73 assert gate.intel_all_repos() is False
74 assert gate.intel_execution_allowed(REPO) is False # no allowlist either
75
76
77 @pytest.mark.parametrize("deployment,muse_env", [
78 (None, "UNSET"), ("", "UNSET"), ("prod", "production"), ("Local", "UNSET"), ("production", "staging"), ("production", "UNSET"), ("production", "test"),
79 ("staging", "production"), ("local", "production"), ("local", "staging"), ("local", "weird"),
80 ])
81 def test_u1_unknown_or_mismatched_identity_refuses_whatever_the_flag_says(env, deployment, muse_env) -> None:
82 env(deployment=deployment, muse_env=muse_env)
83 assert gate.intel_all_repos() is False
84 assert gate.intel_execution_allowed(REPO) is False
85
86
87 def test_u1_a_wildcard_in_the_allowlist_is_still_not_a_wildcard(env) -> None:
88 env(deployment="production", muse_env="production", all_repos=None, allow="*")
89 assert gate.intel_execution_allowed(REPO) is False
90 env(deployment="production", muse_env="production", all_repos=None, allow=REPO)
91 assert gate.intel_execution_allowed(REPO) is True and gate.intel_execution_allowed("sha256:" + "c" * 64) is False
92
93
94 def test_u1_the_flag_is_reread_on_every_call(env, monkeypatch) -> None:
95 env(deployment="staging", muse_env="staging")
96 assert gate.intel_execution_allowed(REPO) is True
97 monkeypatch.delenv(gate.ALL_REPOS_ENV)
98 assert gate.intel_execution_allowed(REPO) is False
99
100
101 def test_u1_hosted_environments_carry_the_flag_from_secrets_sh_and_local_from_env_example() -> None:
102 from pathlib import Path
103
104 root = Path(__file__).resolve().parent.parent
105 secrets_sh = (root / "deploy" / "secrets.sh").read_text()
106 assert secrets_sh.count('INTEL_ALL_REPOS="${INTEL_ALL_REPOS_OVERRIDE:-1}"') == 2 # staging and production blocks (overridable, default on)
107 assert "MUSEHUB_INTEL_ALL_REPOS=${INTEL_ALL_REPOS:-1}" in secrets_sh
108 assert "MUSEHUB_INTEL_ALL_REPOS=1" in (root / ".env.example").read_text()
109 assert secrets_sh.count('INTEL_DB_DUTY="0.25"') == 2 and "MUSEHUB_INTEL_DB_DUTY=${INTEL_DB_DUTY:-0.25}" in secrets_sh and "MUSEHUB_INTEL_DB_BUSY_THRESHOLD=${INTEL_DB_BUSY_THRESHOLD:-1}" in secrets_sh
110 assert "MUSEHUB_INTEL_DB_ROW_BATCH=${INTEL_DB_ROW_BATCH:-250}" in secrets_sh and "MUSEHUB_INTEL_DB_COMMIT_BATCH=${INTEL_DB_COMMIT_BATCH:-5}" in secrets_sh
111
112
113 # ---------------------------------------------------------------------------
114 # DB helpers
115 # ---------------------------------------------------------------------------
116
117 _n = 0
118
119
120 async def add_job(s: AsyncSession, repo_id: str, job_type: str, *, status: str = "pending", age: int = 0, head: str = H, attempt: int = 0,
121 done_ago: int | None = None, payload: dict | None = None) -> str:
122 global _n
123 _n += 1
124 job_id = "sha256:" + f"{_n:064x}"
125 now = datetime.now(tz=timezone.utc)
126 s.add(MusehubBackgroundJob(
127 job_id=job_id, repo_id=repo_id, job_type=job_type, payload=payload or {"head": head, "branch": "main"}, status=status,
128 created_at=now - timedelta(seconds=age), attempt=attempt,
129 claimed_at=now if status == "running" else None, done_at=(now - timedelta(seconds=done_ago)) if done_ago is not None else None,
130 ))
131 await s.commit()
132 return job_id
133
134
135 async def repo_with_head(s: AsyncSession, slug: str, head: str = H, *, domain_id: str | None = None):
136 repo = await create_repo(s, owner="gabriel", slug=slug, **({"domain_id": domain_id} if domain_id else {}))
137 s.add(MusehubBranch(branch_id=secrets.token_hex(16), repo_id=repo.repo_id, name="main", head_commit_id=head))
138 await s.commit()
139 return repo
140
141
142 async def row(s: AsyncSession, job_id: str) -> MusehubBackgroundJob:
143 await s.rollback()
144 return (await s.execute(select(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == job_id))).scalar_one()
145
146
147 async def types_for(s: AsyncSession, repo_id: str, *, status: str = "pending") -> list[str]:
148 await s.rollback()
149 return sorted((await s.execute(select(MusehubBackgroundJob.job_type).where(
150 MusehubBackgroundJob.repo_id == repo_id, MusehubBackgroundJob.status == status))).scalars())
151
152
153 @pytest.fixture
154 def listed(monkeypatch):
155 """Intel allowed for every repository of a staging-identified deployment."""
156 monkeypatch.setenv("MUSEHUB_DEPLOYMENT", "staging")
157 monkeypatch.setenv("MUSE_ENV", "staging")
158 monkeypatch.setenv("MUSEHUB_INTEL_ALL_REPOS", "1")
159 monkeypatch.delenv("MUSEHUB_INTEL_WORKER_REPOS", raising=False)
160
161
162 # ---------------------------------------------------------------------------
163 # S1 / S2 — prebuild priority
164 # ---------------------------------------------------------------------------
165
166 @pytest.mark.asyncio
167 async def test_s1_a_prebuild_is_claimed_before_older_intel_work(worker_db) -> None:
168 repo = await repo_with_head(worker_db, "s1-order")
169 intel_id = await add_job(worker_db, repo.repo_id, "intel.code", age=60)
170 prebuild_id = await add_job(worker_db, repo.repo_id, "fetch.mpack.prebuild", age=1, payload={"tip_commit_ids": [H]})
171 claimed = await jobs.claim_next_job(worker_db)
172 assert claimed is not None and claimed.job_id == prebuild_id
173 await worker_db.rollback()
174 assert (await row(worker_db, intel_id)).status == "pending"
175
176
177 @pytest.mark.asyncio
178 async def test_s1_non_intel_work_keeps_its_place_ahead_of_intel(worker_db) -> None:
179 repo = await repo_with_head(worker_db, "s1-gc")
180 await add_job(worker_db, repo.repo_id, "intel.code", age=600)
181 gc_id = await add_job(worker_db, repo.repo_id, "gc", age=1, payload={})
182 claimed = await jobs.claim_next_job(worker_db)
183 assert claimed is not None and claimed.job_id == gc_id
184
185
186 @pytest.mark.asyncio
187 async def test_s1_an_aged_intel_job_goes_first_once_per_slice_so_it_cannot_starve(worker_db) -> None:
188 repo = await repo_with_head(worker_db, "s1-aged")
189 rid = repo.repo_id
190 aged = await add_job(worker_db, rid, "intel.code", age=jobs.INTEL_STARVATION_SECONDS + 60)
191 prebuild = await add_job(worker_db, rid, "fetch.mpack.prebuild", age=1, payload={"tip_commit_ids": [H]})
192 claimed = await jobs.claim_next_job(worker_db)
193 assert claimed is not None and claimed.job_id == aged # nothing ran lately: the aged analysis takes its turn
194 # ... but only one per slice: with an intel completion inside the slice, the prebuild wins again
195 await worker_db.rollback()
196 await worker_db.execute(update(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == aged).values(status="pending", attempt=0))
197 await worker_db.commit()
198 await add_job(worker_db, rid, "intel.code.dead", status="done", done_ago=5)
199 claimed = await jobs.claim_next_job(worker_db)
200 assert claimed is not None and claimed.job_id == prebuild
201
202
203 @pytest.mark.asyncio
204 async def test_s1_a_young_intel_job_does_not_jump_the_queue(worker_db) -> None:
205 repo = await repo_with_head(worker_db, "s1-young")
206 await add_job(worker_db, repo.repo_id, "intel.code", age=jobs.INTEL_STARVATION_SECONDS - 120)
207 prebuild = await add_job(worker_db, repo.repo_id, "fetch.mpack.prebuild", age=1, payload={"tip_commit_ids": [H]})
208 claimed = await jobs.claim_next_job(worker_db)
209 assert claimed is not None and claimed.job_id == prebuild
210
211
212 @pytest.mark.asyncio
213 async def test_s2_prebuild_waiting_mirrors_the_claim_eligibility(worker_db) -> None:
214 repo = await repo_with_head(worker_db, "s2-wait")
215 assert await jobs.prebuild_waiting(worker_db) is False
216 index = await add_job(worker_db, repo.repo_id, "mpack.index", payload={"mpack_key": "k"})
217 pre = await add_job(worker_db, repo.repo_id, "fetch.mpack.prebuild", payload={"tip_commit_ids": [H]})
218 assert await jobs.prebuild_waiting(worker_db) is False # blocked by its own repository's index job, not by intel
219 await worker_db.execute(update(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == index).values(status="done"))
220 await worker_db.commit()
221 assert await jobs.prebuild_waiting(worker_db) is True
222 await worker_db.execute(update(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == pre).values(attempt=jobs._MAX_ATTEMPTS))
223 await worker_db.commit()
224 assert await jobs.prebuild_waiting(worker_db) is False # exhausted: never claimable
225
226
227 @pytest.mark.asyncio
228 async def test_s2_yield_job_is_fenced_is_not_an_attempt_and_counts_yields(worker_db) -> None:
229 repo = await repo_with_head(worker_db, "s2-yield")
230 jid = await add_job(worker_db, repo.repo_id, "intel.code")
231 claimed = await jobs.claim_next_job(worker_db)
232 assert claimed is not None and claimed.attempt == 1
233 await worker_db.commit()
234 assert await jobs.yield_job(worker_db, jid, attempt=7) is False # a superseded execution cannot yield
235 await worker_db.rollback()
236 assert await jobs.yield_job(worker_db, jid, attempt=1) is True
237 await worker_db.commit()
238 back = await row(worker_db, jid)
239 assert back.status == "pending" and back.attempt == 0 and back.claimed_at is None
240 assert jobs.yields_of(back.payload) == 1 and back.payload["head"] == H and back.created_at is not None
241 assert await jobs.yield_job(worker_db, jid, attempt=1) is False # no longer running
242
243
244 @pytest.mark.asyncio
245 async def test_s3_a_running_execution_ends_with_reason_yield_when_asked(monkeypatch) -> None:
246 import musehub.worker as worker
247
248 async def renewed(job_id: str, attempt: int) -> bool:
249 return True
250
251 monkeypatch.setattr(worker, "_renew_claim_once", renewed)
252 monkeypatch.setattr(worker, "_YIELD_POLL_SECONDS", 0.01)
253 started = asyncio.Event()
254 cancelled = asyncio.Event()
255
256 async def body() -> str:
257 started.set()
258 try:
259 await asyncio.sleep(30)
260 except asyncio.CancelledError:
261 cancelled.set()
262 raise
263 return "finished"
264
265 calls = 0
266
267 async def should_yield() -> bool:
268 nonlocal calls
269 calls += 1
270 return started.is_set() and calls >= 3
271
272 ended, result = await worker._run_leased("j", 1, body(), cancel_on_end=True, yield_when=should_yield)
273 assert result is None and ended.reason == "yield" and cancelled.is_set()
274
275
276 @pytest.mark.asyncio
277 async def test_s3_a_job_that_is_not_asked_to_yield_runs_to_completion(monkeypatch) -> None:
278 import musehub.worker as worker
279
280 async def renewed(job_id: str, attempt: int) -> bool:
281 return True
282
283 monkeypatch.setattr(worker, "_renew_claim_once", renewed)
284 monkeypatch.setattr(worker, "_YIELD_POLL_SECONDS", 0.01)
285
286 async def body() -> str:
287 await asyncio.sleep(0.1)
288 return "finished"
289
290 async def never() -> bool:
291 return False
292
293 ended, result = await worker._run_leased("j", 1, body(), cancel_on_end=True, yield_when=never)
294 assert result == "finished" and ended.reason is None
295
296
297 # ---------------------------------------------------------------------------
298 # R1 / R2 / R3 — reconciler
299 # ---------------------------------------------------------------------------
300
301 def _st(**states: str) -> IntelStatus:
302 providers = {t: states.get(t, states.get("all", "current")) for t in REQUIRED_JOB_TYPES}
303 return IntelStatus(head=H, state="outdated", providers=providers)
304
305
306 def test_r1_missing_providers_returns_exactly_the_work_that_is_not_current_or_in_flight() -> None:
307 now = datetime.now(tz=timezone.utc)
308 needed, terminal = rec.missing_providers("r", H, _st(**{"intel.code.type": "never", "intel.code.dead": "outdated", "intel.code.gravity": "not_run",
309 "intel.code.clones": "analyzing", "intel.code.velocity": "queued"}), {}, now)
310 assert sorted(needed) == ["intel.code.dead", "intel.code.gravity", "intel.code.type"] and terminal == 0
311 assert rec.missing_providers("r", H, _st(all="current"), {}, now) == ([], 0)
312
313
314 def test_r2_retry_is_bounded_with_exponential_backoff() -> None:
315 now = datetime.now(tz=timezone.utc)
316 assert rec.retry_eligible(0, None, now) is True
317 assert rec.retry_eligible(1, now - timedelta(seconds=rec.RETRY_BACKOFF_SECONDS - 5), now) is False
318 assert rec.retry_eligible(1, now - timedelta(seconds=rec.RETRY_BACKOFF_SECONDS + 5), now) is True
319 assert rec.retry_eligible(2, now - timedelta(seconds=rec.RETRY_BACKOFF_SECONDS + 5), now) is False # the back-off doubled
320 assert rec.retry_eligible(2, now - timedelta(seconds=2 * rec.RETRY_BACKOFF_SECONDS + 5), now) is True
321 assert rec.retry_eligible(rec.FAILURE_RETRY_LIMIT, now - timedelta(days=30), now) is False # terminal
322
323
324 def test_r2_a_dead_index_holds_back_its_dependents() -> None:
325 now = datetime.now(tz=timezone.utc)
326 hist = {("r", "intel.code", H): rec.HeadHistory(failures=rec.FAILURE_RETRY_LIMIT, newest_failure=now, newest_status="failed")}
327 needed, terminal = rec.missing_providers("r", H, _st(**{"intel.code": "failed", "intel.code.type": "never"}), hist, now)
328 assert needed == [] and terminal == 1
329 needed, terminal = rec.missing_providers("r", H, _st(**{"intel.code": "failed", "intel.code.type": "never"}), {}, now)
330 assert "intel.code" in needed # a retryable index failure: the index goes first (dependents are ordered behind it by the claim barrier)
331
332
333 @pytest.mark.asyncio
334 async def test_r1_a_repository_with_no_intel_gets_the_whole_registry_set_exactly_once(worker_db, listed) -> None:
335 repo = await repo_with_head(worker_db, "r1-new")
336 report = await rec.reconcile_pass(worker_db)
337 await worker_db.commit()
338 expected = sorted(t for t in job_types_for_push(None) if t.startswith("intel.code"))
339 assert report.enqueued_repos == 1 and report.enqueued_providers == len(expected) >= 14
340 assert await types_for(worker_db, repo.repo_id) == expected
341 again = await rec.reconcile_pass(worker_db) # now queued: nothing to add
342 await worker_db.commit()
343 assert again.enqueued_providers == 0 and await types_for(worker_db, repo.repo_id) == expected
344
345
346 @pytest.mark.asyncio
347 async def test_r1_only_the_missing_providers_are_enqueued(worker_db, listed) -> None:
348 from musehub.db.musehub_intel_models import MusehubIntelResult
349 from musehub.services.musehub_intel_providers import published_types
350
351 repo = await repo_with_head(worker_db, "r1-partial")
352 for t in REQUIRED_JOB_TYPES:
353 if t in ("intel.code.dead", "intel.code.type"):
354 continue
355 for it in published_types(t):
356 worker_db.add(MusehubIntelResult(result_id=f"{repo.repo_id}|{it}", repo_id=repo.repo_id, intel_type=it, domain="code", ref=H, data_json="{}", schema_version=1, computed_at=datetime.now(tz=timezone.utc)))
357 await worker_db.commit()
358 await rec.reconcile_pass(worker_db)
359 await worker_db.commit()
360 assert await types_for(worker_db, repo.repo_id) == ["intel.code.dead", "intel.code.type"]
361
362
363 @pytest.mark.asyncio
364 async def test_r1_a_repository_without_a_default_head_and_a_non_code_domain_are_left_alone(worker_db, listed) -> None:
365 empty = await create_repo(worker_db, owner="gabriel", slug="r1-empty")
366 await worker_db.commit()
367 report = await rec.reconcile_pass(worker_db)
368 await worker_db.commit()
369 assert report.enqueued_providers == 0 and await types_for(worker_db, empty.repo_id) == []
370 assert required_job_types("midi") == ()
371 status = (await load_intel_status(worker_db, [empty.repo_id]))[empty.repo_id]
372 assert status.head is None and status.state == "not_analyzed"
373
374
375 @pytest.mark.asyncio
376 async def test_t1_status_distinguishes_queued_from_analyzing_and_not_applicable(worker_db, listed) -> None:
377 repo = await repo_with_head(worker_db, "t1-status")
378 await add_job(worker_db, repo.repo_id, "intel.code.dead", status="pending")
379 st = (await load_intel_status(worker_db, [repo.repo_id]))[repo.repo_id]
380 assert st.providers["intel.code.dead"] == "queued" and st.state == "queued" and st.state_label == "Queued"
381 await add_job(worker_db, repo.repo_id, "intel.code.type", status="running")
382 st = (await load_intel_status(worker_db, [repo.repo_id]))[repo.repo_id]
383 assert st.providers["intel.code.type"] == "analyzing" and st.state == "analyzing"
384 from musehub.services.musehub_intel_status import repo_state
385
386 assert repo_state({}) == "not_applicable"
387
388
389 @pytest.mark.asyncio
390 async def test_r2_failed_work_is_retried_in_bounds_and_then_stays_terminal(worker_db, listed) -> None:
391 repo = await repo_with_head(worker_db, "r2-fail")
392 from musehub.db.musehub_intel_models import MusehubIntelResult
393 from musehub.services.musehub_intel_providers import published_types
394
395 for t in REQUIRED_JOB_TYPES: # everything current except one provider
396 if t != "intel.code.type":
397 for it in published_types(t):
398 worker_db.add(MusehubIntelResult(result_id=f"{repo.repo_id}|{it}", repo_id=repo.repo_id, intel_type=it, domain="code", ref=H, data_json="{}", schema_version=1, computed_at=datetime.now(tz=timezone.utc)))
399 await worker_db.commit()
400 await add_job(worker_db, repo.repo_id, "intel.code.type", status="failed", age=10, done_ago=1)
401 report = await rec.reconcile_pass(worker_db) # failed one second ago: inside the back-off
402 await worker_db.commit()
403 assert report.enqueued_providers == 0 and await types_for(worker_db, repo.repo_id) == []
404 await worker_db.execute(update(MusehubBackgroundJob).where(MusehubBackgroundJob.repo_id == repo.repo_id).values(done_at=datetime.now(tz=timezone.utc) - timedelta(hours=1)))
405 await worker_db.commit()
406 report = await rec.reconcile_pass(worker_db)
407 await worker_db.commit()
408 assert report.enqueued_providers == 1 and await types_for(worker_db, repo.repo_id) == ["intel.code.type"]
409 # after FAILURE_RETRY_LIMIT failures it is terminal: reported, never re-enqueued
410 await worker_db.execute(update(MusehubBackgroundJob).where(MusehubBackgroundJob.repo_id == repo.repo_id, MusehubBackgroundJob.status == "pending").values(status="failed", done_at=datetime.now(tz=timezone.utc) - timedelta(days=2)))
411 for _ in range(rec.FAILURE_RETRY_LIMIT):
412 await add_job(worker_db, repo.repo_id, "intel.code.type", status="failed", done_ago=86400 * 3)
413 report = await rec.reconcile_pass(worker_db)
414 await worker_db.commit()
415 assert report.enqueued_providers == 0 and report.terminal_failures == 1 and await types_for(worker_db, repo.repo_id) == []
416
417
418 @pytest.mark.asyncio
419 async def test_r2_a_pending_job_with_no_attempts_left_is_failed_not_left_pending_forever(worker_db, listed) -> None:
420 repo = await repo_with_head(worker_db, "r2-exhausted")
421 jid = await add_job(worker_db, repo.repo_id, "intel.code.dead", attempt=jobs._MAX_ATTEMPTS)
422 report = await rec.reconcile_pass(worker_db)
423 await worker_db.commit()
424 assert report.exhausted_failed == 1
425 got = await row(worker_db, jid)
426 assert got.status == "failed" and "exhausted" in (got.error or "")
427
428
429 @pytest.mark.asyncio
430 async def test_r3_a_full_backlog_holds_the_pass(worker_db, listed, monkeypatch) -> None:
431 repo = await repo_with_head(worker_db, "r3-backlog")
432 monkeypatch.setattr(rec, "MAX_BACKLOG", 2)
433 other = await repo_with_head(worker_db, "r3-backlog-other", head="sha256:" + "2" * 64)
434 for t in ("intel.code.dead", "intel.code.type"):
435 await add_job(worker_db, other.repo_id, t, head="sha256:" + "2" * 64)
436 report = await rec.reconcile_pass(worker_db)
437 await worker_db.commit()
438 assert report.held_by_backlog is True and report.enqueued_providers == 0 and await types_for(worker_db, repo.repo_id) == []
439
440
441 @pytest.mark.asyncio
442 async def test_r3_the_pass_is_capped_and_resumable(worker_db, listed, monkeypatch) -> None:
443 monkeypatch.setattr(rec, "MAX_REPOS_PER_PASS", 3)
444 monkeypatch.setattr(rec, "MAX_BACKLOG", 10_000)
445 repos = [await repo_with_head(worker_db, f"r3-cap-{i:02d}", head="sha256:" + f"{i + 3:064x}") for i in range(7)]
446 mine = {r.repo_id for r in repos}
447 seen: set[str] = set()
448 cursor = None
449 for _ in range(8):
450 report = await rec.reconcile_pass(worker_db, cursor)
451 await worker_db.commit()
452 cursor = report.cursor
453 assert report.enqueued_repos <= 3
454 now_seen = {rid for rid in mine if await types_for(worker_db, rid)}
455 assert seen <= now_seen
456 seen = now_seen
457 if seen == mine:
458 break
459 assert seen == mine # every repository was reached, three at a time
460
461
462 @pytest.mark.asyncio
463 async def test_r3_allowlist_mode_reconciles_only_listed_repositories(worker_db, monkeypatch) -> None:
464 a = await repo_with_head(worker_db, "r3-list-a")
465 b = await repo_with_head(worker_db, "r3-list-b", head="sha256:" + "2" * 64)
466 monkeypatch.setenv("MUSEHUB_DEPLOYMENT", "staging")
467 monkeypatch.setenv("MUSE_ENV", "staging")
468 monkeypatch.delenv("MUSEHUB_INTEL_ALL_REPOS", raising=False)
469 monkeypatch.setenv("MUSEHUB_INTEL_WORKER_REPOS", a.repo_id)
470 await rec.reconcile_pass(worker_db)
471 await worker_db.commit()
472 assert await types_for(worker_db, a.repo_id) and not await types_for(worker_db, b.repo_id)
473
474
475 @pytest.mark.asyncio
476 async def test_r3_nothing_is_enqueued_in_an_unidentified_deployment(worker_db, monkeypatch) -> None:
477 repo = await repo_with_head(worker_db, "r3-noid")
478 monkeypatch.setenv("MUSEHUB_INTEL_ALL_REPOS", "1")
479 monkeypatch.setenv("MUSEHUB_DEPLOYMENT", "production")
480 monkeypatch.setenv("MUSE_ENV", "staging") # mismatch
481 report = await rec.reconcile_pass(worker_db)
482 await worker_db.commit()
483 assert report.scanned == 0 and await types_for(worker_db, repo.repo_id) == []
484
485
486 @pytest.mark.asyncio
487 async def test_f1_the_fleet_summary_reports_aggregates_only(worker_db, listed) -> None:
488 await repo_with_head(worker_db, "f1-fleet")
489 await rec.reconcile_pass(worker_db)
490 await worker_db.commit()
491 summary = await rec.fleet_summary(worker_db)
492 assert summary["with_default_head"] >= 1 and summary["updating"] >= 1
493 assert summary["queue"]["pending"] >= 14 and set(summary["queue"]) == {"pending", "running", "oldest_pending_age_s"}
494 text = repr(summary)
495 assert "f1-fleet" not in text and "gabriel" not in text # no repository names anywhere
496
497
498 # ---------------------------------------------------------------------------
499 # M1 — the full-history load shares repeated values (memory), without changing a single entry
500 # ---------------------------------------------------------------------------
501
502 @pytest.mark.asyncio
503 async def test_m1_full_history_entries_share_repeated_strings_and_are_unchanged(worker_db) -> None:
504 from musehub.db.musehub_intel_models import MusehubSymbolHistoryEntry
505 from musehub.services.musehub_symbol_indexer import _load_full_history_from_db
506
507 repo = await repo_with_head(worker_db, "m1-history")
508 rid = repo.repo_id
509 ts = datetime(2026, 1, 2, 3, 4, 5, tzinfo=timezone.utc)
510 for c in ("c1", "c2"):
511 for a in ("m.py::f", "m.py::g", "m.py::h"):
512 worker_db.add(MusehubSymbolHistoryEntry(
513 repo_id=rid, address=a, commit_id="sha256:" + (c * 32), committed_at=ts, author="gabriel", op="insert", op_payload="{}", content_id="x" * 40, message=None, commit_branch="main",
514 ))
515 await worker_db.commit()
516 history = await _load_full_history_from_db(worker_db, rid, {"sha256:" + "c1" * 32: 0, "sha256:" + "c2" * 32: 1})
517 f, g = history["m.py::f"], history["m.py::g"]
518 assert [e["seq"] for e in f] == [0, 1] and f[0]["committed_at"] == ts.isoformat() and f[0]["author"] == "gabriel" and f[0]["op"] == "insert"
519 assert f[0]["commit_id"] == "sha256:" + "c1" * 32 and f[0]["content_id"] == "x" * 40
520 assert f[0]["commit_id"] is g[0]["commit_id"] and f[0]["committed_at"] is g[0]["committed_at"] and f[0]["author"] is g[1]["author"] # one object per value
521
522
523 # ---------------------------------------------------------------------------
524 # Review fixes (Tier A, 545266da): B1/B2 settled heads, M1 yield dedupe, M2 aged jobs, L1 cursor on hold, R1 fleet endpoint
525 # ---------------------------------------------------------------------------
526
527 async def _skipped(s: AsyncSession, repo_id: str, job_type: str, reason: str, head: str = H, age: int = 0) -> None:
528 jid = await add_job(s, repo_id, job_type, status="skipped", head=head, age=age, done_ago=0)
529 await s.execute(update(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == jid).values(result={"executed": False, "reason": reason}))
530 await s.commit()
531
532
533 @pytest.mark.asyncio
534 async def test_b1_a_head_whose_jobs_were_skipped_as_superseded_is_settled_not_re_enqueued_forever(worker_db, listed) -> None:
535 repo = await repo_with_head(worker_db, "b1-superseded")
536 rid = repo.repo_id
537 for t in REQUIRED_JOB_TYPES:
538 await _skipped(worker_db, rid, t, "superseded: a newer revision of this repository is already analysed")
539 for _ in range(4):
540 report = await rec.reconcile_pass(worker_db)
541 await worker_db.commit()
542 assert report.enqueued_providers == 0
543 assert await types_for(worker_db, rid) == []
544
545
546 @pytest.mark.asyncio
547 async def test_b1_a_not_executed_skip_is_re_enqueued_but_only_a_bounded_number_of_times(worker_db, listed) -> None:
548 repo = await repo_with_head(worker_db, "b1-notexec")
549 rid = repo.repo_id
550 for t in REQUIRED_JOB_TYPES:
551 await _skipped(worker_db, rid, t, f"{t} is not executed by this worker")
552 first = await rec.reconcile_pass(worker_db) # the deployment can run it now (backfill): enqueue
553 await worker_db.commit()
554 assert first.enqueued_providers == len(REQUIRED_JOB_TYPES)
555 await worker_db.execute(update(MusehubBackgroundJob).where(MusehubBackgroundJob.repo_id == rid, MusehubBackgroundJob.status == "pending").values(status="skipped", result={"executed": False, "reason": "x is not executed by this worker"}))
556 await worker_db.commit()
557 second = await rec.reconcile_pass(worker_db)
558 await worker_db.commit()
559 assert second.enqueued_providers == 0 # NOT_EXECUTED_RETRY_LIMIT reached: stops asking
560
561
562 @pytest.mark.asyncio
563 async def test_b2_a_repository_whose_index_finished_empty_is_not_re_indexed_every_pass(worker_db, listed) -> None:
564 repo = await repo_with_head(worker_db, "b2-empty")
565 rid = repo.repo_id
566 for t in REQUIRED_JOB_TYPES: # every provider ran for this head and reached a terminal verdict; nothing was published (no symbol ops anywhere)
567 jid = await add_job(worker_db, rid, t, status="done", done_ago=60)
568 await worker_db.execute(update(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == jid).values(result={"executed": True, "outcome": "empty", "intel_types": [], "ref": H}))
569 await worker_db.commit()
570 for _ in range(4):
571 report = await rec.reconcile_pass(worker_db)
572 await worker_db.commit()
573 assert report.enqueued_providers == 0
574 assert await types_for(worker_db, rid) == []
575
576
577 def test_m1_may_yield_rules() -> None:
578 now = datetime.now(tz=timezone.utc)
579 young = now - timedelta(seconds=10)
580 assert jobs.intel_may_yield({"head": H}, young, now) is True
581 assert jobs.intel_may_yield({"head": H, "_yields": jobs.MAX_INTEL_YIELDS}, young, now) is False
582 assert jobs.intel_may_yield({"head": H}, now - timedelta(seconds=jobs.INTEL_STARVATION_SECONDS + 5), now) is False # M2: an aged job keeps its turn
583 assert jobs.intel_may_yield(None, None, now) is True
584
585
586 @pytest.mark.asyncio
587 async def test_m1_a_yielded_job_is_not_re_queued_when_a_pending_run_of_the_same_head_exists(worker_db) -> None:
588 repo = await repo_with_head(worker_db, "m1-dup")
589 rid = repo.repo_id
590 running = await add_job(worker_db, rid, "intel.code", status="running", attempt=1)
591 pending = await add_job(worker_db, rid, "intel.code") # e.g. queued by a push meanwhile
592 assert await jobs.yield_job(worker_db, running, attempt=1) is True
593 await worker_db.commit()
594 got = await row(worker_db, running)
595 assert got.status == "skipped" and "covers" in got.result["reason"]
596 assert (await row(worker_db, pending)).status == "pending"
597 assert await types_for(worker_db, rid) == ["intel.code"] # exactly one pending run of the head
598
599
600 @pytest.mark.asyncio
601 async def test_m1_a_push_retarget_keeps_the_yield_count(worker_db) -> None:
602 repo = await repo_with_head(worker_db, "m1-retarget")
603 rid = repo.repo_id
604 jid = await add_job(worker_db, rid, "intel.code.dead", payload={"head": "unknown-old", "branch": "main", "_yields": 2})
605 await jobs.enqueue_push_intel(worker_db, rid, H, domain_id=repo.domain_id, branch="main", only_types=frozenset({"intel.code.dead"}))
606 await worker_db.commit()
607 got = await row(worker_db, jid)
608 assert got.payload["head"] == H and jobs.yields_of(got.payload) == 2
609
610
611 @pytest.mark.asyncio
612 async def test_l1_a_held_pass_does_not_advance_the_cursor(worker_db, listed, monkeypatch) -> None:
613 await repo_with_head(worker_db, "l1-hold")
614 monkeypatch.setattr(rec, "MAX_BACKLOG", 0)
615 report = await rec.reconcile_pass(worker_db, "sha256:" + "0" * 64)
616 assert report.held_by_backlog is True and report.cursor == "sha256:" + "0" * 64
617
618
619 @pytest.mark.asyncio
620 async def test_r1_the_anonymous_fleet_endpoint_is_aggregate_only_and_cached(client, worker_db, listed, monkeypatch) -> None:
621 import musehub.main as main
622
623 monkeypatch.setitem(main._FLEET_CACHE, "body", None)
624 await repo_with_head(worker_db, "r1-fleet-private")
625 r = await client.get("/api/intel/fleet")
626 assert r.status_code == 200
627 body = r.json()
628 assert set(body) == {"repositories", "with_default_head", "by_state", "fully_current", "updating", "failed_or_incomplete", "all_repos_enabled"}
629 assert "r1-fleet-private" not in r.text and "gabriel" not in r.text
630 for forbidden in ("queue", "prebuild_last_hour", "provider_latency_24h", "database_mb", "host_mem_available_mb"):
631 assert forbidden not in body # no host telemetry anonymously
632 again = await client.get("/api/intel/fleet")
633 assert again.json() == body # served from the 60 s cache
634
635
636 # ---------------------------------------------------------------------------
637 # E1 — a repository with no code symbols is "empty", not "blocked"/"failed" (found on staging: bench repos of .txt files)
638 # ---------------------------------------------------------------------------
639
640 @pytest.mark.asyncio
641 async def test_e1_dependents_of_an_empty_index_finish_empty_and_the_repository_reads_empty(worker_db, listed) -> None:
642 import musehub.worker as worker
643 from tests.test_intel_code_correctness import Dag
644 from tests.test_intel_job_state_machine import add_job as sm_add_job
645
646 dag = Dag()
647 dag.add("d0", [], None, 0) # a commit with no symbol operation at all
648 repo = await create_repo(worker_db, owner="gabriel", slug="e1-empty")
649 await dag.install(worker_db, repo.repo_id, ["d0"])
650 rid = repo.repo_id
651 head = dag.commits_id("d0")
652 worker_db.add(MusehubBranch(branch_id=secrets.token_hex(16), repo_id=rid, name="main", head_commit_id=head))
653 await worker_db.commit()
654 ids = {t: await sm_add_job(worker_db, rid, t, head) for t in ("intel.code", "intel.code.dead", "intel.code.coupling")}
655 for _ in range(3):
656 assert await worker._process_one() is True
657 for t, jid in ids.items():
658 got = await row(worker_db, jid)
659 assert got.status == "done" and got.result["outcome"] == "empty", (t, got.status, got.result)
660 st = (await load_intel_status(worker_db, [rid]))[rid]
661 assert st.state == "empty" and st.state_label == "No code symbols"
662 assert set(st.providers.values()) == {"empty"}
663 again = await rec.reconcile_pass(worker_db) # nothing to do, ever, until the head moves
664 await worker_db.commit()
665 assert again.enqueued_providers == 0 and await types_for(worker_db, rid) == []
666
667
668 @pytest.mark.asyncio
669 async def test_e1_stale_blocked_rows_of_an_empty_index_read_empty_not_failed(worker_db, listed) -> None:
670 repo = await repo_with_head(worker_db, "e1-stale-blocked")
671 rid = repo.repo_id
672 jid = await add_job(worker_db, rid, "intel.code", status="done", done_ago=30)
673 await worker_db.execute(update(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == jid).values(result={"executed": True, "outcome": "empty"}))
674 for t in ("intel.code.dead", "intel.code.coupling", "intel.code.entangle"):
675 b = await add_job(worker_db, rid, t, status="blocked", done_ago=20)
676 await worker_db.execute(update(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == b).values(result={"executed": False, "reason": "intel.code has not published revision"}))
677 await worker_db.commit()
678 st = (await load_intel_status(worker_db, [rid]))[rid]
679 assert st.state == "empty" and st.providers["intel.code.dead"] == "empty" and "failed" not in st.providers.values()
680
681
682 def test_e1_provider_state_empty_only_for_a_finished_empty_newest_job() -> None:
683 from musehub.services.musehub_intel_status import provider_state, repo_state
684
685 assert provider_state(None, H, ["done"], "empty") == "empty"
686 assert provider_state(None, H, ["done"], "computed") == "never" # a finished job that published nothing is still never a current zero
687 assert provider_state(None, H, ["failed", "done"], "empty") == "failed" # the NEWEST job decides
688 assert provider_state(None, H, ["pending"], "empty") == "queued"
689 assert repo_state({"intel.code": "empty", "intel.code.dead": "empty"}) == "empty"
690
691
692 def test_e2_a_provider_that_ran_and_found_nothing_does_not_make_the_repository_outdated() -> None:
693 from musehub.services.musehub_intel_status import repo_state
694
695 everything = {t: "current" for t in REQUIRED_JOB_TYPES}
696 assert repo_state(everything) == "current"
697 assert repo_state({**everything, "intel.code.detect_refactor": "empty"}) == "current" # a repository with no refactors is fully analysed
698 assert repo_state({**everything, "intel.code.detect_refactor": "never"}) == "outdated" # but one that never ran is not
699 assert repo_state({**everything, "intel.code.detect_refactor": "empty", "intel.code.dead": "analyzing"}) == "analyzing"
700
701
702 @pytest.mark.asyncio
703 async def test_b3_a_legacy_done_row_without_an_outcome_settles_nothing(worker_db, listed) -> None:
704 """Found on staging: rows written before the worker reported truthfully are ``done`` with no result (the work never ran); they must not count as a verdict."""
705 repo = await repo_with_head(worker_db, "b3-legacy")
706 rid = repo.repo_id
707 for t in REQUIRED_JOB_TYPES:
708 jid = await add_job(worker_db, rid, t, status="done", done_ago=86400 * 30)
709 await worker_db.execute(update(MusehubBackgroundJob).where(MusehubBackgroundJob.job_id == jid).values(result=None))
710 await worker_db.commit()
711 report = await rec.reconcile_pass(worker_db)
712 await worker_db.commit()
713 assert report.enqueued_providers == len(REQUIRED_JOB_TYPES)
714
715
716 @pytest.mark.asyncio
717 async def test_b3_old_not_executed_skips_do_not_exhaust_the_retry_limit(worker_db, listed) -> None:
718 repo = await repo_with_head(worker_db, "b3-oldskips")
719 rid = repo.repo_id
720 for t in REQUIRED_JOB_TYPES:
721 for _ in range(3): # three old pushes, all skipped while intel was off
722 await _skipped(worker_db, rid, t, f"{t} is not executed by this worker", age=86400 * 20)
723 report = await rec.reconcile_pass(worker_db)
724 await worker_db.commit()
725 assert report.enqueued_providers == len(REQUIRED_JOB_TYPES)
726
727
728 # ---------------------------------------------------------------------------
729 # C1 — the health snapshot's coupling step is bounded in memory and exactly equals the all-pairs result
730 # ---------------------------------------------------------------------------
731
732 def _brute_top_pairs(commit_syms: dict[str, set[str]], limit: int) -> list[tuple[str, str, int]]:
733 counts: dict[tuple[str, str], int] = {}
734 for syms in commit_syms.values():
735 ordered = sorted(syms)
736 for i, a in enumerate(ordered):
737 for b in ordered[i + 1:]:
738 counts[(a, b)] = counts.get((a, b), 0) + 1
739 return [(a, b, n) for (a, b), n in sorted(counts.items(), key=lambda kv: (-kv[1], kv[0][0], kv[0][1]))[:limit]]
740
741
742 def test_c1_top_cochange_pairs_equals_the_all_pairs_result_including_ties() -> None:
743 import random
744
745 from musehub.services.musehub_intel import top_cochange_pairs
746
747 rng = random.Random(7)
748 for trial in range(40):
749 universe = [f"f{i}.py::s{j}" for i in range(6) for j in range(8)]
750 commits = {f"c{k}": set(rng.sample(universe, rng.randint(1, 14))) for k in range(rng.randint(1, 60))}
751 for limit in (1, 5, 17):
752 assert top_cochange_pairs(commits, limit) == _brute_top_pairs(commits, limit), (trial, limit)
753 assert top_cochange_pairs({}, 5) == [] and top_cochange_pairs({"c": {"a"}}, 5) == [] and top_cochange_pairs({"c": {"a", "b"}}, 0) == []
754
755
756 def test_c1_it_never_holds_all_pairs() -> None:
757 """Two hundred commits of 400 symbols each have ~16M co-change pairs; the bounded version must finish with a tiny working set (tracemalloc peak)."""
758 import tracemalloc
759
760 from musehub.services.musehub_intel import top_cochange_pairs
761
762 commits = {f"c{k}": {f"m{(k * 37 + i) % 900}.py::s{i % 40}" for i in range(400)} for k in range(60)}
763 tracemalloc.start()
764 top = top_cochange_pairs(commits, 5)
765 peak = tracemalloc.get_traced_memory()[1]
766 tracemalloc.stop()
767 assert len(top) == 5 and peak < 40 * 1024 * 1024, peak # the all-pairs dict for this input is several hundred MiB
File History 8 commits
sha256:71cfef07c390e977ebb8066cbd052c3c5de869eb5395522f6768fd32c016f8ae feat(intel): small write batches and a stricter yield rule … Sonnet 5.5 patch 3 days ago
sha256:238a7da4341ade2c8d41ea9834d8e75e250647aefec02327f1e25fac55acd036 feat(intel): pace analysis database writes so a cold index … Sonnet 5.5 patch 3 days ago
sha256:0963e6208b3a866e197cb4583c38188e7ee9bff0763e214c152e85abd387d62a fix(intel): bound the memory of the health snapshot's coupl… Sonnet 5.5 patch 3 days ago
sha256:a938e341930f3c2c7442132d20643e00d7a91818572b377d5766ddc6d00a05a3 fix(intel): legacy done rows and old not-executed skips mus… Sonnet 5.5 minor ⚠ 3 days ago
sha256:91c3d2c16937d4affc7a1ab9ebfbec62d14e5d2fcfefd00d0a39769e90415f91 fix(intel): a provider that ran and found nothing (e.g. no … Sonnet 5.5 patch 3 days ago
sha256:0bed760b925ef8ad45036955346e241552d231ea5045477a59c2a6917d0ac84b fix(intel): a repository with no code symbols is 'empty', n… Sonnet 5.5 minor ⚠ 3 days ago
sha256:b0c91e115ce2eb4bec18b9c6e3be75b72f4c78f7c4421b5c87f774d4adefc29c fix(intel): Tier A review fixes for always-on intel (settle… Sonnet 5.5 minor ⚠ 3 days ago
sha256:545266da3f09ffbfe65a21bfc233bb91fe37f83ab714dd2d07db1fee98c3988b feat(intel): always-on code intelligence for every reposito… Sonnet 5.5 minor ⚠ 3 days ago