gabriel / musehub public
test_async_push_unpack.py python
358 lines 13.6 KB
Raw
sha256:cb4b05e9d6caea5f351e3cb452ee6a1db8856169328ad96bfba9ee8e04b8afa6 fix(jobs): fence every execution of a job; an expired or su… Sonnet 5.5 minor ⚠ breaking 4 days ago
1 """TDD — musehub#233: large pushes are deferred to a background job.
2
3 `/push/unpack-mpack` used to always unpack synchronously, inline in the
4 request. A push large enough to take a while risks a reverse proxy
5 (Cloudflare) timing out the HTTP request before the server finishes —
6 confirmed live (muse#108 rollout): a 620-object/400MB push 502'd at the
7 edge even though the server itself would eventually have succeeded.
8
9 Fix: when `blobs_count` exceeds `settings.async_push_blob_threshold`, the
10 route enqueues a `push.unpack` background job and returns 202 + job_id
11 immediately instead of processing inline. The client polls
12 `/push/job-status/{job_id}` until it's `done` (with the same result shape
13 the synchronous path always returned) or `failed`. Pushes at or below the
14 threshold are completely unaffected — same 200 response, same latency, as
15 before this ticket.
16
17 Coverage
18 --------
19 A push above threshold → 202 + job_id, no inline unpack, pending job row
20 B push at/below threshold → unchanged synchronous 200 behavior (regression)
21 C worker's process_push_unpack_job produces the same result shape as the
22 synchronous path, and complete_job() persists it to job.result
23 D job-status endpoint: pending / done (with result) / failed (with error)
24 E job-status 404s for a job_id belonging to a different repo or job_type
25 F enqueue_job(dedup_payload_key="mpack_key") — a retried request for the
26 same mpack_key hands back the existing job's ID, not a duplicate
27 """
28 from __future__ import annotations
29
30 import msgpack
31 import pytest
32 import pytest_asyncio
33 from httpx import AsyncClient, ASGITransport
34 from sqlalchemy import select
35 from sqlalchemy.ext.asyncio import AsyncSession
36
37 from muse.core.ids import hash_snapshot
38 from muse.core.mpack import build_wire_mpack
39 from muse.core.types import blob_id
40 from musehub.auth.request_signing import MSignContext, require_signed_request, optional_signed_request
41 from tests.job_execution import current_attempt, start_execution # noqa: F401
42 from tests.commit_fixtures import with_reproducible_id
43 from musehub.db.database import get_db
44 from musehub.db.musehub_jobs_models import MusehubBackgroundJob
45 from musehub.main import app
46 from musehub.services.musehub_jobs import complete_job, enqueue_job
47 from musehub.services.musehub_wire_push import process_push_unpack_job, wire_push_unpack_mpack
48
49 pytestmark = pytest.mark.wire
50
51 _AUTH_CTX = MSignContext(
52 handle="gabriel",
53 identity_id="sha256:" + "0" * 64,
54 is_agent=False,
55 is_admin=True,
56 )
57
58
59 @pytest_asyncio.fixture()
60 async def client(db_session: AsyncSession) -> None:
61 async def _override_get_db() -> None:
62 yield db_session
63
64 app.dependency_overrides[get_db] = _override_get_db
65 app.dependency_overrides[require_signed_request] = lambda: _AUTH_CTX
66 app.dependency_overrides[optional_signed_request] = lambda: _AUTH_CTX
67
68 async with AsyncClient(
69 transport=ASGITransport(app=app),
70 base_url="https://localhost:1337",
71 ) as c:
72 yield c
73
74 app.dependency_overrides.clear()
75
76
77 @pytest_asyncio.fixture()
78 async def repo(client: AsyncClient) -> None:
79 resp = await client.post(
80 "/api/repos",
81 json={"owner": "gabriel", "name": "async-push-test", "visibility": "public", "initialize": False},
82 )
83 assert resp.status_code in (200, 201), resp.text
84 data = resp.json()
85 yield data
86 await client.delete(f"/api/repos/{data['repoId']}")
87
88
89 def _tiny_mpack() -> tuple[bytes, str]:
90 """A minimal but valid MUSE wire mpack with exactly one commit/snapshot/blob."""
91 oid = blob_id(b"hello")
92 manifest = {"a.txt": oid}
93 snap_id = hash_snapshot(manifest)
94 commit = with_reproducible_id({
95 "commit_id": "", "branch": "main", "message": "init",
96 "author": "gabriel", "committed_at": "2026-01-01T00:00:00+00:00",
97 "parent_commit_id": None, "parent2_commit_id": None,
98 "snapshot_id": snap_id, "agent_id": "", "model_id": "",
99 "toolchain_id": "", "sem_ver_bump": "none", "breaking_changes": [],
100 "signature": "", "signer_key_id": "", "signer_public_key": "",
101 "prompt_hash": "",
102 })
103 cid = str(commit["commit_id"])
104 wire_bytes = build_wire_mpack({
105 "commits": [commit],
106 "snapshots": [{
107 "snapshot_id": snap_id, "parent_snapshot_id": None,
108 "delta_upsert": manifest, "delta_remove": [], "directories": [],
109 }],
110 "blobs": [{"object_id": oid, "content": b"hello"}],
111 "tags": [],
112 })
113 return wire_bytes, cid
114
115
116 async def _push_unpack(
117 client: AsyncClient, slug: str, mpack_key: str, head: str, blobs_count: int,
118 ) -> "httpx.Response": # noqa: F821
119 return await client.post(
120 f"/gabriel/{slug}/push/unpack-mpack",
121 content=msgpack.packb(
122 {
123 "mpack_key": mpack_key, "branch": "main", "head": head,
124 "commits_count": 1, "blobs_count": blobs_count,
125 },
126 use_bin_type=True,
127 ),
128 headers={"Content-Type": "application/x-msgpack"},
129 )
130
131
132 # ---------------------------------------------------------------------------
133 # A — above threshold defers to a background job
134 # ---------------------------------------------------------------------------
135
136
137 @pytest.mark.asyncio
138 async def test_push_above_threshold_returns_202_with_pending_job(
139 client: AsyncClient, repo: dict, db_session: AsyncSession, monkeypatch: pytest.MonkeyPatch,
140 ) -> None:
141 from musehub.config import get_settings
142 monkeypatch.setattr(get_settings(), "async_push_blob_threshold", 0)
143
144 wire_bytes, cid = _tiny_mpack()
145 mpack_key = blob_id(wire_bytes)
146 from musehub.storage.backends import get_backend
147 await get_backend().put_mpack(mpack_key, wire_bytes)
148
149 resp = await _push_unpack(client, repo["slug"], mpack_key, cid, blobs_count=1)
150
151 assert resp.status_code == 202, resp.text
152 body = resp.json()
153 assert body["status"] == "accepted"
154 assert body["job_id"]
155
156 job = await db_session.get(MusehubBackgroundJob, body["job_id"])
157 assert job is not None
158 assert job.job_type == "push.unpack"
159 assert job.status == "pending"
160 assert job.payload["mpack_key"] == mpack_key
161 assert job.payload["head_commit_id"] == cid
162
163 # The push must NOT have been applied inline — no branch advance yet.
164 refs = await client.get(f"/gabriel/{repo['slug']}/refs")
165 assert refs.json()["branch_heads"].get("main") != cid
166
167
168 # ---------------------------------------------------------------------------
169 # B — at/below threshold: unchanged synchronous behavior (regression guard)
170 # ---------------------------------------------------------------------------
171
172
173 @pytest.mark.asyncio
174 async def test_push_at_or_below_threshold_is_still_synchronous(
175 client: AsyncClient, repo: dict,
176 ) -> None:
177 """Default threshold (200) — a 1-blob push must return 200 inline, exactly
178 as it always has. This is the common case; it must never regress."""
179 wire_bytes, cid = _tiny_mpack()
180 mpack_key = blob_id(wire_bytes)
181 from musehub.storage.backends import get_backend
182 await get_backend().put_mpack(mpack_key, wire_bytes)
183
184 resp = await _push_unpack(client, repo["slug"], mpack_key, cid, blobs_count=1)
185
186 assert resp.status_code == 200, resp.text
187 body = resp.json()
188 assert body["head"] == cid
189 assert body["blobs_written"] == 1
190 assert body["commits_written"] == 1
191
192 refs = await client.get(f"/gabriel/{repo['slug']}/refs")
193 assert refs.json()["branch_heads"]["main"] == cid
194
195
196 # ---------------------------------------------------------------------------
197 # C — worker handler produces the same result shape; complete_job persists it
198 # ---------------------------------------------------------------------------
199
200
201 @pytest.mark.asyncio
202 async def test_process_push_unpack_job_matches_sync_result_and_is_persisted(
203 db_session: AsyncSession,
204 ) -> None:
205 from musehub.services.musehub_repository import create_repo
206 from musehub.core.genesis import compute_identity_id
207
208 repo_row = await create_repo(
209 db_session, name="async-push-worker-test", owner="gabriel",
210 owner_user_id=compute_identity_id(b"gabriel"), visibility="public",
211 initialize=False,
212 )
213 await db_session.commit()
214
215 wire_bytes, cid = _tiny_mpack()
216 mpack_key = blob_id(wire_bytes)
217 from musehub.storage.backends import get_backend
218 await get_backend().put_mpack(mpack_key, wire_bytes)
219
220 job_id = await enqueue_job(
221 db_session, repo_row.repo_id, "push.unpack",
222 {
223 "mpack_key": mpack_key, "pusher_id": "gabriel", "branch": "main",
224 "head_commit_id": cid, "commits_count": 1, "blobs_count": 1,
225 "force": False,
226 },
227 dedup_payload_key="mpack_key",
228 )
229 await db_session.commit()
230 assert job_id is not None
231
232 result = await process_push_unpack_job(db_session, job_id)
233 await db_session.commit()
234
235 assert result["head"] == cid
236 assert result["branch"] == "main"
237 assert result["blobs_written"] == 1
238 assert result["commits_written"] == 1
239
240 await complete_job(db_session, job_id, result=result, attempt=await start_execution(db_session, job_id))
241 await db_session.commit()
242
243 job = await db_session.get(MusehubBackgroundJob, job_id)
244 assert job.status == "done"
245 assert job.result == result
246
247
248 # ---------------------------------------------------------------------------
249 # D — job-status endpoint: pending / done / failed
250 # ---------------------------------------------------------------------------
251
252
253 @pytest.mark.asyncio
254 async def test_job_status_pending_then_done(
255 client: AsyncClient, repo: dict, db_session: AsyncSession,
256 ) -> None:
257 job_id = await enqueue_job(
258 db_session, repo["repoId"], "push.unpack",
259 {"mpack_key": "sha256:" + "a" * 64, "branch": "main", "head_commit_id": "sha256:" + "b" * 64},
260 )
261 await db_session.commit()
262
263 resp = await client.get(f"/gabriel/{repo['slug']}/push/job-status/{job_id}")
264 assert resp.status_code == 200
265 assert resp.json()["status"] == "pending"
266
267 await complete_job(db_session, job_id, result={"head": "sha256:" + "b" * 64, "blobs_written": 3}, attempt=await start_execution(db_session, job_id))
268 await db_session.commit()
269
270 resp = await client.get(f"/gabriel/{repo['slug']}/push/job-status/{job_id}")
271 body = resp.json()
272 assert body["status"] == "done"
273 assert body["result"]["blobs_written"] == 3
274
275
276 @pytest.mark.asyncio
277 async def test_job_status_failed_includes_error(
278 client: AsyncClient, repo: dict, db_session: AsyncSession,
279 ) -> None:
280 from musehub.services.musehub_jobs import fail_job
281
282 job_id = await enqueue_job(
283 db_session, repo["repoId"], "push.unpack",
284 {"mpack_key": "sha256:" + "c" * 64},
285 )
286 await db_session.commit()
287 await fail_job(db_session, job_id, "push rejected: not a fast-forward", attempt=await start_execution(db_session, job_id))
288 # fail_job resets to pending while attempts remain -- exhaust them for this test.
289 await fail_job(db_session, job_id, "push rejected: not a fast-forward", attempt=await start_execution(db_session, job_id, attempt=99))
290 await db_session.commit()
291
292 resp = await client.get(f"/gabriel/{repo['slug']}/push/job-status/{job_id}")
293 body = resp.json()
294 assert body["status"] == "failed"
295 assert "not a fast-forward" in body["error"]
296
297
298 # ---------------------------------------------------------------------------
299 # E — job-status 404s across repo/job_type boundaries
300 # ---------------------------------------------------------------------------
301
302
303 @pytest.mark.asyncio
304 async def test_job_status_404_for_wrong_repo(
305 client: AsyncClient, repo: dict, db_session: AsyncSession,
306 ) -> None:
307 from musehub.services.musehub_repository import create_repo
308 from musehub.core.genesis import compute_identity_id
309
310 other_repo = await create_repo(
311 db_session, name="other-repo-for-404-test", owner="gabriel",
312 owner_user_id=compute_identity_id(b"gabriel"), visibility="public",
313 initialize=False,
314 )
315 await db_session.commit()
316
317 job_id = await enqueue_job(
318 db_session, other_repo.repo_id, "push.unpack",
319 {"mpack_key": "sha256:" + "d" * 64},
320 )
321 await db_session.commit()
322
323 # Polling via THIS repo's slug for a job that belongs to a different repo.
324 resp = await client.get(f"/gabriel/{repo['slug']}/push/job-status/{job_id}")
325 assert resp.status_code == 404
326
327
328 # ---------------------------------------------------------------------------
329 # F — dedup on mpack_key: a retried request hands back the existing job
330 # ---------------------------------------------------------------------------
331
332
333 @pytest.mark.asyncio
334 async def test_duplicate_mpack_key_push_reuses_pending_job(
335 client: AsyncClient, repo: dict, db_session: AsyncSession, monkeypatch: pytest.MonkeyPatch,
336 ) -> None:
337 from musehub.config import get_settings
338 monkeypatch.setattr(get_settings(), "async_push_blob_threshold", 0)
339
340 wire_bytes, cid = _tiny_mpack()
341 mpack_key = blob_id(wire_bytes)
342 from musehub.storage.backends import get_backend
343 await get_backend().put_mpack(mpack_key, wire_bytes)
344
345 resp1 = await _push_unpack(client, repo["slug"], mpack_key, cid, blobs_count=1)
346 resp2 = await _push_unpack(client, repo["slug"], mpack_key, cid, blobs_count=1)
347
348 job_id_1 = resp1.json()["job_id"]
349 job_id_2 = resp2.json()["job_id"]
350 assert job_id_1 == job_id_2, "retried push with the same mpack_key must reuse the pending job"
351
352 count = (await db_session.execute(
353 select(MusehubBackgroundJob).where(
354 MusehubBackgroundJob.job_type == "push.unpack",
355 MusehubBackgroundJob.payload["mpack_key"].astext == mpack_key,
356 )
357 )).scalars().all()
358 assert len(count) == 1, "must not create a duplicate job row for the same mpack_key"
File History 4 commits
sha256:cb4b05e9d6caea5f351e3cb452ee6a1db8856169328ad96bfba9ee8e04b8afa6 fix(jobs): fence every execution of a job; an expired or su… Sonnet 5.5 minor ⚠ 4 days ago
sha256:383b9b15b1a7c18348bf797dbec252b8842199567b4b04371578fedd5f7891d4 merge: integrate push-ingest commit verification into dev Opus 5.5 minor ⚠ 11 days ago
sha256:39b90cebf3ab4c72ef67a29fe9d451ba1394084edf4053dff98878651f5d2378 test: build reproducible commit ids in push fixtures Opus 5.5 11 days ago
sha256:5ea41f687b567512d786a8bff6f949aa626b4d52c7cdcebfa9e950d18b93e5a5 feat: defer large pushes to a background job (musehub#233) Sonnet 5 minor ⚠ 17 days ago