gabriel / musehub public
test_async_push_unpack.py python
359 lines 13.4 KB
Raw
sha256:cb4b05e9d6caea5f351e3cb452ee6a1db8856169328ad96bfba9ee8e04b8afa6 fix(jobs): fence every execution of a job; an expired or su… Sonnet 5.5 minor ⚠ breaking 4 days ago
1 """TDD — musehub#233: large pushes are deferred to a background job.
2
3 `/push/unpack-mpack` used to always unpack synchronously, inline in the
4 request. A push large enough to take a while risks a reverse proxy
5 (Cloudflare) timing out the HTTP request before the server finishes —
6 confirmed live (muse#108 rollout): a 620-object/400MB push 502'd at the
7 edge even though the server itself would eventually have succeeded.
8
9 Fix: when `blobs_count` exceeds `settings.async_push_blob_threshold`, the
10 route enqueues a `push.unpack` background job and returns 202 + job_id
11 immediately instead of processing inline. The client polls
12 `/push/job-status/{job_id}` until it's `done` (with the same result shape
13 the synchronous path always returned) or `failed`. Pushes at or below the
14 threshold are completely unaffected — same 200 response, same latency, as
15 before this ticket.
16
17 Coverage
18 --------
19 A push above threshold → 202 + job_id, no inline unpack, pending job row
20 B push at/below threshold → unchanged synchronous 200 behavior (regression)
21 C worker's process_push_unpack_job produces the same result shape as the
22 synchronous path, and complete_job() persists it to job.result
23 D job-status endpoint: pending / done (with result) / failed (with error)
24 E job-status 404s for a job_id belonging to a different repo or job_type
25 F enqueue_job(dedup_payload_key="mpack_key") — a retried request for the
26 same mpack_key hands back the existing job's ID, not a duplicate
27 """
28 from __future__ import annotations
29
30 import msgpack
31 import pytest
32 import pytest_asyncio
33 from httpx import AsyncClient, ASGITransport
34 from sqlalchemy import select
35 from sqlalchemy.ext.asyncio import AsyncSession
36
37 from muse.core.ids import hash_snapshot
38 from muse.core.mpack import build_wire_mpack
39 from muse.core.types import blob_id
40 from musehub.auth.request_signing import MSignContext, require_signed_request, optional_signed_request
41 from tests.commit_fixtures import with_reproducible_id
42 from musehub.db.database import get_db
43 from musehub.db.musehub_jobs_models import MusehubBackgroundJob
44 from musehub.main import app
45 from musehub.services.musehub_jobs import complete_job, enqueue_job
46 from musehub.services.musehub_wire_push import process_push_unpack_job, wire_push_unpack_mpack
47
48 pytestmark = pytest.mark.wire
49
50 _AUTH_CTX = MSignContext(
51 handle="gabriel",
52 identity_id="sha256:" + "0" * 64,
53 is_agent=False,
54 is_admin=True,
55 )
56
57
58 @pytest_asyncio.fixture()
59 async def client(db_session: AsyncSession) -> None:
60 async def _override_get_db() -> None:
61 yield db_session
62
63 app.dependency_overrides[get_db] = _override_get_db
64 app.dependency_overrides[require_signed_request] = lambda: _AUTH_CTX
65 app.dependency_overrides[optional_signed_request] = lambda: _AUTH_CTX
66
67 async with AsyncClient(
68 transport=ASGITransport(app=app),
69 base_url="https://localhost:1337",
70 ) as c:
71 yield c
72
73 app.dependency_overrides.clear()
74
75
76 @pytest_asyncio.fixture()
77 async def repo(client: AsyncClient) -> None:
78 resp = await client.post(
79 "/api/repos",
80 json={"owner": "gabriel", "name": "async-push-test", "visibility": "public", "initialize": False},
81 )
82 assert resp.status_code in (200, 201), resp.text
83 data = resp.json()
84 yield data
85 await client.delete(f"/api/repos/{data['repoId']}")
86
87
88 def _tiny_mpack() -> tuple[bytes, str]:
89 """A minimal but valid MUSE wire mpack with exactly one commit/snapshot/blob."""
90 oid = blob_id(b"hello")
91 manifest = {"a.txt": oid}
92 snap_id = hash_snapshot(manifest)
93 commit = with_reproducible_id({
94 "commit_id": "", "branch": "main", "message": "init",
95 "author": "gabriel", "committed_at": "2026-01-01T00:00:00+00:00",
96 "parent_commit_id": None, "parent2_commit_id": None,
97 "snapshot_id": snap_id, "agent_id": "", "model_id": "",
98 "toolchain_id": "", "sem_ver_bump": "none", "breaking_changes": [],
99 "signature": "", "signer_key_id": "", "signer_public_key": "",
100 "prompt_hash": "",
101 })
102 cid = str(commit["commit_id"])
103 wire_bytes = build_wire_mpack({
104 "commits": [commit],
105 "snapshots": [{
106 "snapshot_id": snap_id, "parent_snapshot_id": None,
107 "delta_upsert": manifest, "delta_remove": [], "directories": [],
108 }],
109 "blobs": [{"object_id": oid, "content": b"hello"}],
110 "tags": [],
111 })
112 return wire_bytes, cid
113
114
115 async def _push_unpack(
116 client: AsyncClient, slug: str, mpack_key: str, head: str, blobs_count: int,
117 ) -> "httpx.Response": # noqa: F821
118 return await client.post(
119 f"/gabriel/{slug}/push/unpack-mpack",
120 content=msgpack.packb(
121 {
122 "mpack_key": mpack_key, "branch": "main", "head": head,
123 "commits_count": 1, "blobs_count": blobs_count,
124 },
125 use_bin_type=True,
126 ),
127 headers={"Content-Type": "application/x-msgpack"},
128 )
129
130
131 # ---------------------------------------------------------------------------
132 # A — above threshold defers to a background job
133 # ---------------------------------------------------------------------------
134
135
136 @pytest.mark.asyncio
137 async def test_push_above_threshold_returns_202_with_pending_job(
138 client: AsyncClient, repo: dict, db_session: AsyncSession, monkeypatch: pytest.MonkeyPatch,
139 ) -> None:
140 from musehub.config import get_settings
141 monkeypatch.setattr(get_settings(), "async_push_blob_threshold", 0)
142
143 wire_bytes, cid = _tiny_mpack()
144 mpack_key = blob_id(wire_bytes)
145 from musehub.storage.backends import get_backend
146 await get_backend().put_mpack(mpack_key, wire_bytes)
147
148 resp = await _push_unpack(client, repo["slug"], mpack_key, cid, blobs_count=1)
149
150 assert resp.status_code == 202, resp.text
151 body = resp.json()
152 assert body["status"] == "accepted"
153 assert body["job_id"]
154
155 job = await db_session.get(MusehubBackgroundJob, body["job_id"])
156 assert job is not None
157 assert job.job_type == "push.unpack"
158 assert job.status == "pending"
159 assert job.payload["mpack_key"] == mpack_key
160 assert job.payload["head_commit_id"] == cid
161
162 # The push must NOT have been applied inline — no branch advance yet.
163 refs = await client.get(f"/gabriel/{repo['slug']}/refs")
164 assert refs.json()["branch_heads"].get("main") != cid
165
166
167 # ---------------------------------------------------------------------------
168 # B — at/below threshold: unchanged synchronous behavior (regression guard)
169 # ---------------------------------------------------------------------------
170
171
172 @pytest.mark.asyncio
173 async def test_push_at_or_below_threshold_is_still_synchronous(
174 client: AsyncClient, repo: dict,
175 ) -> None:
176 """Default threshold (200) — a 1-blob push must return 200 inline, exactly
177 as it always has. This is the common case; it must never regress."""
178 wire_bytes, cid = _tiny_mpack()
179 mpack_key = blob_id(wire_bytes)
180 from musehub.storage.backends import get_backend
181 await get_backend().put_mpack(mpack_key, wire_bytes)
182
183 resp = await _push_unpack(client, repo["slug"], mpack_key, cid, blobs_count=1)
184
185 assert resp.status_code == 200, resp.text
186 body = resp.json()
187 assert body["head"] == cid
188 assert body["blobs_written"] == 1
189 assert body["commits_written"] == 1
190
191 refs = await client.get(f"/gabriel/{repo['slug']}/refs")
192 assert refs.json()["branch_heads"]["main"] == cid
193
194
195 # ---------------------------------------------------------------------------
196 # C — worker handler produces the same result shape; complete_job persists it
197 # ---------------------------------------------------------------------------
198
199
200 @pytest.mark.asyncio
201 async def test_process_push_unpack_job_matches_sync_result_and_is_persisted(
202 db_session: AsyncSession,
203 ) -> None:
204 from musehub.services.musehub_repository import create_repo
205 from musehub.core.genesis import compute_identity_id
206
207 repo_row = await create_repo(
208 db_session, name="async-push-worker-test", owner="gabriel",
209 owner_user_id=compute_identity_id(b"gabriel"), visibility="public",
210 initialize=False,
211 )
212 await db_session.commit()
213
214 wire_bytes, cid = _tiny_mpack()
215 mpack_key = blob_id(wire_bytes)
216 from musehub.storage.backends import get_backend
217 await get_backend().put_mpack(mpack_key, wire_bytes)
218
219 job_id = await enqueue_job(
220 db_session, repo_row.repo_id, "push.unpack",
221 {
222 "mpack_key": mpack_key, "pusher_id": "gabriel", "branch": "main",
223 "head_commit_id": cid, "commits_count": 1, "blobs_count": 1,
224 "force": False,
225 },
226 dedup_payload_key="mpack_key",
227 )
228 await db_session.commit()
229 assert job_id is not None
230
231 result = await process_push_unpack_job(db_session, job_id)
232 await db_session.commit()
233
234 assert result["head"] == cid
235 assert result["branch"] == "main"
236 assert result["blobs_written"] == 1
237 assert result["commits_written"] == 1
238
239 await complete_job(db_session, job_id, result=result)
240 await db_session.commit()
241
242 job = await db_session.get(MusehubBackgroundJob, job_id)
243 assert job.status == "done"
244 assert job.result == result
245
246
247 # ---------------------------------------------------------------------------
248 # D — job-status endpoint: pending / done / failed
249 # ---------------------------------------------------------------------------
250
251
252 @pytest.mark.asyncio
253 async def test_job_status_pending_then_done(
254 client: AsyncClient, repo: dict, db_session: AsyncSession,
255 ) -> None:
256 job_id = await enqueue_job(
257 db_session, repo["repoId"], "push.unpack",
258 {"mpack_key": "sha256:" + "a" * 64, "branch": "main", "head_commit_id": "sha256:" + "b" * 64},
259 )
260 await db_session.commit()
261
262 resp = await client.get(f"/gabriel/{repo['slug']}/push/job-status/{job_id}")
263 assert resp.status_code == 200
264 assert resp.json()["status"] == "pending"
265
266 await complete_job(db_session, job_id, result={"head": "sha256:" + "b" * 64, "blobs_written": 3})
267 await db_session.commit()
268
269 resp = await client.get(f"/gabriel/{repo['slug']}/push/job-status/{job_id}")
270 body = resp.json()
271 assert body["status"] == "done"
272 assert body["result"]["blobs_written"] == 3
273
274
275 @pytest.mark.asyncio
276 async def test_job_status_failed_includes_error(
277 client: AsyncClient, repo: dict, db_session: AsyncSession,
278 ) -> None:
279 from musehub.services.musehub_jobs import fail_job
280
281 job_id = await enqueue_job(
282 db_session, repo["repoId"], "push.unpack",
283 {"mpack_key": "sha256:" + "c" * 64},
284 )
285 await db_session.commit()
286 await fail_job(db_session, job_id, "push rejected: not a fast-forward")
287 # fail_job resets to pending while attempts remain -- exhaust them for this test.
288 job = await db_session.get(MusehubBackgroundJob, job_id)
289 job.attempt = 99
290 await fail_job(db_session, job_id, "push rejected: not a fast-forward")
291 await db_session.commit()
292
293 resp = await client.get(f"/gabriel/{repo['slug']}/push/job-status/{job_id}")
294 body = resp.json()
295 assert body["status"] == "failed"
296 assert "not a fast-forward" in body["error"]
297
298
299 # ---------------------------------------------------------------------------
300 # E — job-status 404s across repo/job_type boundaries
301 # ---------------------------------------------------------------------------
302
303
304 @pytest.mark.asyncio
305 async def test_job_status_404_for_wrong_repo(
306 client: AsyncClient, repo: dict, db_session: AsyncSession,
307 ) -> None:
308 from musehub.services.musehub_repository import create_repo
309 from musehub.core.genesis import compute_identity_id
310
311 other_repo = await create_repo(
312 db_session, name="other-repo-for-404-test", owner="gabriel",
313 owner_user_id=compute_identity_id(b"gabriel"), visibility="public",
314 initialize=False,
315 )
316 await db_session.commit()
317
318 job_id = await enqueue_job(
319 db_session, other_repo.repo_id, "push.unpack",
320 {"mpack_key": "sha256:" + "d" * 64},
321 )
322 await db_session.commit()
323
324 # Polling via THIS repo's slug for a job that belongs to a different repo.
325 resp = await client.get(f"/gabriel/{repo['slug']}/push/job-status/{job_id}")
326 assert resp.status_code == 404
327
328
329 # ---------------------------------------------------------------------------
330 # F — dedup on mpack_key: a retried request hands back the existing job
331 # ---------------------------------------------------------------------------
332
333
334 @pytest.mark.asyncio
335 async def test_duplicate_mpack_key_push_reuses_pending_job(
336 client: AsyncClient, repo: dict, db_session: AsyncSession, monkeypatch: pytest.MonkeyPatch,
337 ) -> None:
338 from musehub.config import get_settings
339 monkeypatch.setattr(get_settings(), "async_push_blob_threshold", 0)
340
341 wire_bytes, cid = _tiny_mpack()
342 mpack_key = blob_id(wire_bytes)
343 from musehub.storage.backends import get_backend
344 await get_backend().put_mpack(mpack_key, wire_bytes)
345
346 resp1 = await _push_unpack(client, repo["slug"], mpack_key, cid, blobs_count=1)
347 resp2 = await _push_unpack(client, repo["slug"], mpack_key, cid, blobs_count=1)
348
349 job_id_1 = resp1.json()["job_id"]
350 job_id_2 = resp2.json()["job_id"]
351 assert job_id_1 == job_id_2, "retried push with the same mpack_key must reuse the pending job"
352
353 count = (await db_session.execute(
354 select(MusehubBackgroundJob).where(
355 MusehubBackgroundJob.job_type == "push.unpack",
356 MusehubBackgroundJob.payload["mpack_key"].astext == mpack_key,
357 )
358 )).scalars().all()
359 assert len(count) == 1, "must not create a duplicate job row for the same mpack_key"
File History 4 commits
sha256:cb4b05e9d6caea5f351e3cb452ee6a1db8856169328ad96bfba9ee8e04b8afa6 fix(jobs): fence every execution of a job; an expired or su… Sonnet 5.5 minor ⚠ 4 days ago
sha256:383b9b15b1a7c18348bf797dbec252b8842199567b4b04371578fedd5f7891d4 merge: integrate push-ingest commit verification into dev Opus 5.5 minor ⚠ 11 days ago
sha256:39b90cebf3ab4c72ef67a29fe9d451ba1394084edf4053dff98878651f5d2378 test: build reproducible commit ids in push fixtures Opus 5.5 11 days ago
sha256:5ea41f687b567512d786a8bff6f949aa626b4d52c7cdcebfa9e950d18b93e5a5 feat: defer large pushes to a background job (musehub#233) Sonnet 5 minor ⚠ 17 days ago