test_async_push_unpack.py
python
sha256:cb4b05e9d6caea5f351e3cb452ee6a1db8856169328ad96bfba9ee8e04b8afa6
fix(jobs): fence every execution of a job; an expired or su…
Sonnet 5.5
minor
⚠ breaking
4 days ago
| 1 | """TDD — musehub#233: large pushes are deferred to a background job. |
| 2 | |
| 3 | `/push/unpack-mpack` used to always unpack synchronously, inline in the |
| 4 | request. A push large enough to take a while risks a reverse proxy |
| 5 | (Cloudflare) timing out the HTTP request before the server finishes — |
| 6 | confirmed live (muse#108 rollout): a 620-object/400MB push 502'd at the |
| 7 | edge even though the server itself would eventually have succeeded. |
| 8 | |
| 9 | Fix: when `blobs_count` exceeds `settings.async_push_blob_threshold`, the |
| 10 | route enqueues a `push.unpack` background job and returns 202 + job_id |
| 11 | immediately instead of processing inline. The client polls |
| 12 | `/push/job-status/{job_id}` until it's `done` (with the same result shape |
| 13 | the synchronous path always returned) or `failed`. Pushes at or below the |
| 14 | threshold are completely unaffected — same 200 response, same latency, as |
| 15 | before this ticket. |
| 16 | |
| 17 | Coverage |
| 18 | -------- |
| 19 | A push above threshold → 202 + job_id, no inline unpack, pending job row |
| 20 | B push at/below threshold → unchanged synchronous 200 behavior (regression) |
| 21 | C worker's process_push_unpack_job produces the same result shape as the |
| 22 | synchronous path, and complete_job() persists it to job.result |
| 23 | D job-status endpoint: pending / done (with result) / failed (with error) |
| 24 | E job-status 404s for a job_id belonging to a different repo or job_type |
| 25 | F enqueue_job(dedup_payload_key="mpack_key") — a retried request for the |
| 26 | same mpack_key hands back the existing job's ID, not a duplicate |
| 27 | """ |
| 28 | from __future__ import annotations |
| 29 | |
| 30 | import msgpack |
| 31 | import pytest |
| 32 | import pytest_asyncio |
| 33 | from httpx import AsyncClient, ASGITransport |
| 34 | from sqlalchemy import select |
| 35 | from sqlalchemy.ext.asyncio import AsyncSession |
| 36 | |
| 37 | from muse.core.ids import hash_snapshot |
| 38 | from muse.core.mpack import build_wire_mpack |
| 39 | from muse.core.types import blob_id |
| 40 | from musehub.auth.request_signing import MSignContext, require_signed_request, optional_signed_request |
| 41 | from tests.job_execution import current_attempt, start_execution # noqa: F401 |
| 42 | from tests.commit_fixtures import with_reproducible_id |
| 43 | from musehub.db.database import get_db |
| 44 | from musehub.db.musehub_jobs_models import MusehubBackgroundJob |
| 45 | from musehub.main import app |
| 46 | from musehub.services.musehub_jobs import complete_job, enqueue_job |
| 47 | from musehub.services.musehub_wire_push import process_push_unpack_job, wire_push_unpack_mpack |
| 48 | |
| 49 | pytestmark = pytest.mark.wire |
| 50 | |
| 51 | _AUTH_CTX = MSignContext( |
| 52 | handle="gabriel", |
| 53 | identity_id="sha256:" + "0" * 64, |
| 54 | is_agent=False, |
| 55 | is_admin=True, |
| 56 | ) |
| 57 | |
| 58 | |
| 59 | @pytest_asyncio.fixture() |
| 60 | async def client(db_session: AsyncSession) -> None: |
| 61 | async def _override_get_db() -> None: |
| 62 | yield db_session |
| 63 | |
| 64 | app.dependency_overrides[get_db] = _override_get_db |
| 65 | app.dependency_overrides[require_signed_request] = lambda: _AUTH_CTX |
| 66 | app.dependency_overrides[optional_signed_request] = lambda: _AUTH_CTX |
| 67 | |
| 68 | async with AsyncClient( |
| 69 | transport=ASGITransport(app=app), |
| 70 | base_url="https://localhost:1337", |
| 71 | ) as c: |
| 72 | yield c |
| 73 | |
| 74 | app.dependency_overrides.clear() |
| 75 | |
| 76 | |
| 77 | @pytest_asyncio.fixture() |
| 78 | async def repo(client: AsyncClient) -> None: |
| 79 | resp = await client.post( |
| 80 | "/api/repos", |
| 81 | json={"owner": "gabriel", "name": "async-push-test", "visibility": "public", "initialize": False}, |
| 82 | ) |
| 83 | assert resp.status_code in (200, 201), resp.text |
| 84 | data = resp.json() |
| 85 | yield data |
| 86 | await client.delete(f"/api/repos/{data['repoId']}") |
| 87 | |
| 88 | |
| 89 | def _tiny_mpack() -> tuple[bytes, str]: |
| 90 | """A minimal but valid MUSE wire mpack with exactly one commit/snapshot/blob.""" |
| 91 | oid = blob_id(b"hello") |
| 92 | manifest = {"a.txt": oid} |
| 93 | snap_id = hash_snapshot(manifest) |
| 94 | commit = with_reproducible_id({ |
| 95 | "commit_id": "", "branch": "main", "message": "init", |
| 96 | "author": "gabriel", "committed_at": "2026-01-01T00:00:00+00:00", |
| 97 | "parent_commit_id": None, "parent2_commit_id": None, |
| 98 | "snapshot_id": snap_id, "agent_id": "", "model_id": "", |
| 99 | "toolchain_id": "", "sem_ver_bump": "none", "breaking_changes": [], |
| 100 | "signature": "", "signer_key_id": "", "signer_public_key": "", |
| 101 | "prompt_hash": "", |
| 102 | }) |
| 103 | cid = str(commit["commit_id"]) |
| 104 | wire_bytes = build_wire_mpack({ |
| 105 | "commits": [commit], |
| 106 | "snapshots": [{ |
| 107 | "snapshot_id": snap_id, "parent_snapshot_id": None, |
| 108 | "delta_upsert": manifest, "delta_remove": [], "directories": [], |
| 109 | }], |
| 110 | "blobs": [{"object_id": oid, "content": b"hello"}], |
| 111 | "tags": [], |
| 112 | }) |
| 113 | return wire_bytes, cid |
| 114 | |
| 115 | |
| 116 | async def _push_unpack( |
| 117 | client: AsyncClient, slug: str, mpack_key: str, head: str, blobs_count: int, |
| 118 | ) -> "httpx.Response": # noqa: F821 |
| 119 | return await client.post( |
| 120 | f"/gabriel/{slug}/push/unpack-mpack", |
| 121 | content=msgpack.packb( |
| 122 | { |
| 123 | "mpack_key": mpack_key, "branch": "main", "head": head, |
| 124 | "commits_count": 1, "blobs_count": blobs_count, |
| 125 | }, |
| 126 | use_bin_type=True, |
| 127 | ), |
| 128 | headers={"Content-Type": "application/x-msgpack"}, |
| 129 | ) |
| 130 | |
| 131 | |
| 132 | # --------------------------------------------------------------------------- |
| 133 | # A — above threshold defers to a background job |
| 134 | # --------------------------------------------------------------------------- |
| 135 | |
| 136 | |
| 137 | @pytest.mark.asyncio |
| 138 | async def test_push_above_threshold_returns_202_with_pending_job( |
| 139 | client: AsyncClient, repo: dict, db_session: AsyncSession, monkeypatch: pytest.MonkeyPatch, |
| 140 | ) -> None: |
| 141 | from musehub.config import get_settings |
| 142 | monkeypatch.setattr(get_settings(), "async_push_blob_threshold", 0) |
| 143 | |
| 144 | wire_bytes, cid = _tiny_mpack() |
| 145 | mpack_key = blob_id(wire_bytes) |
| 146 | from musehub.storage.backends import get_backend |
| 147 | await get_backend().put_mpack(mpack_key, wire_bytes) |
| 148 | |
| 149 | resp = await _push_unpack(client, repo["slug"], mpack_key, cid, blobs_count=1) |
| 150 | |
| 151 | assert resp.status_code == 202, resp.text |
| 152 | body = resp.json() |
| 153 | assert body["status"] == "accepted" |
| 154 | assert body["job_id"] |
| 155 | |
| 156 | job = await db_session.get(MusehubBackgroundJob, body["job_id"]) |
| 157 | assert job is not None |
| 158 | assert job.job_type == "push.unpack" |
| 159 | assert job.status == "pending" |
| 160 | assert job.payload["mpack_key"] == mpack_key |
| 161 | assert job.payload["head_commit_id"] == cid |
| 162 | |
| 163 | # The push must NOT have been applied inline — no branch advance yet. |
| 164 | refs = await client.get(f"/gabriel/{repo['slug']}/refs") |
| 165 | assert refs.json()["branch_heads"].get("main") != cid |
| 166 | |
| 167 | |
| 168 | # --------------------------------------------------------------------------- |
| 169 | # B — at/below threshold: unchanged synchronous behavior (regression guard) |
| 170 | # --------------------------------------------------------------------------- |
| 171 | |
| 172 | |
| 173 | @pytest.mark.asyncio |
| 174 | async def test_push_at_or_below_threshold_is_still_synchronous( |
| 175 | client: AsyncClient, repo: dict, |
| 176 | ) -> None: |
| 177 | """Default threshold (200) — a 1-blob push must return 200 inline, exactly |
| 178 | as it always has. This is the common case; it must never regress.""" |
| 179 | wire_bytes, cid = _tiny_mpack() |
| 180 | mpack_key = blob_id(wire_bytes) |
| 181 | from musehub.storage.backends import get_backend |
| 182 | await get_backend().put_mpack(mpack_key, wire_bytes) |
| 183 | |
| 184 | resp = await _push_unpack(client, repo["slug"], mpack_key, cid, blobs_count=1) |
| 185 | |
| 186 | assert resp.status_code == 200, resp.text |
| 187 | body = resp.json() |
| 188 | assert body["head"] == cid |
| 189 | assert body["blobs_written"] == 1 |
| 190 | assert body["commits_written"] == 1 |
| 191 | |
| 192 | refs = await client.get(f"/gabriel/{repo['slug']}/refs") |
| 193 | assert refs.json()["branch_heads"]["main"] == cid |
| 194 | |
| 195 | |
| 196 | # --------------------------------------------------------------------------- |
| 197 | # C — worker handler produces the same result shape; complete_job persists it |
| 198 | # --------------------------------------------------------------------------- |
| 199 | |
| 200 | |
| 201 | @pytest.mark.asyncio |
| 202 | async def test_process_push_unpack_job_matches_sync_result_and_is_persisted( |
| 203 | db_session: AsyncSession, |
| 204 | ) -> None: |
| 205 | from musehub.services.musehub_repository import create_repo |
| 206 | from musehub.core.genesis import compute_identity_id |
| 207 | |
| 208 | repo_row = await create_repo( |
| 209 | db_session, name="async-push-worker-test", owner="gabriel", |
| 210 | owner_user_id=compute_identity_id(b"gabriel"), visibility="public", |
| 211 | initialize=False, |
| 212 | ) |
| 213 | await db_session.commit() |
| 214 | |
| 215 | wire_bytes, cid = _tiny_mpack() |
| 216 | mpack_key = blob_id(wire_bytes) |
| 217 | from musehub.storage.backends import get_backend |
| 218 | await get_backend().put_mpack(mpack_key, wire_bytes) |
| 219 | |
| 220 | job_id = await enqueue_job( |
| 221 | db_session, repo_row.repo_id, "push.unpack", |
| 222 | { |
| 223 | "mpack_key": mpack_key, "pusher_id": "gabriel", "branch": "main", |
| 224 | "head_commit_id": cid, "commits_count": 1, "blobs_count": 1, |
| 225 | "force": False, |
| 226 | }, |
| 227 | dedup_payload_key="mpack_key", |
| 228 | ) |
| 229 | await db_session.commit() |
| 230 | assert job_id is not None |
| 231 | |
| 232 | result = await process_push_unpack_job(db_session, job_id) |
| 233 | await db_session.commit() |
| 234 | |
| 235 | assert result["head"] == cid |
| 236 | assert result["branch"] == "main" |
| 237 | assert result["blobs_written"] == 1 |
| 238 | assert result["commits_written"] == 1 |
| 239 | |
| 240 | await complete_job(db_session, job_id, result=result, attempt=await start_execution(db_session, job_id)) |
| 241 | await db_session.commit() |
| 242 | |
| 243 | job = await db_session.get(MusehubBackgroundJob, job_id) |
| 244 | assert job.status == "done" |
| 245 | assert job.result == result |
| 246 | |
| 247 | |
| 248 | # --------------------------------------------------------------------------- |
| 249 | # D — job-status endpoint: pending / done / failed |
| 250 | # --------------------------------------------------------------------------- |
| 251 | |
| 252 | |
| 253 | @pytest.mark.asyncio |
| 254 | async def test_job_status_pending_then_done( |
| 255 | client: AsyncClient, repo: dict, db_session: AsyncSession, |
| 256 | ) -> None: |
| 257 | job_id = await enqueue_job( |
| 258 | db_session, repo["repoId"], "push.unpack", |
| 259 | {"mpack_key": "sha256:" + "a" * 64, "branch": "main", "head_commit_id": "sha256:" + "b" * 64}, |
| 260 | ) |
| 261 | await db_session.commit() |
| 262 | |
| 263 | resp = await client.get(f"/gabriel/{repo['slug']}/push/job-status/{job_id}") |
| 264 | assert resp.status_code == 200 |
| 265 | assert resp.json()["status"] == "pending" |
| 266 | |
| 267 | await complete_job(db_session, job_id, result={"head": "sha256:" + "b" * 64, "blobs_written": 3}, attempt=await start_execution(db_session, job_id)) |
| 268 | await db_session.commit() |
| 269 | |
| 270 | resp = await client.get(f"/gabriel/{repo['slug']}/push/job-status/{job_id}") |
| 271 | body = resp.json() |
| 272 | assert body["status"] == "done" |
| 273 | assert body["result"]["blobs_written"] == 3 |
| 274 | |
| 275 | |
| 276 | @pytest.mark.asyncio |
| 277 | async def test_job_status_failed_includes_error( |
| 278 | client: AsyncClient, repo: dict, db_session: AsyncSession, |
| 279 | ) -> None: |
| 280 | from musehub.services.musehub_jobs import fail_job |
| 281 | |
| 282 | job_id = await enqueue_job( |
| 283 | db_session, repo["repoId"], "push.unpack", |
| 284 | {"mpack_key": "sha256:" + "c" * 64}, |
| 285 | ) |
| 286 | await db_session.commit() |
| 287 | await fail_job(db_session, job_id, "push rejected: not a fast-forward", attempt=await start_execution(db_session, job_id)) |
| 288 | # fail_job resets to pending while attempts remain -- exhaust them for this test. |
| 289 | await fail_job(db_session, job_id, "push rejected: not a fast-forward", attempt=await start_execution(db_session, job_id, attempt=99)) |
| 290 | await db_session.commit() |
| 291 | |
| 292 | resp = await client.get(f"/gabriel/{repo['slug']}/push/job-status/{job_id}") |
| 293 | body = resp.json() |
| 294 | assert body["status"] == "failed" |
| 295 | assert "not a fast-forward" in body["error"] |
| 296 | |
| 297 | |
| 298 | # --------------------------------------------------------------------------- |
| 299 | # E — job-status 404s across repo/job_type boundaries |
| 300 | # --------------------------------------------------------------------------- |
| 301 | |
| 302 | |
| 303 | @pytest.mark.asyncio |
| 304 | async def test_job_status_404_for_wrong_repo( |
| 305 | client: AsyncClient, repo: dict, db_session: AsyncSession, |
| 306 | ) -> None: |
| 307 | from musehub.services.musehub_repository import create_repo |
| 308 | from musehub.core.genesis import compute_identity_id |
| 309 | |
| 310 | other_repo = await create_repo( |
| 311 | db_session, name="other-repo-for-404-test", owner="gabriel", |
| 312 | owner_user_id=compute_identity_id(b"gabriel"), visibility="public", |
| 313 | initialize=False, |
| 314 | ) |
| 315 | await db_session.commit() |
| 316 | |
| 317 | job_id = await enqueue_job( |
| 318 | db_session, other_repo.repo_id, "push.unpack", |
| 319 | {"mpack_key": "sha256:" + "d" * 64}, |
| 320 | ) |
| 321 | await db_session.commit() |
| 322 | |
| 323 | # Polling via THIS repo's slug for a job that belongs to a different repo. |
| 324 | resp = await client.get(f"/gabriel/{repo['slug']}/push/job-status/{job_id}") |
| 325 | assert resp.status_code == 404 |
| 326 | |
| 327 | |
| 328 | # --------------------------------------------------------------------------- |
| 329 | # F — dedup on mpack_key: a retried request hands back the existing job |
| 330 | # --------------------------------------------------------------------------- |
| 331 | |
| 332 | |
| 333 | @pytest.mark.asyncio |
| 334 | async def test_duplicate_mpack_key_push_reuses_pending_job( |
| 335 | client: AsyncClient, repo: dict, db_session: AsyncSession, monkeypatch: pytest.MonkeyPatch, |
| 336 | ) -> None: |
| 337 | from musehub.config import get_settings |
| 338 | monkeypatch.setattr(get_settings(), "async_push_blob_threshold", 0) |
| 339 | |
| 340 | wire_bytes, cid = _tiny_mpack() |
| 341 | mpack_key = blob_id(wire_bytes) |
| 342 | from musehub.storage.backends import get_backend |
| 343 | await get_backend().put_mpack(mpack_key, wire_bytes) |
| 344 | |
| 345 | resp1 = await _push_unpack(client, repo["slug"], mpack_key, cid, blobs_count=1) |
| 346 | resp2 = await _push_unpack(client, repo["slug"], mpack_key, cid, blobs_count=1) |
| 347 | |
| 348 | job_id_1 = resp1.json()["job_id"] |
| 349 | job_id_2 = resp2.json()["job_id"] |
| 350 | assert job_id_1 == job_id_2, "retried push with the same mpack_key must reuse the pending job" |
| 351 | |
| 352 | count = (await db_session.execute( |
| 353 | select(MusehubBackgroundJob).where( |
| 354 | MusehubBackgroundJob.job_type == "push.unpack", |
| 355 | MusehubBackgroundJob.payload["mpack_key"].astext == mpack_key, |
| 356 | ) |
| 357 | )).scalars().all() |
| 358 | assert len(count) == 1, "must not create a duplicate job row for the same mpack_key" |
File History
4 commits
sha256:cb4b05e9d6caea5f351e3cb452ee6a1db8856169328ad96bfba9ee8e04b8afa6
fix(jobs): fence every execution of a job; an expired or su…
Sonnet 5.5
minor
⚠
4 days ago
sha256:383b9b15b1a7c18348bf797dbec252b8842199567b4b04371578fedd5f7891d4
merge: integrate push-ingest commit verification into dev
Opus 5.5
minor
⚠
11 days ago
sha256:39b90cebf3ab4c72ef67a29fe9d451ba1394084edf4053dff98878651f5d2378
test: build reproducible commit ids in push fixtures
Opus 5.5
11 days ago
sha256:5ea41f687b567512d786a8bff6f949aa626b4d52c7cdcebfa9e950d18b93e5a5
feat: defer large pushes to a background job (musehub#233)
Sonnet 5
minor
⚠
17 days ago