gabriel / musehub public
test_deployment.py python
903 lines 43.2 KB Hotspot Cold
Raw
sha256:d454adcf39faea9356d59c38007eac129e66c9939be75f21d0ecaf7e09acb4fc fix(deploy): free disk before the image pull — remove image… Opus 5.5 patch 1 day ago
1 """Section 7.3 — Deployment readiness tests.
2
3 Covers:
4 Zero-downtime : deploy.sh is blue-green (two slots, nginx flip); health
5 check URLs point to /healthz not a UI page.
6 /healthz : returns 200 when DB + storage healthy; 503 with JSON body
7 when either is down; exempt from auth; fast.
8 Non-root user : Dockerfile uses USER instruction (non-root); container
9 runs as the 'musehub' system user.
10 Read-only FS : docker-compose.yml sets read_only: true on musehub service;
11 /tmp is mounted as tmpfs; /data is a named volume.
12 Resource limits: CPU and memory limits set on musehub, postgres, runner.
13 """
14 from __future__ import annotations
15
16 import re
17 import shutil
18 import subprocess
19 from dataclasses import dataclass
20 from pathlib import Path
21 from unittest.mock import AsyncMock, MagicMock, patch
22
23 import pytest
24 from httpx import AsyncClient
25
26 _ROOT = Path(__file__).resolve().parents[1]
27 _DOCKERFILE = _ROOT / "Dockerfile"
28 _COMPOSE = _ROOT / "docker-compose.yml"
29 _DEPLOY_SH = _ROOT / "deploy" / "deploy.sh"
30 _PUSH_SH = _ROOT / "deploy" / "push.sh"
31 _PUBLISH_MUSE_RELEASE_SH = _ROOT / "deploy" / "publish_muse_release.sh"
32 _ACTIVATE_MUSE_RELEASE_SH = _ROOT / "deploy" / "activate_muse_release.sh"
33 _RELEASE_ENV_SH = _ROOT / "deploy" / "_release_env.sh"
34
35
36 # ═══════════════════════════════════════════════════════════════════════════════
37 # Zero-downtime deploy
38 # ═══════════════════════════════════════════════════════════════════════════════
39
40 class TestZeroDowntimeDeploy:
41 _src = _DEPLOY_SH.read_text()
42
43 def test_two_slots_defined(self) -> None:
44 """deploy.sh must define both blue and green slots."""
45 assert "blue" in self._src and "green" in self._src
46
47 def test_nginx_flip_present(self) -> None:
48 """deploy.sh must reload nginx after health check passes (atomic flip)."""
49 assert "nginx -s reload" in self._src or "nginx_point_to" in self._src
50
51 def test_health_check_before_nginx_flip(self) -> None:
52 """Health check must happen before the nginx flip — never flip a sick slot."""
53 src = self._src
54 health_pos = src.find("health_check")
55 nginx_pos = src.find("nginx_point_to")
56 assert health_pos != -1, "health_check function not found in deploy.sh"
57 assert nginx_pos != -1, "nginx_point_to not found in deploy.sh"
58 assert health_pos < nginx_pos, (
59 "nginx flip happens before health check — would route to an unhealthy slot"
60 )
61
62 def test_health_urls_point_to_healthz(self) -> None:
63 """deploy.sh must use /healthz not a UI page as the readiness signal."""
64 assert "/healthz" in self._src, (
65 "deploy.sh does not use /healthz — health check may pass even when "
66 "DB or storage is down (UI pages don't probe dependencies)"
67 )
68 assert "/explore" not in self._src, (
69 "deploy.sh still references /explore as a health URL — update to /healthz"
70 )
71
72 def test_old_slot_stopped_after_flip(self) -> None:
73 """deploy.sh must stop the old slot after the nginx flip to free resources."""
74 src = self._src
75 nginx_pos = src.find("nginx_point_to")
76 stop_pos = src.find("docker rm -f", nginx_pos)
77 assert stop_pos != -1, (
78 "deploy.sh does not stop the old slot after the nginx flip"
79 )
80
81 def test_dockerfile_healthcheck_uses_healthz(self) -> None:
82 """Dockerfile HEALTHCHECK must probe /healthz."""
83 src = _DOCKERFILE.read_text()
84 hc_lines = [l for l in src.splitlines() if "HEALTHCHECK" in l or l.strip().startswith("CMD")]
85 combined = " ".join(hc_lines)
86 assert "/healthz" in combined, (
87 "Dockerfile HEALTHCHECK does not probe /healthz — "
88 "docker will report 'healthy' even when DB is down"
89 )
90
91
92 # ═══════════════════════════════════════════════════════════════════════════════
93 # /healthz endpoint
94 # ═══════════════════════════════════════════════════════════════════════════════
95
96 class TestHealthzEndpoint:
97 async def test_healthz_returns_200_when_healthy(self, client: AsyncClient) -> None:
98 """GET /healthz must return 200 when DB and storage are reachable."""
99 resp = await client.get("/healthz")
100 assert resp.status_code == 200
101 body = resp.json()
102 assert body["status"] == "ok"
103 assert body["db"] is True
104 assert body["storage"] is True
105
106 async def test_healthz_returns_json(self, client: AsyncClient) -> None:
107 resp = await client.get("/healthz")
108 assert resp.headers["content-type"].startswith("application/json")
109
110 async def test_healthz_no_auth_required(self, client: AsyncClient) -> None:
111 """Healthz must be reachable without any Authorization header."""
112 resp = await client.get("/healthz")
113 # Must not be 401 or 403
114 assert resp.status_code not in (401, 403), (
115 f"/healthz returned {resp.status_code} — health check must be unauthenticated"
116 )
117
118 async def test_healthz_503_when_db_down(self, client: AsyncClient) -> None:
119 """GET /healthz must return 503 when the DB probe fails."""
120 from sqlalchemy.exc import OperationalError
121
122 # Patch the DB execute to simulate a broken connection
123 with patch(
124 "musehub.main.AsyncSession.execute",
125 new_callable=AsyncMock,
126 side_effect=OperationalError("connection refused", None, None),
127 ):
128 resp = await client.get("/healthz")
129
130 assert resp.status_code == 503
131 body = resp.json()
132 assert body["status"] == "unhealthy"
133 assert body["db"] is False
134
135 async def test_healthz_503_when_storage_down(self, client: AsyncClient) -> None:
136 """GET /healthz must return 503 when the storage probe fails."""
137 from musehub.storage.backends import BlobBackend
138
139 # Backend pointing at an unreachable endpoint — head_bucket will fail.
140 bad_backend = BlobBackend(
141 bucket="muse-objects",
142 endpoint_url="http://127.0.0.1:19999", # nothing listening here
143 access_key_id="x",
144 secret_access_key="x",
145 region="us-east-1",
146 )
147
148 with patch("musehub.storage.backends.get_backend", return_value=bad_backend):
149 resp = await client.get("/healthz")
150
151 assert resp.status_code == 503
152 body = resp.json()
153 assert body["status"] == "unhealthy"
154 assert body["storage"] is False
155
156 async def test_healthz_body_has_db_and_storage_keys(self, client: AsyncClient) -> None:
157 """Response body must expose both db and storage status for monitoring."""
158 resp = await client.get("/healthz")
159 body = resp.json()
160 assert "db" in body, "healthz response missing 'db' key"
161 assert "storage" in body, "healthz response missing 'storage' key"
162
163 async def test_healthz_fast(self, client: AsyncClient) -> None:
164 """Healthz must respond in under 2 s (load balancer timeout is typically 5 s)."""
165 import time
166 start = time.monotonic()
167 await client.get("/healthz")
168 elapsed = time.monotonic() - start
169 assert elapsed < 2.0, f"/healthz took {elapsed:.2f}s — too slow for a probe"
170
171 def test_healthz_route_registered(self) -> None:
172 """The /healthz route must be registered in the FastAPI app."""
173 from musehub.main import app
174 paths = [route.path for route in app.routes]
175 assert "/healthz" in paths, "/healthz route not registered in app"
176
177
178 # ═══════════════════════════════════════════════════════════════════════════════
179 # Non-root container user
180 # ═══════════════════════════════════════════════════════════════════════════════
181
182 class TestNonRootUser:
183 _src = _DOCKERFILE.read_text()
184
185 def test_dockerfile_has_user_instruction(self) -> None:
186 """Dockerfile must have a USER instruction."""
187 user_lines = [l.strip() for l in self._src.splitlines()
188 if l.strip().upper().startswith("USER ")]
189 assert user_lines, "Dockerfile has no USER instruction — container runs as root"
190
191 def test_dockerfile_user_is_not_root(self) -> None:
192 """Dockerfile USER must not be root or UID 0."""
193 user_lines = [l.strip() for l in self._src.splitlines()
194 if l.strip().upper().startswith("USER ")]
195 for line in user_lines:
196 user = line.split()[1].lower()
197 assert user not in ("root", "0"), (
198 f"Dockerfile sets USER to {user!r} — container must run as non-root"
199 )
200
201 def test_dockerfile_creates_system_user(self) -> None:
202 """Dockerfile must create a dedicated system user (groupadd + useradd)."""
203 assert "groupadd" in self._src and "useradd" in self._src, (
204 "Dockerfile does not create a dedicated system user"
205 )
206
207 def test_dockerfile_user_applied_after_installs(self) -> None:
208 """USER instruction must come after RUN pip install (installs need root)."""
209 lines = self._src.splitlines()
210 user_idx = next(
211 (i for i, l in enumerate(lines) if l.strip().upper().startswith("USER ")), None
212 )
213 pip_idx = max(
214 (i for i, l in enumerate(lines) if "pip install" in l), default=None
215 )
216 assert user_idx is not None and pip_idx is not None
217 assert user_idx > pip_idx, (
218 "USER instruction appears before pip install — "
219 "package installation would fail without root"
220 )
221
222
223 # ═══════════════════════════════════════════════════════════════════════════════
224 # Read-only filesystem
225 # ═══════════════════════════════════════════════════════════════════════════════
226
227 class TestReadOnlyFilesystem:
228 def _parse_yaml(self) -> None:
229 import yaml
230 return yaml.safe_load(_COMPOSE.read_text())
231
232 def test_musehub_service_read_only(self) -> None:
233 """musehub service must have read_only: true."""
234 src = _COMPOSE.read_text()
235 # Structural check: read_only appears in the musehub service block
236 # Find musehub service block (between 'musehub:' and the next top-level key)
237 in_musehub = False
238 for line in src.splitlines():
239 if re.match(r'^ musehub:', line):
240 in_musehub = True
241 elif re.match(r'^ \w', line) and in_musehub:
242 in_musehub = False
243 if in_musehub and "read_only: true" in line:
244 return
245 pytest.fail(
246 "musehub service in docker-compose.yml does not have read_only: true"
247 )
248
249 def test_tmp_is_tmpfs_or_volume(self) -> None:
250 """/tmp must be writable (tmpfs or volume) so uvicorn can write temp files."""
251 src = _COMPOSE.read_text()
252 assert "tmpfs" in src or "/tmp" in src, (
253 "No tmpfs mount for /tmp — uvicorn and Python will fail to write temp files "
254 "when the root filesystem is read-only"
255 )
256
257 def test_data_volume_is_explicit(self) -> None:
258 """/data object store must be an explicit named volume (not read-only)."""
259 src = _COMPOSE.read_text()
260 assert "musehub_data:/data" in src, (
261 "/data is not mounted as an explicit volume — objects cannot be written "
262 "when the root filesystem is read-only"
263 )
264
265
266 # ═══════════════════════════════════════════════════════════════════════════════
267 # Resource limits
268 # ═══════════════════════════════════════════════════════════════════════════════
269
270 class TestResourceLimits:
271 _src = _COMPOSE.read_text()
272
273 def _service_limits(self, service_name: str) -> str:
274 """Extract the text block for a given service."""
275 lines = self._src.splitlines()
276 in_service = False
277 block_lines = []
278 for line in lines:
279 if re.match(rf'^ {re.escape(service_name)}:', line):
280 in_service = True
281 elif re.match(r'^ \w', line) and in_service:
282 break
283 if in_service:
284 block_lines.append(line)
285 return "\n".join(block_lines)
286
287 def test_musehub_has_cpu_limit(self) -> None:
288 block = self._service_limits("musehub")
289 assert "cpus:" in block, "musehub service has no CPU limit"
290
291 def test_musehub_has_memory_limit(self) -> None:
292 block = self._service_limits("musehub")
293 assert "memory:" in block, "musehub service has no memory limit"
294
295 def test_musehub_memory_limit_sane(self) -> None:
296 """musehub memory limit must be ≥ 256 MiB (app needs headroom)."""
297 block = self._service_limits("musehub")
298 m = re.search(r'memory:\s*(\d+)([MmGg])', block)
299 if m:
300 amount = int(m.group(1))
301 unit = m.group(2).upper()
302 mb = amount * 1024 if unit == "G" else amount
303 assert mb >= 256, f"musehub memory limit {mb}M is below 256M minimum"
304
305 def test_postgres_has_cpu_limit(self) -> None:
306 block = self._service_limits("postgres")
307 assert "cpus:" in block, "postgres service has no CPU limit"
308
309 def test_postgres_has_memory_limit(self) -> None:
310 block = self._service_limits("postgres")
311 assert "memory:" in block, "postgres service has no memory limit"
312
313 def test_runner_has_cpu_limit(self) -> None:
314 block = self._service_limits("musehub-runner")
315 assert "cpus:" in block, "musehub-runner service has no CPU limit"
316
317 def test_runner_has_memory_limit(self) -> None:
318 block = self._service_limits("musehub-runner")
319 assert "memory:" in block, "musehub-runner service has no memory limit"
320
321 def test_all_services_have_deploy_block(self) -> None:
322 """All three services must have a deploy: block (where limits live)."""
323 for svc in ("musehub", "postgres", "musehub-runner"):
324 block = self._service_limits(svc)
325 assert "deploy:" in block, f"{svc} service has no deploy: block"
326
327
328 # ═══════════════════════════════════════════════════════════════════════════════
329 # musehub#194 — ECR_REGISTRY must derive from ECR_IMAGE, never be independently
330 # hardcoded. A "prod" deploy passes ECR_IMAGE pointing at Production's registry
331 # (see push.sh's per-environment ECR_REGISTRY map) — deploy.sh's docker
332 # login/logout must authenticate against that SAME registry, not a hardcoded
333 # Nonproduction one, or `docker pull` fails with an auth mismatch.
334 # ═══════════════════════════════════════════════════════════════════════════════
335
336 class TestEcrRegistryDerivedFromImage:
337 """Executes the real variable-derivation lines from deploy.sh in a
338 subprocess (not just a text/regex check) — this is a bash variable
339 scoping bug, so the test proves actual runtime behavior."""
340
341 _STAGING_IMAGE = "992382692655.dkr.ecr.us-east-1.amazonaws.com/musehub/musehub"
342 _PROD_IMAGE = "672469410277.dkr.ecr.us-east-1.amazonaws.com/musehub/musehub"
343
344 @staticmethod
345 def _extract_var_block() -> str:
346 """Return the exact source lines from ECR_IMAGE's definition through
347 FULL_IMAGE's definition — the minimal slice needed to resolve
348 ECR_REGISTRY/ECR_IMAGE/FULL_IMAGE, without deploy.sh's logging setup,
349 `cd`, or anything requiring a real /opt/musehub."""
350 src = _DEPLOY_SH.read_text()
351 start_marker = 'ECR_IMAGE="${ECR_IMAGE:-'
352 end_marker = 'FULL_IMAGE="${ECR_IMAGE}:${IMAGE_TAG}"'
353 start = src.index(start_marker)
354 # Back up to the start of that line (the marker may not be at col 0).
355 line_start = src.rfind("\n", 0, start) + 1
356 end = src.index(end_marker) + len(end_marker)
357 return src[line_start:end]
358
359 def _resolve(self, ecr_image: str | None) -> dict[str, str]:
360 block = self._extract_var_block()
361 script = (
362 f"{block}\n"
363 'echo "ECR_REGISTRY=$ECR_REGISTRY"\n'
364 'echo "ECR_IMAGE=$ECR_IMAGE"\n'
365 'echo "FULL_IMAGE=$FULL_IMAGE"\n'
366 )
367 env: dict[str, str] = {"PATH": "/usr/bin:/bin", "IMAGE_TAG": "abc123"}
368 if ecr_image is not None:
369 env["ECR_IMAGE"] = ecr_image
370 result = subprocess.run(
371 ["bash", "-c", script],
372 env=env,
373 capture_output=True,
374 text=True,
375 timeout=5,
376 )
377 assert result.returncode == 0, result.stderr
378 out: dict[str, str] = {}
379 for line in result.stdout.splitlines():
380 key, _, value = line.partition("=")
381 out[key] = value
382 return out
383
384 def test_staging_registry_derived_from_ecr_image(self) -> None:
385 resolved = self._resolve(self._STAGING_IMAGE)
386 assert resolved["ECR_REGISTRY"] == "992382692655.dkr.ecr.us-east-1.amazonaws.com"
387
388 def test_production_registry_derived_from_ecr_image(self) -> None:
389 """musehub#194: the exact regression — a prod ECR_IMAGE must resolve
390 to Production's registry, never the hardcoded staging one."""
391 resolved = self._resolve(self._PROD_IMAGE)
392 assert resolved["ECR_REGISTRY"] == "672469410277.dkr.ecr.us-east-1.amazonaws.com"
393 assert resolved["ECR_REGISTRY"] != "992382692655.dkr.ecr.us-east-1.amazonaws.com"
394
395 def test_full_image_matches_passed_ecr_image_and_tag(self) -> None:
396 resolved = self._resolve(self._PROD_IMAGE)
397 assert resolved["FULL_IMAGE"] == f"{self._PROD_IMAGE}:abc123"
398
399 def test_default_ecr_image_falls_back_to_staging_registry(self) -> None:
400 """No ECR_IMAGE set (manual on-instance invocation) defaults to
401 staging — matches the script's documented manual-use fallback."""
402 resolved = self._resolve(None)
403 assert resolved["ECR_REGISTRY"] == "992382692655.dkr.ecr.us-east-1.amazonaws.com"
404
405 def test_no_independently_hardcoded_production_or_staging_registry_constant(self) -> None:
406 """Guards against reintroducing a second, independent ECR_REGISTRY
407 source of truth — the account ID must only ever appear as part of
408 ECR_IMAGE's default value, never as a bare ECR_REGISTRY="..." literal."""
409 src = _DEPLOY_SH.read_text()
410 assert not re.search(r'^ECR_REGISTRY="\d', src, re.MULTILINE), (
411 "ECR_REGISTRY is hardcoded to a literal account ID again — it must "
412 "be derived from ECR_IMAGE so docker login and docker pull always "
413 "target the same registry."
414 )
415
416
417 # ═══════════════════════════════════════════════════════════════════════════════
418 # musehub#199 — push.sh bash 3.2 (macOS default) compatibility
419 # ═══════════════════════════════════════════════════════════════════════════════
420
421 _SYSTEM_BASH = Path("/bin/bash")
422
423
424 def _system_bash_major_version() -> int | None:
425 """The major version of whatever /bin/bash actually is on this machine.
426 On macOS this is the real, un-upgradeable system bash 3.2 — the exact
427 binary the reported bug ran under. Returns None if /bin/bash is absent."""
428 if not _SYSTEM_BASH.exists():
429 return None
430 result = subprocess.run(
431 [str(_SYSTEM_BASH), "-c", "echo ${BASH_VERSINFO[0]}"],
432 capture_output=True,
433 text=True,
434 timeout=5,
435 )
436 try:
437 return int(result.stdout.strip())
438 except ValueError:
439 return None
440
441
442 def _extract_function(src: str, name: str) -> str:
443 """Return one `name() { ... }` function body verbatim from push.sh."""
444 start = src.index(f"{name}() {{")
445 end = src.index("\n}", start) + len("\n}")
446 return src[start:end]
447
448
449 class TestPushShBash32Compat:
450 """push.sh used `declare -A` (associative arrays, bash 4+ only) with no
451 version guard, so it crashed outright under macOS's default /bin/bash
452 3.2 rather than a Homebrew-installed bash 4/5 — 'unbound variable' at
453 the first array reference, before the script did anything useful.
454 Rewritten as `case` dispatch functions, which need no minimum bash
455 version at all."""
456
457 def test_no_declare_a_in_push_sh(self) -> None:
458 src = _PUSH_SH.read_text()
459 assert not re.search(r"^\s*declare -A", src, re.MULTILINE), (
460 "push.sh uses declare -A again, which requires bash 4+ and "
461 "crashes outright under macOS's default bash 3.2."
462 )
463
464 @pytest.mark.skipif(
465 _system_bash_major_version() is None or _system_bash_major_version() >= 4,
466 reason="requires a real bash < 4 (e.g. macOS's /bin/bash) to prove the fix",
467 )
468 def test_usage_message_prints_under_real_bash_32(self) -> None:
469 """The exact reported failure: `bash deploy/push.sh` with no args
470 used to crash at `declare -A` with 'unbound variable' before ever
471 reaching the usage message. Runs the real script end to end."""
472 result = subprocess.run(
473 [str(_SYSTEM_BASH), str(_PUSH_SH)],
474 capture_output=True,
475 text=True,
476 timeout=10,
477 )
478 assert result.returncode == 1
479 assert "Usage: bash deploy/push.sh" in result.stdout
480 assert "unbound variable" not in result.stderr
481
482 @pytest.mark.skipif(
483 _system_bash_major_version() is None or _system_bash_major_version() >= 4,
484 reason="requires a real bash < 4 (e.g. macOS's /bin/bash) to prove the fix",
485 )
486 def test_env_helper_functions_resolve_correctly_under_bash_32(self) -> None:
487 """The replacement case-dispatch helpers must resolve the exact same
488 values the old associative arrays held, not just avoid crashing."""
489 src = _PUSH_SH.read_text()
490 block = "\n".join(
491 _extract_function(src, name)
492 for name in ("instance_id_for", "ecr_registry_for", "aws_profile_for")
493 )
494 script = (
495 f"{block}\n"
496 'echo "INSTANCE_STAGING=$(instance_id_for staging)"\n'
497 'echo "INSTANCE_PROD=$(instance_id_for prod)"\n'
498 'echo "REGISTRY_STAGING=$(ecr_registry_for staging)"\n'
499 'echo "REGISTRY_PROD=$(ecr_registry_for prod)"\n'
500 'echo "PROFILE_STAGING=$(aws_profile_for staging)"\n'
501 'echo "PROFILE_PROD=$(aws_profile_for prod)"\n'
502 )
503 result = subprocess.run(
504 [str(_SYSTEM_BASH), "-c", script],
505 capture_output=True,
506 text=True,
507 timeout=5,
508 )
509 assert result.returncode == 0, result.stderr
510 out = dict(line.split("=", 1) for line in result.stdout.splitlines())
511 assert out["INSTANCE_STAGING"] == "i-07547cd20bee2dea5"
512 assert out["INSTANCE_PROD"] == "i-043aaed71bef11903"
513 assert out["REGISTRY_STAGING"] == "992382692655.dkr.ecr.us-east-1.amazonaws.com"
514 assert out["REGISTRY_PROD"] == "672469410277.dkr.ecr.us-east-1.amazonaws.com"
515 assert out["PROFILE_STAGING"] == "musehub-nonproduction"
516 assert out["PROFILE_PROD"] == "musehub-production"
517
518
519 # ═══════════════════════════════════════════════════════════════════════════════
520 # publish_muse_release.sh — production must use the musehub-production SSO
521 # profile for its SSM calls, not the ambient default credentials
522 # ═══════════════════════════════════════════════════════════════════════════════
523
524 class TestPublishMuseReleaseProfile:
525 """publish_muse_release.sh's SSM calls had no --profile at all, so they
526 silently ran under whatever the ambient default AWS credentials happened
527 to be (musehub-infra, the Nonproduction/staging account) regardless of
528 which environment was requested. Production's instance lives in a
529 separate AWS account with no static IAM user by design, so every SSM
530 call against production failed with a confusing
531 'InvalidInstanceId ... not in a valid state for account' error that
532 looks like a broken instance rather than a missing --profile."""
533
534 def _resolve_profile_args(self, env: str) -> str:
535 """Source the shared env resolver (no AWS/network) and print PROFILE_ARGS."""
536 script = (
537 f'. "{_RELEASE_ENV_SH}"\n'
538 f'resolve_release_env "{env}"\n'
539 'echo "PROFILE_ARGS=${PROFILE_ARGS[@]+"${PROFILE_ARGS[@]}"}"\n'
540 )
541 result = subprocess.run(["bash", "-c", script], capture_output=True, text=True, timeout=5)
542 assert result.returncode == 0, result.stderr
543 line = next(l for l in result.stdout.splitlines() if l.startswith("PROFILE_ARGS="))
544 return line.removeprefix("PROFILE_ARGS=")
545
546 def test_production_resolves_musehub_production_profile(self) -> None:
547 assert self._resolve_profile_args("production") == "--profile musehub-production"
548
549 def test_staging_resolves_no_profile(self) -> None:
550 """Staging's instance is in the same account as the ambient default
551 credentials (musehub-infra) — must stay unset, not regress to also
552 requiring an explicit profile."""
553 assert self._resolve_profile_args("staging") == ""
554
555 def test_every_ssm_call_site_passes_profile_args(self) -> None:
556 """Guards against reintroducing the exact regression this fixes: a
557 new `aws ssm send-command` or `aws ssm get-command-invocation` call
558 added later without ``"${PROFILE_ARGS[@]}"`` would silently run
559 under the wrong account for production again."""
560 src = _RELEASE_ENV_SH.read_text()
561 ssm_call_count = len(re.findall(r"aws ssm (?:send-command|get-command-invocation)", src))
562 profile_args_count = src.count('${PROFILE_ARGS[@]+"${PROFILE_ARGS[@]}"}')
563 assert ssm_call_count > 0, "expected to find aws ssm calls in _release_env.sh"
564 assert profile_args_count == ssm_call_count, (
565 f"found {ssm_call_count} aws ssm call(s) but only {profile_args_count} "
566 "PROFILE_ARGS usage(s) — every SSM call must pass it"
567 )
568 for script in (_PUBLISH_MUSE_RELEASE_SH, _ACTIVATE_MUSE_RELEASE_SH):
569 assert "aws ssm" not in script.read_text(), f"{script.name} must go through ssm_run"
570
571 def test_s3_calls_do_not_use_profile_args(self) -> None:
572 """The release bucket is a single shared, public-read bucket (per
573 the script's own header comment) — S3 calls must keep using the
574 ambient default credentials for both environments, not be
575 accidentally scoped to $PROFILE_ARGS."""
576 src = _PUBLISH_MUSE_RELEASE_SH.read_text()
577 for line in src.splitlines():
578 if line.strip().startswith("aws s3 "):
579 assert "PROFILE_ARGS" not in line
580
581 def test_remote_download_uses_no_sign_request(self) -> None:
582 """musehub#(none yet): discovered live against production -- an
583 authenticated `aws s3 cp` using the instance's own IAM role got a
584 403 (an account-level guardrail denying s3:GetObject outright,
585 overriding the bucket's public Allow), while an anonymous request
586 succeeded. The remote instance-side download of the release
587 tarball must use --no-sign-request so it never depends on the
588 instance role having S3 permissions at all -- matching how
589 install.sh itself fetches releases with no credentials."""
590 src = _PUBLISH_MUSE_RELEASE_SH.read_text()
591 match = re.search(r'CMD="aws s3 cp ([^\n]*)', src)
592 assert match is not None, "expected to find the remote CMD's aws s3 cp invocation"
593 assert "--no-sign-request" in match.group(1)
594
595
596 # ═══════════════════════════════════════════════════════════════════════════════
597 # deploy.sh step 2 fails closed — a failed or unverifiable migration stops the
598 # deploy before the new slot starts, and nothing marks a revision as applied
599 # without running it. The previous fallback ran `alembic stamp --purge head`
600 # whenever `alembic upgrade head` failed, so a failed migration was recorded as
601 # applied and could ship behind a passing schema gate.
602 # ═══════════════════════════════════════════════════════════════════════════════
603
604 _BASH = shutil.which("bash")
605 _HARNESS_IMAGE = "992382692655.dkr.ecr.us-east-1.amazonaws.com/musehub/musehub"
606 _HARNESS_TAG = "abc12345-20261105000000"
607 _HARNESS_FULL_IMAGE = f"{_HARNESS_IMAGE}:{_HARNESS_TAG}"
608
609 # Stand-ins for the host commands deploy.sh calls. Every call is appended to
610 # $FAKE_DIR/calls.log. The fake database's Alembic revision is $FAKE_DIR/db_rev
611 # (absent: no revision). Output and exit codes follow Alembic 1.18: a
612 # CommandError prints "FAILED: …" on stdout and exits 255; an exception in a
613 # migration or a lost connection exits 1 with the error on stderr.
614 _DEPLOY_STUBS: dict[str, str] = {
615 "sudo": 'exec "$@"\n',
616 "aws": 'echo "aws $*" >> "$FAKE_DIR/calls.log"\necho fake-token\n',
617 "sleep": ":\n",
618 "nginx": 'echo "nginx $*" >> "$FAKE_DIR/calls.log"\n',
619 "musehub-set-slot": 'echo "SWITCH $1" >> "$FAKE_DIR/calls.log"\n',
620 "curl": """\
621 echo "curl $*" >> "$FAKE_DIR/calls.log"
622 case "$*" in
623 */healthz*) [ "${FAKE_HEALTH:-ok}" = ok ] ;;
624 */static/*) printf '200 1234' ;;
625 *) exit 7 ;;
626 esac
627 """,
628 "docker": """\
629 echo "docker $*" >> "$FAKE_DIR/calls.log"
630 [ "$1" != login ] || cat > /dev/null
631 [ "$1" = run ] || exit 0 # login, logout, pull, rm, stop, rmi; images lists nothing
632 shift
633 detached=0
634 while [ $# -gt 0 ] && [ "$1" != "$FAKE_IMAGE" ]; do
635 [ "$1" != -d ] || detached=1
636 shift
637 done
638 [ $# -gt 0 ] || { echo "docker stub: $FAKE_IMAGE is not in the arguments" >&2; exit 98; }
639 shift
640 [ "$detached" = 0 ] || { echo fake-container-id; exit 0; }
641
642 head="$(printf '%s\\n' "$FAKE_HEADS" | awk 'NF { print $1; exit }')"
643 nheads="$(printf '%s\\n' "$FAKE_HEADS" | awk 'NF { n++ } END { print n + 0 }')"
644 rev="$(cat "$FAKE_DIR/db_rev" 2>/dev/null || true)"
645 multiple_heads() { echo "FAILED: Multiple head revisions are present for given argument 'head'"; exit 255; }
646 unreachable() { echo "sqlalchemy.exc.OperationalError: connection refused" >&2; exit 1; }
647 in_history() { case " $FAKE_HISTORY " in *" $1 "*) return 0 ;; esac; return 1; }
648
649 case "$*" in
650 "alembic heads")
651 [ -z "$FAKE_HEADS" ] || printf '%s\\n' "$FAKE_HEADS"
652 exit "${FAKE_HEADS_RC:-0}" ;;
653 "alembic current")
654 n=$(( $(cat "$FAKE_DIR/current_calls" 2>/dev/null || echo 0) + 1 ))
655 echo "$n" > "$FAKE_DIR/current_calls"
656 mode="$(echo "${FAKE_CURRENT:-}" | awk -v n="$n" '{ print $n }')"
657 case "${mode:-ok}" in
658 unreachable) unreachable ;;
659 garbage) echo "something unexpected"; exit 0 ;;
660 esac
661 if [ -z "$rev" ]; then exit 0
662 elif [ "$rev" = "$head" ]; then echo "$rev (head)"
663 elif in_history "$rev"; then echo "$rev"
664 else echo "FAILED: Can't locate revision identified by '$rev'"; exit 255
665 fi ;;
666 "alembic upgrade head")
667 [ "$nheads" -le 1 ] || multiple_heads
668 [ "$rev" != "$head" ] || exit 0 # nothing to run, as after a stamp
669 case "${FAKE_UPGRADE:-ok}" in
670 ok) echo "$head" > "$FAKE_DIR/db_rev" ;;
671 fail) echo "RuntimeError: simulated migration failure" >&2; exit 1 ;;
672 unreachable) unreachable ;;
673 behind:*) echo "${FAKE_UPGRADE#behind:}" > "$FAKE_DIR/db_rev" ;;
674 esac ;;
675 alembic\\ stamp*)
676 [ "$nheads" -le 1 ] || multiple_heads
677 echo "$head" > "$FAKE_DIR/db_rev" ;;
678 "python -m musehub.db.schema_gate")
679 exit "${FAKE_SCHEMA_GATE_RC:-0}" ;;
680 *)
681 echo "docker stub: unexpected command: $*" >&2
682 exit 99 ;;
683 esac
684 """,
685 }
686
687
688 @dataclass
689 class _DeployRun:
690 returncode: int
691 output: str
692 calls: list[str]
693 db_rev: str | None
694
695 def ran(self, command: str) -> int:
696 """How many one-off containers ran ``command`` on the image."""
697 return sum(c.endswith(f"{_HARNESS_FULL_IMAGE} {command}") for c in self.calls)
698
699 def index(self, fragment: str) -> int:
700 return next(i for i, c in enumerate(self.calls) if fragment in c)
701
702 @property
703 def stamped(self) -> bool:
704 return any("alembic stamp" in c for c in self.calls)
705
706 @property
707 def started_new_slot(self) -> bool:
708 return any("run -d --name musehub-green" in c for c in self.calls)
709
710 @property
711 def switched(self) -> bool:
712 return any(c.startswith("SWITCH") for c in self.calls)
713
714 @property
715 def touched_old_slot_or_worker(self) -> bool:
716 return any("musehub-blue" in c or "musehub-worker" in c for c in self.calls)
717
718
719 def _harness_deploy_sh(root: Path) -> Path:
720 """deploy.sh with its fixed host paths moved under ``root``."""
721 src = _DEPLOY_SH.read_text()
722 for host_path in ('"/opt/musehub"', '"/tmp/musehub-deploy.log"', '"/etc/nginx/'):
723 assert host_path in src, f"deploy.sh no longer contains {host_path}; update the harness"
724 src = src.replace(host_path, f'"{root}{host_path[1:]}')
725 assert host_path not in src
726 script = root / "deploy.sh"
727 script.write_text(src)
728 return script
729
730
731 def _run_deploy(root: Path, db_rev: str | None, **fake: str) -> _DeployRun:
732 """Run deploy.sh (blue active, deploying green) against the stubs."""
733 app_dir = root / "opt" / "musehub"
734 app_dir.mkdir(parents=True)
735 (app_dir / ".active-slot").write_text("blue")
736 (app_dir / ".env").write_text("")
737 (root / "tmp").mkdir()
738 fake_dir = root / "fake"
739 fake_dir.mkdir()
740 (fake_dir / "calls.log").write_text("")
741 if db_rev is not None:
742 (fake_dir / "db_rev").write_text(f"{db_rev}\n")
743 bin_dir = root / "bin"
744 bin_dir.mkdir()
745 for name, body in _DEPLOY_STUBS.items():
746 stub = bin_dir / name
747 stub.write_text(f"#!/bin/bash\n{body}")
748 stub.chmod(0o755)
749 env = {
750 "PATH": f"{bin_dir}:/usr/bin:/bin",
751 "HOME": str(root),
752 "ECR_IMAGE": _HARNESS_IMAGE,
753 "IMAGE_TAG": _HARNESS_TAG,
754 "FAKE_DIR": str(fake_dir),
755 "FAKE_IMAGE": _HARNESS_FULL_IMAGE,
756 "FAKE_HEADS": "0084 (head)",
757 "FAKE_HISTORY": "0082 0083",
758 **fake,
759 }
760 assert _BASH is not None
761 result = subprocess.run(
762 [_BASH, str(_harness_deploy_sh(root))],
763 env=env,
764 capture_output=True,
765 text=True,
766 timeout=60,
767 )
768 db_rev_file = fake_dir / "db_rev"
769 return _DeployRun(
770 returncode=result.returncode,
771 output=result.stdout + result.stderr,
772 calls=(fake_dir / "calls.log").read_text().splitlines(),
773 db_rev=db_rev_file.read_text().strip() if db_rev_file.exists() else None,
774 )
775
776
777 @pytest.mark.skipif(_BASH is None, reason="requires bash")
778 class TestMigrationStepFailsClosed:
779 """Runs the real deploy.sh end to end with docker, sudo, aws, curl and the
780 nginx slot switch replaced by stubs, so the order of what it does — and
781 what it never does — is observed, not inferred from the source text."""
782
783 def test_database_at_head_deploys_without_running_a_migration(self, tmp_path: Path) -> None:
784 run = _run_deploy(tmp_path, "0084")
785 assert run.returncode == 0, run.output
786 assert "no migration to run" in run.output
787 assert run.ran("alembic upgrade head") == 0
788 assert run.ran("python -m musehub.db.schema_gate") == 1
789 assert not run.stamped
790 assert run.switched and "Deploy complete" in run.output
791 assert run.index("SWITCH green") < run.index("stop --time 15 musehub-blue")
792 assert run.index("stop --time 15 musehub-blue") < run.index("run -d --name musehub-worker")
793
794 def test_pending_migration_is_applied_and_verified_before_traffic_moves(self, tmp_path: Path) -> None:
795 run = _run_deploy(tmp_path, "0083")
796 assert run.returncode == 0, run.output
797 assert run.db_rev == "0084"
798 assert run.ran("alembic upgrade head") == 1
799 assert run.ran("alembic current") == 2, "the head must be re-read after the upgrade"
800 assert not run.stamped
801 upgrade = run.index(f"{_HARNESS_FULL_IMAGE} alembic upgrade head")
802 gate = run.index(f"{_HARNESS_FULL_IMAGE} python -m musehub.db.schema_gate")
803 assert upgrade < gate < run.index("run -d --name musehub-green") < run.index("SWITCH green")
804
805 @pytest.mark.parametrize(
806 ("db_rev", "fake", "message"),
807 [
808 pytest.param("0083", {"FAKE_UPGRADE": "fail"}, "alembic upgrade head failed (exit 1)", id="upgrade-fails"),
809 pytest.param("0083", {"FAKE_UPGRADE": "unreachable"}, "alembic upgrade head failed (exit 1)", id="connection-lost-during-upgrade"),
810 pytest.param("0083", {"FAKE_CURRENT": "unreachable"}, "cannot read the database's Alembic revision", id="database-unreachable"),
811 pytest.param("0082", {"FAKE_UPGRADE": "behind:0083"}, "is at 0083, not image head 0084", id="upgrade-leaves-database-behind-head"),
812 pytest.param("0083", {"FAKE_HEADS": "0084 (head)\n0084b (head)"}, "more than one Alembic head (0084, 0084b)", id="multiple-heads"),
813 pytest.param("0083", {"FAKE_HEADS": ""}, "has no Alembic head", id="no-head"),
814 pytest.param("0083", {"FAKE_HEADS": "FAILED: Path doesn't exist", "FAKE_HEADS_RC": "255"}, "cannot read the migration graph", id="heads-lookup-fails"),
815 pytest.param("0085", {}, "cannot read the database's Alembic revision", id="database-at-revision-unknown-to-image"),
816 pytest.param("0084", {"FAKE_CURRENT": "garbage"}, "unexpected alembic current output", id="revision-output-unparseable"),
817 pytest.param("0083", {"FAKE_CURRENT": "ok unreachable"}, "cannot read the database's Alembic revision", id="revision-lookup-fails-after-upgrade"),
818 pytest.param(None, {}, "the database has no Alembic revision", id="database-has-no-revision"),
819 pytest.param("0084", {"FAKE_SCHEMA_GATE_RC": "1"}, "Schema gate failed", id="schema-gate-fails"),
820 pytest.param("0083", {"FAKE_SCHEMA_GATE_RC": "1"}, "Schema gate failed", id="schema-gate-fails-after-migration"),
821 ],
822 )
823 def test_failure_stops_the_deploy_before_the_new_slot(
824 self, tmp_path: Path, db_rev: str | None, fake: dict[str, str], message: str
825 ) -> None:
826 run = _run_deploy(tmp_path, db_rev, **fake)
827 assert run.returncode != 0, run.output
828 assert message in run.output, run.output
829 assert not run.stamped, "a revision was marked as applied without running it"
830 assert not run.started_new_slot
831 assert not run.switched
832 assert not run.touched_old_slot_or_worker
833 assert "Deploy complete" not in run.output
834
835 def test_failed_upgrade_keeps_its_error_and_the_database_revision(self, tmp_path: Path) -> None:
836 run = _run_deploy(tmp_path, "0083", FAKE_UPGRADE="fail")
837 assert "RuntimeError: simulated migration failure" in run.output
838 assert run.db_rev == "0083"
839 assert run.ran("python -m musehub.db.schema_gate") == 0
840 assert "Database migrated" not in run.output
841
842 def test_health_check_failure_keeps_traffic_on_the_old_slot(self, tmp_path: Path) -> None:
843 run = _run_deploy(tmp_path, "0084", FAKE_HEALTH="fail")
844 assert run.returncode != 0
845 assert "green failed health check" in run.output
846 assert run.started_new_slot
847 assert not run.switched
848 assert not run.touched_old_slot_or_worker
849
850 def test_one_off_containers_never_migrate_implicitly(self, tmp_path: Path) -> None:
851 run = _run_deploy(tmp_path, "0083")
852 one_off = [c for c in run.calls if c.startswith("docker run --rm")]
853 assert one_off and all("-e SKIP_MIGRATIONS=1" in c for c in one_off), one_off
854
855 def test_no_deploy_script_runs_alembic_stamp(self) -> None:
856 for script in (_DEPLOY_SH, _PUSH_SH):
857 code = [l for l in script.read_text().splitlines() if not l.lstrip().startswith("#")]
858 assert not [l for l in code if re.search(r"alembic\s+stamp", l)], script.name
859
860
861 @pytest.mark.skipif(_BASH is None, reason="requires bash")
862 class TestAlembicRevisionParsing:
863 """deploy.sh's ``alembic_revisions`` against output shapes from Alembic
864 1.18's ``heads`` and ``current``. Anything that is not a revision line
865 must fail the parse rather than be read as a revision."""
866
867 @staticmethod
868 def _parse(output: str) -> tuple[int, list[str]]:
869 fn = _extract_function(_DEPLOY_SH.read_text(), "alembic_revisions")
870 assert _BASH is not None
871 result = subprocess.run(
872 [_BASH, "-c", f'{fn}\nalembic_revisions "$1"', "_", output],
873 capture_output=True,
874 text=True,
875 timeout=5,
876 )
877 return result.returncode, result.stdout.splitlines()
878
879 @pytest.mark.parametrize(
880 ("output", "revisions"),
881 [
882 ("0084 (head)", ["0084"]),
883 ("0083", ["0083"]),
884 ("", []),
885 ("0084 (head)\n0085 (head)", ["0084", "0085"]),
886 ("0086 (head) (mergepoint)", ["0086"]),
887 ("0084 (effective head)", ["0084"]),
888 ],
889 )
890 def test_revision_lines(self, output: str, revisions: list[str]) -> None:
891 assert self._parse(output) == (0, revisions)
892
893 @pytest.mark.parametrize(
894 "output",
895 [
896 "FAILED: Can't locate revision identified by '0099'",
897 "INFO [alembic.runtime.migration] Will assume transactional DDL.",
898 "0084 (head)\nsomething else",
899 "0084 (head) trailing",
900 ],
901 )
902 def test_anything_else_fails(self, output: str) -> None:
903 assert self._parse(output)[0] != 0
File History 13 commits
sha256:d454adcf39faea9356d59c38007eac129e66c9939be75f21d0ecaf7e09acb4fc fix(deploy): free disk before the image pull — remove image… Opus 5.5 patch 1 day ago
sha256:369f691c7aaec3ca84a864ad1d099d5409045ddc12d185a922b3c84b9e2a1bad merge: 100% passing, parallelized MuseHub test suite (task/… Sonnet 5.5 patch 3 days ago
sha256:8166b51ae53275831023a7ce334228b3ccc8237b55d68a35172389db9ff46ad5 test: 100% passing MuseHub suite, parallelized (xdist, one … Sonnet 5.5 patch 3 days ago
sha256:db5120371e0ff3edd7a3920ffa9f31fe81583e4462228510f6db188972f0a31c deploy: memory headroom gate is 1200 MB in staging and production Sonnet 5.5 minor ⚠ 4 days ago
sha256:b8818aebe51ac130d097273b5277ea0d2b2c8b795a6de123247dc4f3d0e38820 release: one-GO batch release flow; fail-closed memory head… Sonnet 5.5 patch 4 days ago
sha256:a452f8cd19f4f38475bef222c24a408b4f20a3381aa0c9db9bf703ecd9ab26bd fix(deploy): stop the deploy when a migration fails instead… Opus 5.5 minor ⚠ 11 days ago
sha256:86464e3e1451e0c4629967e7d2812f4d6db4fe412f2931dd5dc28b5e20b4adbe feat(deploy): publish is gated and additive; activation is … Opus 5.5 minor ⚠ 13 days ago
sha256:4c406127fd42cddb7bd2282519704e86a5ed7129fe2682617e23821e2bddb259 fix: publish_muse_release.sh failed against production on t… Sonnet 5 minor ⚠ 66 days ago
sha256:915b1f581ce1d3bca63153d03c07a5ccbbfbe77de3bf561e75e85f9427dd32a3 fix(musehub#199): remove declare -A from push.sh for bash 3… Sonnet 5 minor ⚠ 66 days ago
sha256:8e5bf0f59b0dbd014580dec546aafd5eac90ea0915244fda362261505ebafc8f fix(#194): deploy.sh's ECR_REGISTRY now derives from ECR_IM… Sonnet 5 minor ⚠ 66 days ago
sha256:54954e48c799d81d5132424ce016a4e9d8b769b08166bce60af610959529061e remove LocalBackend: BlobBackend is the only storage backend Sonnet 4.6 minor ⚠ 191 days ago
sha256:610bebdf8c345da9ee65454fc6f3b19060ef5fb36e008d675b6d036104deb1c3 chore: fix all param_object typing violations — achieve 0 ratchet Sonnet 4.6 minor ⚠ 201 days ago
sha256:1023a5ce87c517287e801d82ad4243278310d0420cd8d628b253ebdff5caa80b refactor(tests): strip _sectionN suffixes from all test filenames Human major ⚠ 218 days ago