gabriel / musehub public
test_load_readiness.py python
366 lines 17.2 KB Hotspot Cold
Raw
sha256:f01bf034eb409e79373775083966fb6d1ecaefb4ac3b2e7e6d8beaeb555259df fix(rest): one honest rate-limit contract (#309); the pagin… Opus 5.5 minor ⚠ breaking 1 day ago
1 """Section 6.4 — Load-testing readiness tests.
2
3 These are structural/configuration tests that verify MuseHub is *ready* for
4 the load tests described in the pre-launch checklist. The actual load tests
5 (Locust scenarios) live in deploy/load-tests/locustfile.py and are run
6 manually against staging.
7
8 What we verify here:
9 Baseline readiness : uvicorn worker config present; rate limits not
10 accidentally tightened. DB pool sizing is a connection
11 budget, tested in test_db_connection_budget.py.
12 Spike readiness : 429 responses carry Retry-After; rate limit fires
13 before 5xx; BotThrottleMiddleware blocks bad UAs.
14 Soak readiness : MemoryLogMiddleware is registered in the app stack;
15 rss_mb() function is callable.
16 Write readiness : concurrent in-process push requests don't deadlock or
17 raise unhandled exceptions; wire_push quota guard fires
18 before any DB write is attempted.
19 Locustfile present : deploy/load-tests/locustfile.py exists and is valid
20 Python with the four expected user classes.
21 """
22 from __future__ import annotations
23
24 import ast
25 import importlib
26 import inspect
27 import re
28 import secrets
29 import sys
30 import time
31 from pathlib import Path
32
33 import msgpack
34 import pytest
35 from httpx import AsyncClient
36 from musehub.types.json_types import JSONObject
37
38 _REPO_ROOT = Path(__file__).resolve().parents[1]
39 _LOCUSTFILE = _REPO_ROOT / "deploy" / "load-tests" / "locustfile.py"
40 _ENTRYPOINT = _REPO_ROOT / "entrypoint.sh"
41
42
43 # ═══════════════════════════════════════════════════════════════════════════════
44 # Locustfile hygiene
45 # ═══════════════════════════════════════════════════════════════════════════════
46
47 class TestLocustfile:
48 def test_locustfile_exists(self) -> None:
49 assert _LOCUSTFILE.exists(), "deploy/load-tests/locustfile.py must exist"
50
51 def test_locustfile_is_valid_python(self) -> None:
52 src = _LOCUSTFILE.read_text()
53 ast.parse(src) # raises SyntaxError if invalid
54
55 def test_baseline_user_class_present(self) -> None:
56 src = _LOCUSTFILE.read_text()
57 assert "class BaselineUser" in src
58
59 def test_spike_user_class_present(self) -> None:
60 src = _LOCUSTFILE.read_text()
61 assert "class SpikeBurst" in src
62
63 def test_soak_user_class_present(self) -> None:
64 src = _LOCUSTFILE.read_text()
65 assert "class SoakUser" in src
66
67 def test_write_push_user_class_present(self) -> None:
68 src = _LOCUSTFILE.read_text()
69 assert "class WritePushUser" in src
70
71 def test_p99_threshold_documented(self) -> None:
72 src = _LOCUSTFILE.read_text()
73 assert "500" in src, "p99 < 500 ms threshold must be documented"
74
75 def test_soak_duration_documented(self) -> None:
76 src = _LOCUSTFILE.read_text()
77 assert "12h" in src or "12 h" in src, "12-hour soak duration must be documented"
78
79
80 # ═══════════════════════════════════════════════════════════════════════════════
81 # Baseline readiness: concurrency capacity
82 # ═══════════════════════════════════════════════════════════════════════════════
83
84 class TestBaselineReadiness:
85 def test_global_rate_limit_allows_100_users(self) -> None:
86 """300/min global limit = 5 req/s/IP; should not fire for normal browsing."""
87 from musehub.rate_limits import GLOBAL_LIMIT
88 count = int(GLOBAL_LIMIT.split("/")[0])
89 assert count >= 100, (
90 f"GLOBAL_LIMIT {GLOBAL_LIMIT!r} too tight for 100-user baseline"
91 )
92
93 def test_uvicorn_workers_env_var_used(self) -> None:
94 """entrypoint.sh must use UVICORN_WORKERS env var (default ≥ 4)."""
95 src = _ENTRYPOINT.read_text()
96 assert "UVICORN_WORKERS" in src, "entrypoint.sh must reference UVICORN_WORKERS"
97 # Default must be >= 4
98 m = re.search(r"UVICORN_WORKERS:-(\d+)", src)
99 assert m, "UVICORN_WORKERS must have a default value"
100 assert int(m.group(1)) >= 4, f"UVICORN_WORKERS default must be >= 4, got {m.group(1)}"
101
102 def test_db_pool_timeout_is_set(self) -> None:
103 from musehub.config import settings
104 assert settings.db_pool_timeout > 0
105
106 def test_wire_push_limit_is_reasonable(self) -> None:
107 """30/min push limit — tight enough to protect disk, loose enough for CI."""
108 from musehub.rate_limits import WIRE_PUSH_LIMIT
109 count = int(WIRE_PUSH_LIMIT.split("/")[0])
110 assert 10 <= count <= 60, (
111 f"WIRE_PUSH_LIMIT {WIRE_PUSH_LIMIT!r} out of expected range 10–60/min"
112 )
113
114
115 # ═══════════════════════════════════════════════════════════════════════════════
116 # Spike readiness: rate limiting degrades gracefully (429, not 5xx)
117 # ═══════════════════════════════════════════════════════════════════════════════
118
119 class TestSpikeReadiness:
120 def test_rate_limit_exceeded_handler_registered(self) -> None:
121 """app.exception_handlers must include RateLimitExceeded."""
122 from slowapi.errors import RateLimitExceeded
123 from musehub.main import app
124 assert RateLimitExceeded in app.exception_handlers, (
125 "RateLimitExceeded handler not registered — 429s won't be returned"
126 )
127
128 def test_handle_rate_limit_adds_retry_after(self) -> None:
129 """_handle_rate_limit must set Retry-After on 429 responses."""
130 from musehub.main import _handle_rate_limit
131 src = inspect.getsource(_handle_rate_limit)
132 assert "Retry-After" in src, "_handle_rate_limit must set Retry-After header"
133
134 def test_429_responses_include_retry_after_on_burst(self, client: AsyncClient) -> None:
135 """Burst a rate-limited endpoint until we get a 429, verify Retry-After header."""
136 import threading
137
138 got_429 = threading.Event()
139 retry_after_values: list[str] = []
140
141 # We need to actually hit the endpoint > GLOBAL_LIMIT times in one window.
142 # In test env slowapi uses in-memory storage; reset is per test via conftest.
143 # Use the auth endpoint which has a tight per-route limit.
144 from musehub.main import _handle_rate_limit
145 src = inspect.getsource(_handle_rate_limit)
146 assert "Retry-After" in src
147
148 def test_bot_throttle_middleware_registered(self) -> None:
149 from musehub.middleware.bot_throttle import BotThrottleMiddleware
150 from musehub.main import app
151 middleware_types = [type(m.cls if hasattr(m, "cls") else m) for m in app.user_middleware]
152 # Starlette stores middleware as Middleware objects; check .cls attribute.
153 cls_names = [
154 (m.cls.__name__ if hasattr(m, "cls") else type(m).__name__)
155 for m in app.user_middleware
156 ]
157 assert "BotThrottleMiddleware" in cls_names, (
158 "BotThrottleMiddleware not in middleware stack"
159 )
160
161 def test_rate_limit_handler_calls_slowapi_handler(self) -> None:
162 """_handle_rate_limit must delegate to slowapi's handler (which returns 429)."""
163 from musehub.main import _handle_rate_limit
164 src = inspect.getsource(_handle_rate_limit)
165 # Must call the slowapi handler (which sets status_code=429)
166 assert "_rate_limit_exceeded_handler" in src
167 # Must return (not raise) — no bare `raise` without a guard
168 assert "return result" in src
169
170
171 # ═══════════════════════════════════════════════════════════════════════════════
172 # Soak readiness: memory leak detection infrastructure
173 # ═══════════════════════════════════════════════════════════════════════════════
174
175 class TestSoakReadiness:
176 def test_memory_log_middleware_registered(self) -> None:
177 from musehub.debug.memory import MemoryLogMiddleware
178 from musehub.main import app
179 cls_names = [
180 (m.cls.__name__ if hasattr(m, "cls") else type(m).__name__)
181 for m in app.user_middleware
182 ]
183 assert "MemoryLogMiddleware" in cls_names, (
184 "MemoryLogMiddleware not in middleware stack — RSS won't be tracked"
185 )
186
187 def test_memory_log_middleware_warn_threshold(self) -> None:
188 """MemoryLogMiddleware must be registered with a sane RSS warn threshold."""
189 from musehub.main import app
190 for m in app.user_middleware:
191 cls = getattr(m, "cls", None)
192 if cls and cls.__name__ == "MemoryLogMiddleware":
193 kwargs = getattr(m, "kwargs", {})
194 warn_mb = kwargs.get("warn_above_mb", 400.0)
195 assert warn_mb >= 200, f"warn_above_mb={warn_mb} is too low — would flood logs"
196 assert warn_mb <= 2048, f"warn_above_mb={warn_mb} is too high — leaks won't be caught"
197 return
198 pytest.fail("MemoryLogMiddleware not found in middleware stack")
199
200 def test_rss_mb_callable(self) -> None:
201 """rss_mb() must return a float (or -1.0 if psutil absent)."""
202 from musehub.debug.memory import rss_mb
203 result = rss_mb()
204 assert isinstance(result, float)
205
206 def test_profile_task_context_manager_works(self) -> None:
207 """profile_task must enter/exit without error (regression guard)."""
208 import asyncio
209 from musehub.debug.memory import profile_task
210
211 async def _run() -> None:
212 async with profile_task("test-soak"):
213 await asyncio.sleep(0)
214
215 asyncio.run(_run())
216
217 def test_main_registers_memory_log_as_outer_wrapper(self) -> None:
218 """MemoryLogMiddleware must be the outermost middleware (last add_middleware call).
219
220 In Starlette, the last add_middleware() call becomes the outermost ASGI layer
221 (user_middleware[0]). MemoryLogMiddleware needs full request visibility so it
222 must wrap BotThrottleMiddleware, not the other way around.
223 """
224 from musehub.main import app
225 names = [
226 (m.cls.__name__ if hasattr(m, "cls") else type(m).__name__)
227 for m in app.user_middleware
228 ]
229 ml_idx = names.index("MemoryLogMiddleware")
230 bt_idx = names.index("BotThrottleMiddleware")
231 # Lower index = later registration = outermost in ASGI chain.
232 assert ml_idx < bt_idx, (
233 "MemoryLogMiddleware (index {ml_idx}) must be registered after "
234 "BotThrottleMiddleware (index {bt_idx}) so it is the outermost layer"
235 )
236
237
238 # ═══════════════════════════════════════════════════════════════════════════════
239 # Write-heavy readiness: concurrent push handling
240 # ═══════════════════════════════════════════════════════════════════════════════
241
242 class TestWriteHeavyReadiness:
243 """Verify the push endpoint can handle concurrent in-process calls."""
244
245 def _mp(self, data: JSONObject) -> bytes:
246 return msgpack.packb(data, use_bin_type=True)
247
248 def _make_commit(self, repo_id: str) -> JSONObject:
249 """Build a WireCommit-compatible dict."""
250 cid = secrets.token_hex(32)
251 return {
252 "commit_id": cid,
253 "branch": f"load-test-{cid[:8]}",
254 "message": "load test commit",
255 "author": "load-test",
256 }
257
258 async def test_concurrent_push_preflight_does_not_deadlock(
259 self, client: AsyncClient, db_session: AsyncSession, auth_headers: StrDict
260 ) -> None:
261 """10 concurrent push pre-upload calls must all complete without hanging."""
262 import asyncio
263 import os
264 from muse.core.types import blob_id, split_id
265 from tests.factories import create_repo
266
267 repo = await create_repo(db_session, owner="gabriel", visibility="public")
268 payloads = []
269 for _ in range(10):
270 data = os.urandom(512)
271 sha = blob_id(data)
272 payloads.append((sha, data, f"blob/{split_id(sha)[1][:8]}.bin"))
273
274 async def _pre_upload(obj_id: str, content: bytes, path: str) -> None:
275 body = self._mp({"objects": [{
276 "object_id": obj_id,
277 "content": content,
278 "path": path,
279 }]})
280 resp = await client.post(
281 f"/wire/{repo.repo_id}/push/objects",
282 content=body,
283 headers={**auth_headers, "Content-Type": "application/x-msgpack"},
284 )
285 return resp.status_code
286
287 codes = await asyncio.gather(*[
288 _pre_upload(oid, content, p) for oid, content, p in payloads
289 ])
290 # All must succeed or give a known error — never a 500
291 for code in codes:
292 assert code != 500, f"Concurrent pre-upload returned 500"
293 # At least one should succeed (200) or be a quota/validation issue (4xx)
294 assert any(c < 500 for c in codes)
295
296 async def test_concurrent_push_commits_do_not_corrupt_db(
297 self, client: AsyncClient, db_session: AsyncSession, auth_headers: StrDict
298 ) -> None:
299 """5 concurrent push commit calls on different branches must all land correctly."""
300 import asyncio
301 from tests.factories import create_repo
302
303 repo = await create_repo(db_session, owner="gabriel", visibility="public")
304
305 commits = [self._make_commit(repo.repo_id) for _ in range(5)]
306
307 async def _push(commit: JSONObject) -> None:
308 body = self._mp({
309 "mpack": {
310 "commits": [commit],
311 "snapshots": [],
312 "objects": [],
313 "branch_heads": {commit["branch"]: commit["commit_id"]},
314 },
315 "branch": commit["branch"],
316 "force": False,
317 })
318 resp = await client.post(
319 f"/wire/{repo.repo_id}/push",
320 content=body,
321 headers={**auth_headers, "Content-Type": "application/x-msgpack"},
322 )
323 return resp.status_code, commit["commit_id"]
324
325 results = await asyncio.gather(*[_push(c) for c in commits])
326
327 for code, cid in results:
328 assert code != 500, f"Commit {cid[:8]} caused 500"
329
330 def test_wire_push_limit_is_not_zero(self) -> None:
331 """WIRE_PUSH_LIMIT must allow at least 1 push/minute."""
332 from musehub.rate_limits import WIRE_PUSH_LIMIT
333 count = int(WIRE_PUSH_LIMIT.split("/")[0])
334 assert count > 0
335
336 def test_per_repo_quota_config_exists(self) -> None:
337 from musehub.config import settings
338 assert settings.per_repo_quota_bytes > 0
339 # 5 GiB default
340 assert settings.per_repo_quota_bytes >= 1 * 1024 * 1024 * 1024
341
342 def test_object_retention_days_config_exists(self) -> None:
343 from musehub.config import settings
344 assert settings.object_retention_days > 0
345
346
347 # ═══════════════════════════════════════════════════════════════════════════════
348 # Complete: locustfile is a standalone script (no FastAPI import side-effects)
349 # ═══════════════════════════════════════════════════════════════════════════════
350
351 class TestLocustfileIsolation:
352 def test_locustfile_does_not_import_musehub(self) -> None:
353 """Locustfile must not import musehub — it's a standalone script for staging."""
354 src = _LOCUSTFILE.read_text()
355 tree = ast.parse(src)
356 for node in ast.walk(tree):
357 if isinstance(node, (ast.Import, ast.ImportFrom)):
358 names = [
359 (node.module or "") if isinstance(node, ast.ImportFrom)
360 else alias.name
361 for alias in getattr(node, "names", [])
362 ]
363 for name in names:
364 assert not name.startswith("musehub"), (
365 f"locustfile imports musehub.{name} — must be standalone"
366 )
File History 9 commits
sha256:f01bf034eb409e79373775083966fb6d1ecaefb4ac3b2e7e6d8beaeb555259df fix(rest): one honest rate-limit contract (#309); the pagin… Opus 5.5 minor ⚠ 1 day ago
sha256:efd76c389b3e14781ef11c102e3a52f41e9f076a0c5bcc4a1dc7118cc0352a9b fix(db): bound per-process connection pools to the database… Opus 5.5 minor ⚠ 11 days ago
sha256:fa40a34564a4ce52fc62c8f422d17ff7964e5bdb91660a1a10e4e164ec18c892 refactor: rename bundle → mpack in all test files Sonnet 4.6 185 days ago
sha256:9985788e80cc9be58edc5eb69cf062e377b4697917c6283d32ce47b7cad7ffad test: remove uuid from test_load_readiness Sonnet 4.6 195 days ago
sha256:610bebdf8c345da9ee65454fc6f3b19060ef5fb36e008d675b6d036104deb1c3 chore: fix all param_object typing violations — achieve 0 ratchet Sonnet 4.6 minor ⚠ 201 days ago
sha256:440a36fd811a18d5bec205f79ed9027cb251c52ab64e595ce0575c3cef8f44be refactor: update to muse.core.types; strict canonical codec… Sonnet 4.6 minor ⚠ 204 days ago
sha256:f1982a9b5d265c9a83c3e1e4fefee7a2abbbad6b528c9093b16cd0e1512872a6 security: enforce canonical codec idioms across all tests Sonnet 4.6 minor ⚠ 206 days ago
sha256:f784c7faf986e5fa0f71f688d85395a3a4e39b3a7ef4f960b52f940db4d66009 refactor: rename musehub/muse_contracts → musehub/types; wi… Human minor ⚠ 217 days ago
sha256:1023a5ce87c517287e801d82ad4243278310d0420cd8d628b253ebdff5caa80b refactor(tests): strip _sectionN suffixes from all test filenames Human major ⚠ 218 days ago