conftest.py
python
sha256:9590cee1e0ccd6c76528f005b95d634d80f5019f0dcb7c371e149adc31d1fb65
refactor: enforce gRPC framing on all MWP wire traffic
Sonnet 4.6
minor
⚠ breaking
156 days ago
| 1 | """Pytest configuration and fixtures.""" |
| 2 | from __future__ import annotations |
| 3 | |
| 4 | from pathlib import Path |
| 5 | |
| 6 | import logging |
| 7 | import os |
| 8 | import typing |
| 9 | from collections.abc import AsyncGenerator, Generator |
| 10 | |
| 11 | if not os.environ.get("MUSE_ENV"): |
| 12 | os.environ["MUSE_ENV"] = "test" |
| 13 | |
| 14 | import pytest |
| 15 | import pytest_asyncio |
| 16 | from httpx import AsyncClient, ASGITransport |
| 17 | from sqlalchemy.ext.asyncio import ( |
| 18 | AsyncSession, |
| 19 | async_sessionmaker, |
| 20 | create_async_engine, |
| 21 | ) |
| 22 | from sqlalchemy.pool import NullPool |
| 23 | |
| 24 | from musehub.db import database |
| 25 | from musehub.db.database import Base, get_db |
| 26 | from musehub.db.musehub_models import MusehubIdentity |
| 27 | # Force all ORM models into Base.metadata before any create_all/drop_all. |
| 28 | # muse_cli_models is only imported inside init_db() in production; without |
| 29 | # this explicit import, Base.metadata is non-deterministic in tests (depends |
| 30 | # on import order), causing drop_all to miss tables that create_all later |
| 31 | # tries to create — resulting in duplicate-key errors on pg_type. |
| 32 | import musehub.db.muse_cli_models as _muse_cli_models # noqa: F401 |
| 33 | from musehub.auth.request_signing import MSignContext, optional_signed_request, require_signed_request |
| 34 | from musehub.main import app |
| 35 | from musehub.rate_limits import limiter |
| 36 | |
| 37 | type _JobPayload = dict[str, str | int | bool | None] |
| 38 | import musehub.auth.failure_limiter as _failure_limiter |
| 39 | |
| 40 | |
| 41 | @pytest.fixture(autouse=True) |
| 42 | def _stub_push_background_tasks(monkeypatch: pytest.MonkeyPatch) -> None: |
| 43 | """Replace enqueue_push_intel with a no-op spy during tests. |
| 44 | |
| 45 | The push endpoint enqueues intel jobs into the DB. During tests we don't |
| 46 | want a live worker processing those jobs concurrently. This fixture |
| 47 | replaces enqueue_push_intel with a no-op that records calls in a |
| 48 | module-level list so integration tests can assert on what was enqueued |
| 49 | without touching the DB. |
| 50 | """ |
| 51 | import musehub.services.musehub_jobs as _jobs |
| 52 | |
| 53 | _jobs._test_enqueued_calls.clear() |
| 54 | |
| 55 | async def _spy_enqueue( |
| 56 | session: AsyncSession, repo_id: str, head: str, domain_id: str | None = None |
| 57 | ) -> None: |
| 58 | _jobs._test_enqueued_calls.append((repo_id, "enqueue_push_intel", {"head": head, "domain_id": domain_id})) |
| 59 | |
| 60 | monkeypatch.setattr(_jobs, "enqueue_push_intel", _spy_enqueue) |
| 61 | |
| 62 | |
| 63 | @pytest.fixture(autouse=True) |
| 64 | def _tmp_objects_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: |
| 65 | """Redirect object storage to a per-test temp directory. |
| 66 | |
| 67 | Prevents tests from writing to the real storage path and isolates |
| 68 | object state between tests. autouse=True so every test gets a |
| 69 | fresh, empty object store without needing to request the fixture. |
| 70 | """ |
| 71 | import musehub.storage.backends as _backends |
| 72 | import musehub.services.musehub_wire as _wire_svc |
| 73 | import musehub.api.routes.wire as _wire_route |
| 74 | from musehub.config import settings |
| 75 | |
| 76 | obj_dir = str(tmp_path) + "/objects" |
| 77 | os.makedirs(obj_dir, exist_ok=True) |
| 78 | test_backend = _backends.LocalBackend(objects_dir=obj_dir) |
| 79 | monkeypatch.setattr(_wire_svc, "get_backend", lambda: test_backend) |
| 80 | monkeypatch.setattr(_wire_route, "get_backend", lambda: test_backend) |
| 81 | # Keep settings.musehub_objects_dir in sync so disk_path containment |
| 82 | # checks in get_object_content / get_blob_meta see the same root. |
| 83 | monkeypatch.setattr(settings, "musehub_objects_dir", obj_dir) |
| 84 | |
| 85 | # Redirect the /releases StaticFiles mount to a temp dir so tests that |
| 86 | # hit /releases/* don't fail because /data/releases doesn't exist locally. |
| 87 | releases_dir = str(tmp_path) + "/releases" |
| 88 | os.makedirs(releases_dir, exist_ok=True) |
| 89 | from musehub.main import app as _app |
| 90 | for _route in _app.routes: |
| 91 | if getattr(_route, "name", None) == "releases": |
| 92 | _static = _route.app # type: ignore[attr-defined] |
| 93 | _static.directory = releases_dir |
| 94 | _static.config_checked = False # force re-check with new dir |
| 95 | break |
| 96 | |
| 97 | |
| 98 | def pytest_configure(config: pytest.Config) -> None: |
| 99 | """Ensure asyncio_mode is auto so async fixtures work (e.g. in Docker when pyproject not in cwd).""" |
| 100 | if hasattr(config.option, "asyncio_mode") and config.option.asyncio_mode is None: |
| 101 | config.option.asyncio_mode = "auto" |
| 102 | # Suppress verbose library loggers that flood the test output with DEBUG lines. |
| 103 | for name in ("httpcore", "httpx", "sqlalchemy", "asyncio", "faker"): |
| 104 | logging.getLogger(name).setLevel(logging.WARNING) |
| 105 | |
| 106 | |
| 107 | @pytest.fixture(autouse=True) |
| 108 | def reset_rate_limiter() -> Generator[None, None, None]: |
| 109 | """Reset in-memory rate-limit counters before every test. |
| 110 | |
| 111 | Without this, the shared MemoryStorage accumulates hits across all tests |
| 112 | in a session. Auth endpoints cap at 20/minute; running 30+ auth tests |
| 113 | back-to-back exhausts that budget and causes 429s for legitimate calls. |
| 114 | """ |
| 115 | limiter.reset() |
| 116 | _failure_limiter._failures.clear() |
| 117 | yield |
| 118 | |
| 119 | |
| 120 | @pytest.fixture |
| 121 | def anyio_backend() -> str: |
| 122 | return "asyncio" |
| 123 | |
| 124 | |
| 125 | _WIRE_CONTEXT = MSignContext( |
| 126 | handle="test-user-wire", |
| 127 | identity_id="wire-test-user-id", |
| 128 | is_agent=False, |
| 129 | is_admin=False, |
| 130 | ) |
| 131 | |
| 132 | |
| 133 | @pytest.fixture |
| 134 | def wire_headers() -> Generator[dict[str, str], None, None]: |
| 135 | """Override auth deps to inject a fake MSignContext for wire protocol tests.""" |
| 136 | app.dependency_overrides[require_signed_request] = lambda: _WIRE_CONTEXT |
| 137 | app.dependency_overrides[optional_signed_request] = lambda: _WIRE_CONTEXT |
| 138 | yield { |
| 139 | "Content-Type": "application/x-msgpack", |
| 140 | "Accept": "application/x-msgpack", |
| 141 | } |
| 142 | app.dependency_overrides.pop(require_signed_request, None) |
| 143 | app.dependency_overrides.pop(optional_signed_request, None) |
| 144 | |
| 145 | |
| 146 | @pytest.fixture(autouse=True) |
| 147 | def _reset_variation_store() -> Generator[None, None, None]: |
| 148 | """Reset the singleton VariationStore between tests to prevent cross-test pollution. |
| 149 | |
| 150 | Gracefully no-ops if the variation module has been removed (MuseHub extraction). |
| 151 | """ |
| 152 | yield |
| 153 | try: |
| 154 | from musehub.variation.storage.variation_store import reset_variation_store |
| 155 | reset_variation_store() |
| 156 | except ModuleNotFoundError: |
| 157 | pass |
| 158 | |
| 159 | |
| 160 | _TEST_DATABASE_URL = os.environ.get( |
| 161 | "TEST_DATABASE_URL", |
| 162 | "postgresql+asyncpg://musehub:musehub@localhost:5434/musehub_test", |
| 163 | ) |
| 164 | |
| 165 | # Sync URL for psycopg2 — used by the session-scoped schema fixture. |
| 166 | _TEST_DATABASE_URL_SYNC = _TEST_DATABASE_URL.replace("+asyncpg", "") |
| 167 | |
| 168 | # Shared async engine for the whole test session (NullPool = no connection |
| 169 | # reuse between tests, but engine object creation is cheap so we create it |
| 170 | # once and share it). |
| 171 | _TEST_ENGINE = create_async_engine(_TEST_DATABASE_URL, poolclass=NullPool) |
| 172 | _TEST_SESSION_FACTORY = async_sessionmaker( |
| 173 | bind=_TEST_ENGINE, |
| 174 | class_=AsyncSession, |
| 175 | expire_on_commit=False, |
| 176 | ) |
| 177 | |
| 178 | # Pre-compute the TRUNCATE statement for all tables so we don't rebuild it |
| 179 | # each test. Reversed sorted_tables respects FK dependency order. |
| 180 | _TRUNCATE_SQL = "TRUNCATE {} RESTART IDENTITY CASCADE".format( |
| 181 | ", ".join(t.name for t in reversed(Base.metadata.sorted_tables)) |
| 182 | ) |
| 183 | |
| 184 | |
| 185 | @pytest.fixture(scope="session", autouse=True) |
| 186 | def _db_schema() -> Generator[None, None, None]: |
| 187 | """Create the test schema once per test session using a sync psycopg2 engine. |
| 188 | |
| 189 | This replaces per-test drop_all/create_all (which took ~3 s per test on |
| 190 | PostgreSQL) with a single DDL pass at session start and end. Individual |
| 191 | tests get a clean slate via TRUNCATE in the db_session fixture instead. |
| 192 | """ |
| 193 | from sqlalchemy import create_engine as _create_engine |
| 194 | |
| 195 | from sqlalchemy import text as _text |
| 196 | |
| 197 | # connect_timeout=10: if postgres is unreachable or still starting (e.g. |
| 198 | # Docker container not ready), fail fast instead of blocking in C forever. |
| 199 | # Without this, Ctrl+C cannot kill the process because psycopg2's socket |
| 200 | # read is a non-interruptible C-level call. |
| 201 | sync_engine = _create_engine( |
| 202 | _TEST_DATABASE_URL_SYNC, |
| 203 | connect_args={"connect_timeout": 10}, |
| 204 | ) |
| 205 | # Terminate any leftover connections from interrupted test runs before |
| 206 | # running drop_all. If a previous pytest session was killed with SIGQUIT |
| 207 | # (Ctrl+\) it leaves postgres backends idle-in-transaction holding locks on |
| 208 | # the test tables. drop_all then waits forever for those locks, which |
| 209 | # makes the next test run freeze with Ctrl+C unresponsive. |
| 210 | with sync_engine.connect() as _conn: |
| 211 | _conn.execute(_text( |
| 212 | "SELECT pg_terminate_backend(pid) FROM pg_stat_activity " |
| 213 | "WHERE datname = current_database() AND pid != pg_backend_pid()" |
| 214 | )) |
| 215 | _conn.commit() |
| 216 | Base.metadata.drop_all(sync_engine) |
| 217 | Base.metadata.create_all(sync_engine) |
| 218 | sync_engine.dispose() |
| 219 | yield |
| 220 | sync_engine2 = _create_engine( |
| 221 | _TEST_DATABASE_URL_SYNC, |
| 222 | connect_args={"connect_timeout": 10}, |
| 223 | ) |
| 224 | Base.metadata.drop_all(sync_engine2) |
| 225 | sync_engine2.dispose() |
| 226 | |
| 227 | |
| 228 | @pytest_asyncio.fixture |
| 229 | async def db_session(_db_schema: None) -> AsyncGenerator[AsyncSession, None]: |
| 230 | """Provide a clean DB session for each test. |
| 231 | |
| 232 | Tables are truncated (not dropped/recreated) between tests — a single |
| 233 | TRUNCATE … CASCADE is ~100× faster than drop_all + create_all on |
| 234 | PostgreSQL, cutting per-test overhead from ~3 s to ~30 ms. |
| 235 | """ |
| 236 | from sqlalchemy import text as _text |
| 237 | |
| 238 | async with _TEST_ENGINE.begin() as conn: |
| 239 | # Terminate any idle-in-transaction backends left by prior tests |
| 240 | # (e.g. aborted SSE streams, asyncio.run() calls in migration tests) |
| 241 | # before acquiring the lock needed for TRUNCATE. |
| 242 | await conn.execute(_text( |
| 243 | "SELECT pg_terminate_backend(pid) FROM pg_stat_activity " |
| 244 | "WHERE datname = current_database() AND pid != pg_backend_pid() " |
| 245 | "AND state = 'idle in transaction'" |
| 246 | )) |
| 247 | await conn.execute(_text(_TRUNCATE_SQL)) |
| 248 | |
| 249 | old_engine = database._engine |
| 250 | old_factory = database._async_session_factory |
| 251 | database._engine = _TEST_ENGINE |
| 252 | database._async_session_factory = _TEST_SESSION_FACTORY |
| 253 | try: |
| 254 | async with _TEST_SESSION_FACTORY() as session: |
| 255 | async def override_get_db() -> AsyncGenerator[AsyncSession, None]: |
| 256 | # Each request gets its own session so concurrent requests |
| 257 | # (e.g. stress tests) don't share a single connection and |
| 258 | # raise "concurrent operations are not permitted". |
| 259 | # All test setup data is committed, so independent sessions |
| 260 | # see it without needing to share the test session. |
| 261 | async with _TEST_SESSION_FACTORY() as req_session: |
| 262 | yield req_session |
| 263 | app.dependency_overrides[get_db] = override_get_db |
| 264 | yield session |
| 265 | app.dependency_overrides.clear() |
| 266 | finally: |
| 267 | database._engine = old_engine |
| 268 | database._async_session_factory = old_factory |
| 269 | |
| 270 | |
| 271 | class _Asgi24Wrapper: |
| 272 | """Inject spec_version='2.4' into every HTTP scope. |
| 273 | |
| 274 | Without this, Starlette's StreamingResponse (spec_version < 2.4 path) runs |
| 275 | listen_for_disconnect concurrently with stream_response via anyio task_group. |
| 276 | listen_for_disconnect calls receive() and steals the request body chunks |
| 277 | before _AsyncExactReader can read them — causing a deadlock where both tasks |
| 278 | block on response_complete.wait() waiting for each other. |
| 279 | |
| 280 | ASGI 2.4 tells Starlette to skip listen_for_disconnect and just stream the |
| 281 | response directly, which is correct for our streaming push handler. |
| 282 | """ |
| 283 | |
| 284 | def __init__(self, app: typing.Any) -> None: |
| 285 | self._app = app |
| 286 | |
| 287 | async def __call__(self, scope: dict, receive: typing.Any, send: typing.Any) -> None: |
| 288 | if scope.get("type") == "http": |
| 289 | scope.setdefault("asgi", {})["spec_version"] = "2.4" |
| 290 | await self._app(scope, receive, send) |
| 291 | |
| 292 | |
| 293 | @pytest_asyncio.fixture |
| 294 | async def client(db_session: AsyncSession) -> AsyncGenerator[AsyncClient, None]: |
| 295 | """Create an async test client. Depends on db_session so auth revocation check uses test DB.""" |
| 296 | transport = ASGITransport(app=_Asgi24Wrapper(app)) |
| 297 | async with AsyncClient(transport=transport, base_url="http://test") as ac: |
| 298 | yield ac |
| 299 | |
| 300 | |
| 301 | # ----------------------------------------------------------------------------- |
| 302 | # Auth fixtures for API contract and integration tests |
| 303 | # Uses dependency_overrides to inject a fake MSignContext so tests don't need |
| 304 | # real Ed25519 key pairs. Only active for tests that request auth_headers. |
| 305 | # ----------------------------------------------------------------------------- |
| 306 | |
| 307 | _TEST_IDENTITY_ID = "550e8400-e29b-41d4-a716-446655440000" |
| 308 | _TEST_HANDLE = "testuser" |
| 309 | |
| 310 | _TEST_CONTEXT = MSignContext( |
| 311 | handle=_TEST_HANDLE, |
| 312 | identity_id=_TEST_IDENTITY_ID, |
| 313 | is_agent=False, |
| 314 | is_admin=False, |
| 315 | ) |
| 316 | |
| 317 | |
| 318 | @pytest_asyncio.fixture |
| 319 | async def test_user(db_session: AsyncSession) -> MusehubIdentity: |
| 320 | """Create a test identity in the DB for authenticated route tests.""" |
| 321 | identity = MusehubIdentity( |
| 322 | identity_id=_TEST_IDENTITY_ID, |
| 323 | handle=_TEST_HANDLE, |
| 324 | display_name="Test User", |
| 325 | identity_type="human", |
| 326 | ) |
| 327 | db_session.add(identity) |
| 328 | await db_session.commit() |
| 329 | await db_session.refresh(identity) |
| 330 | # Close the autobegin transaction started by refresh() so subsequent |
| 331 | # test-body commits don't hit "another operation is in progress". |
| 332 | await db_session.commit() |
| 333 | return identity |
| 334 | |
| 335 | |
| 336 | @pytest.fixture |
| 337 | def auth_headers(test_user: MusehubIdentity) -> Generator[dict[str, str], None, None]: |
| 338 | """Override auth dependencies to inject a fake MSignContext for the test duration. |
| 339 | |
| 340 | Tests that need to verify 401 behaviour for *unauthenticated* requests should |
| 341 | use a separate client call without passing ``auth_headers`` — note that while |
| 342 | this fixture is active the app-level dep overrides are set globally, so any |
| 343 | request made within the same test function will be treated as authenticated. |
| 344 | Tests that need to distinguish authed/unauthed flows within one function should |
| 345 | use ``app.dependency_overrides`` directly or split into two test functions. |
| 346 | """ |
| 347 | app.dependency_overrides[require_signed_request] = lambda: _TEST_CONTEXT |
| 348 | app.dependency_overrides[optional_signed_request] = lambda: _TEST_CONTEXT |
| 349 | yield {"Content-Type": "application/json"} |
| 350 | app.dependency_overrides.pop(require_signed_request, None) |
| 351 | app.dependency_overrides.pop(optional_signed_request, None) |
File History
1 commit
sha256:9590cee1e0ccd6c76528f005b95d634d80f5019f0dcb7c371e149adc31d1fb65
refactor: enforce gRPC framing on all MWP wire traffic
Sonnet 4.6
minor
⚠
156 days ago