test_heavy_page_admission.py
python
sha256:69676bc7346817557178c2c769cf545a7bf8e8a25ff63c862d267619b6d5a7d6
fix(web): never gate navigation pages; gated pages queue up…
Opus 5.5
minor
⚠ breaking
2 days ago
| 1 | """Crawler resilience (musehub#330): heavy pages are admitted before they touch the database, crawl traps are disallowed, and |
| 2 | non-canonical hosts (staging, localhost) are never crawled or indexed. |
| 3 | |
| 4 | Staging, 2026-11-14: ~2,900 requests in 20 minutes from many IPs on blob/raw/blame/symbols pages; 948 QueuePool timeouts; 32% HTTP 500. |
| 5 | """ |
| 6 | from __future__ import annotations |
| 7 | |
| 8 | import asyncio |
| 9 | |
| 10 | import pytest |
| 11 | from httpx import ASGITransport, AsyncClient |
| 12 | from starlette.types import Receive, Scope, Send |
| 13 | |
| 14 | from musehub.config import settings |
| 15 | from musehub.middleware.heavy_pages import HEAVY_PAGE_PATH, RETRY_AFTER_SECONDS, HeavyPageAdmissionMiddleware |
| 16 | |
| 17 | |
| 18 | @pytest.mark.parametrize("path", [ |
| 19 | "/gabriel/muse/blob/main/README.md", "/gabriel/muse/raw/sha256:abc/x.py", "/gabriel/muse/blame/dev/x.py", "/gabriel/muse/tree/dev", |
| 20 | "/gabriel/muse/symbols", "/gabriel/muse/symbol/x.py::f", "/gabriel/muse/intel/blast-risk/detail", "/gabriel/muse/commits", |
| 21 | "/gabriel/muse/commit/sha256:abc", "/musehub/oembed", |
| 22 | ]) |
| 23 | def test_heavy_page_paths_are_gated(path: str) -> None: |
| 24 | assert HEAVY_PAGE_PATH.match(path) |
| 25 | |
| 26 | |
| 27 | @pytest.mark.parametrize("path", [ |
| 28 | "/gabriel/muse", "/gabriel/muse/refs", "/gabriel/muse/push/mpack-presign", "/gabriel/muse/fetch/mpack", "/gabriel/muse/issues", |
| 29 | "/api/repos/abc/objects", "/static/app.css", "/healthz", "/robots.txt", "/gabriel", "/mcp", "/mcp/sse", "/o/sha256:abc", |
| 30 | "/api/health/live", "/explorer-not-a-page", "/gabriel/muse/push/unpack-mpack", |
| 31 | # navigation pages people use to find their way are never gated |
| 32 | "/explore", "/explore/repos", "/search", "/gabriel/muse/proposals", "/gabriel/x/proposals/7", "/gabriel/muse/insights", |
| 33 | "/gabriel/muse/branches", "/gabriel/muse/releases", "/gabriel/muse/opengraph-image", "/gabriel/muse/issues/327", |
| 34 | ]) |
| 35 | def test_cheap_pages_wire_and_api_are_not_gated(path: str) -> None: |
| 36 | assert not HEAVY_PAGE_PATH.match(path) |
| 37 | |
| 38 | |
| 39 | def test_heavy_page_concurrency_leaves_web_pool_connections_for_cheap_pages() -> None: |
| 40 | assert 1 <= settings.heavy_page_concurrency < settings.db_pool_size + settings.db_max_overflow |
| 41 | |
| 42 | |
| 43 | def _app_holding(release: asyncio.Event, entered: list[str]): # type: ignore[no-untyped-def] |
| 44 | async def app(scope: Scope, receive: Receive, send: Send) -> None: |
| 45 | entered.append(scope["path"]) |
| 46 | await release.wait() |
| 47 | await send({"type": "http.response.start", "status": 200, "headers": [(b"content-type", b"text/plain")]}) |
| 48 | await send({"type": "http.response.body", "body": b"ok"}) |
| 49 | return app |
| 50 | |
| 51 | |
| 52 | @pytest.mark.asyncio |
| 53 | async def test_excess_heavy_requests_get_a_fast_503_and_never_reach_the_app(monkeypatch: pytest.MonkeyPatch) -> None: |
| 54 | monkeypatch.setattr(settings, "heavy_page_concurrency", 1) |
| 55 | monkeypatch.setattr(settings, "heavy_page_wait_seconds", 0.2) |
| 56 | release, entered = asyncio.Event(), [] |
| 57 | gate = HeavyPageAdmissionMiddleware(_app_holding(release, entered)) |
| 58 | async with AsyncClient(transport=ASGITransport(app=gate), base_url="https://musehub.ai") as client: |
| 59 | first = asyncio.create_task(client.get("/gabriel/muse/blob/dev/a.py")) |
| 60 | while not entered: |
| 61 | await asyncio.sleep(0.01) |
| 62 | busy_html = await client.get("/gabriel/muse/blame/dev/a.py", headers={"accept": "text/html"}) |
| 63 | busy_json = await client.get("/gabriel/muse/symbols?q=x") |
| 64 | cheap = asyncio.create_task(client.get("/gabriel/muse/issues")) # not gated: reaches the app while the slot is taken |
| 65 | while len(entered) < 2: |
| 66 | await asyncio.sleep(0.01) |
| 67 | release.set() |
| 68 | assert (await first).status_code == 200 and (await cheap).status_code == 200 |
| 69 | assert entered == ["/gabriel/muse/blob/dev/a.py", "/gabriel/muse/issues"], "un-admitted requests must not reach the app (no DB work)" |
| 70 | assert busy_html.status_code == 503 and busy_html.headers["retry-after"] == str(RETRY_AFTER_SECONDS) |
| 71 | assert "text/html" in busy_html.headers["content-type"] and "reload" in busy_html.text |
| 72 | assert busy_json.status_code == 503 and busy_json.json() == {"detail": "busy: try again shortly"} |
| 73 | |
| 74 | |
| 75 | @pytest.mark.asyncio |
| 76 | async def test_slot_is_released_after_the_response_and_after_an_error(monkeypatch: pytest.MonkeyPatch) -> None: |
| 77 | monkeypatch.setattr(settings, "heavy_page_concurrency", 1) |
| 78 | monkeypatch.setattr(settings, "heavy_page_wait_seconds", 0.2) |
| 79 | |
| 80 | async def boom(scope: Scope, receive: Receive, send: Send) -> None: |
| 81 | if scope["path"].endswith("/boom"): |
| 82 | raise RuntimeError("render failed") |
| 83 | await send({"type": "http.response.start", "status": 200, "headers": []}) |
| 84 | await send({"type": "http.response.body", "body": b""}) |
| 85 | |
| 86 | gate = HeavyPageAdmissionMiddleware(boom) |
| 87 | async with AsyncClient(transport=ASGITransport(app=gate, raise_app_exceptions=False), base_url="https://musehub.ai") as client: |
| 88 | assert (await client.get("/gabriel/muse/blob/boom")).status_code == 500 |
| 89 | for _ in range(3): |
| 90 | assert (await client.get("/gabriel/muse/blob/ok")).status_code == 200 |
| 91 | |
| 92 | |
| 93 | @pytest.mark.asyncio |
| 94 | async def test_staging_robots_disallows_everything_and_pages_carry_noindex(client: AsyncClient) -> None: |
| 95 | staging = await client.get("/robots.txt", headers={"host": "staging.musehub.ai"}) |
| 96 | assert staging.status_code == 200 |
| 97 | assert staging.text.splitlines()[-2:] == ["User-agent: *", "Disallow: /"] |
| 98 | page = await client.get("/robots.txt", headers={"host": "staging.musehub.ai"}) |
| 99 | assert page.headers.get("x-robots-tag") == "noindex, nofollow" |
| 100 | canonical = await client.get("/robots.txt", headers={"host": "musehub.ai"}) |
| 101 | assert "x-robots-tag" not in canonical.headers |
| 102 | |
| 103 | |
| 104 | @pytest.mark.asyncio |
| 105 | async def test_canonical_robots_repeats_the_crawl_trap_disallows_in_every_group(client: AsyncClient) -> None: |
| 106 | text = (await client.get("/robots.txt", headers={"host": "musehub.ai"})).text |
| 107 | groups = text.count("User-agent:") |
| 108 | for rule in ("Disallow: /*/raw/", "Disallow: /*/blame/", "Disallow: /*/symbols?", "Disallow: /*/intel/*?", "Disallow: /musehub/oembed"): |
| 109 | assert text.count(rule) == groups, f"{rule!r} must be in all {groups} groups (a named group replaces the * group)" |
| 110 | assert "Disallow: /\n" not in text |
| 111 | for group in text.split("User-agent:")[1:]: # first-match crawlers: every Disallow precedes the group's Allow |
| 112 | lines = [ln for ln in group.splitlines() if ln.startswith(("Allow:", "Disallow:"))] |
| 113 | if "Allow: /" in lines: |
| 114 | assert all(ln.startswith("Disallow:") for ln in lines[: lines.index("Allow: /")]) and lines.index("Allow: /") == len(lines) - 1 |
| 115 | |
| 116 | |
| 117 | @pytest.mark.asyncio |
| 118 | async def test_a_gate_503_still_carries_the_security_headers(monkeypatch: pytest.MonkeyPatch, client: AsyncClient) -> None: |
| 119 | """The gate sits inside SecurityHeaders: its 503 gets nosniff/X-Frame-Options (and staging's X-Robots-Tag) like any response.""" |
| 120 | monkeypatch.setattr(settings, "heavy_page_wait_seconds", 0.0) |
| 121 | monkeypatch.setattr(settings, "heavy_page_concurrency", 1) |
| 122 | import musehub.middleware.heavy_pages as hp |
| 123 | |
| 124 | taken = asyncio.Semaphore(1) |
| 125 | await taken.acquire() # the only slot is taken: a gated request must be refused (proves the gate is installed in the app, too) |
| 126 | monkeypatch.setattr(hp.HeavyPageAdmissionMiddleware, "_semaphore", lambda self: taken) |
| 127 | resp = await client.get("/gabriel/muse/blob/dev/a.py", headers={"host": "staging.musehub.ai", "accept": "text/html"}) |
| 128 | assert resp.status_code == 503 |
| 129 | assert resp.headers.get("x-content-type-options") == "nosniff" and resp.headers.get("x-robots-tag") == "noindex, nofollow" |
| 130 | |
| 131 | |
| 132 | def test_generated_symbol_search_links_are_nofollow() -> None: |
| 133 | import pathlib |
| 134 | import re |
| 135 | |
| 136 | root = pathlib.Path(__file__).resolve().parent.parent / "musehub" / "templates" |
| 137 | for tpl in root.rglob("*.html"): |
| 138 | for tag in re.findall(r"<a\s[^>]*?/symbols\?q=[^>]*>", tpl.read_text()): |
| 139 | assert 'rel="nofollow"' in tag, (tpl.name, tag[:120]) |
| 140 | |
| 141 | |
| 142 | def test_pages_advertise_no_oembed_endpoint() -> None: |
| 143 | """There is no oEmbed route: the discovery link sent every crawler that followed it to a failing repo lookup (staging/prod 500s).""" |
| 144 | import pathlib |
| 145 | |
| 146 | base = (pathlib.Path(__file__).resolve().parent.parent / "musehub" / "templates" / "musehub" / "base.html").read_text() |
| 147 | assert "oembed" not in base.lower() |
| 148 | |
| 149 | |
| 150 | |
| 151 | def test_gated_pages_queue_before_the_busy_page() -> None: |
| 152 | """The busy page is a last resort: a gated request waits (holding no DB connection) well before giving up.""" |
| 153 | assert settings.heavy_page_wait_seconds >= 10 |
File History
4 commits
sha256:69676bc7346817557178c2c769cf545a7bf8e8a25ff63c862d267619b6d5a7d6
fix(web): never gate navigation pages; gated pages queue up…
Opus 5.5
minor
⚠
2 days ago
sha256:013ac5cc92862a516c303d63d7bb46532118568e6049bc6890aa2e41f3223529
fix(web): stop advertising an oEmbed endpoint that does not…
Opus 5.5
minor
⚠
2 days ago
sha256:32b8dc1897e4aab3cd9c94fad9d11d513470c095fdd289e2bd2b8f57eaae0be6
fix(web): review of b9b1ef31 — gate the remaining heavy pag…
Opus 5.5
patch
2 days ago
sha256:b9b1ef31dc867066f6954ca33840604c51c6e168c27db004176357466b815374
fix(web): crawler resilience — admit heavy pages before the…
Opus 5.5
minor
⚠
2 days ago