gabriel / musehub public
test_heavy_page_admission.py python
153 lines 8.3 KB
Raw
sha256:69676bc7346817557178c2c769cf545a7bf8e8a25ff63c862d267619b6d5a7d6 fix(web): never gate navigation pages; gated pages queue up… Opus 5.5 minor ⚠ breaking 2 days ago
1 """Crawler resilience (musehub#330): heavy pages are admitted before they touch the database, crawl traps are disallowed, and
2 non-canonical hosts (staging, localhost) are never crawled or indexed.
3
4 Staging, 2026-11-14: ~2,900 requests in 20 minutes from many IPs on blob/raw/blame/symbols pages; 948 QueuePool timeouts; 32% HTTP 500.
5 """
6 from __future__ import annotations
7
8 import asyncio
9
10 import pytest
11 from httpx import ASGITransport, AsyncClient
12 from starlette.types import Receive, Scope, Send
13
14 from musehub.config import settings
15 from musehub.middleware.heavy_pages import HEAVY_PAGE_PATH, RETRY_AFTER_SECONDS, HeavyPageAdmissionMiddleware
16
17
18 @pytest.mark.parametrize("path", [
19 "/gabriel/muse/blob/main/README.md", "/gabriel/muse/raw/sha256:abc/x.py", "/gabriel/muse/blame/dev/x.py", "/gabriel/muse/tree/dev",
20 "/gabriel/muse/symbols", "/gabriel/muse/symbol/x.py::f", "/gabriel/muse/intel/blast-risk/detail", "/gabriel/muse/commits",
21 "/gabriel/muse/commit/sha256:abc", "/musehub/oembed",
22 ])
23 def test_heavy_page_paths_are_gated(path: str) -> None:
24 assert HEAVY_PAGE_PATH.match(path)
25
26
27 @pytest.mark.parametrize("path", [
28 "/gabriel/muse", "/gabriel/muse/refs", "/gabriel/muse/push/mpack-presign", "/gabriel/muse/fetch/mpack", "/gabriel/muse/issues",
29 "/api/repos/abc/objects", "/static/app.css", "/healthz", "/robots.txt", "/gabriel", "/mcp", "/mcp/sse", "/o/sha256:abc",
30 "/api/health/live", "/explorer-not-a-page", "/gabriel/muse/push/unpack-mpack",
31 # navigation pages people use to find their way are never gated
32 "/explore", "/explore/repos", "/search", "/gabriel/muse/proposals", "/gabriel/x/proposals/7", "/gabriel/muse/insights",
33 "/gabriel/muse/branches", "/gabriel/muse/releases", "/gabriel/muse/opengraph-image", "/gabriel/muse/issues/327",
34 ])
35 def test_cheap_pages_wire_and_api_are_not_gated(path: str) -> None:
36 assert not HEAVY_PAGE_PATH.match(path)
37
38
39 def test_heavy_page_concurrency_leaves_web_pool_connections_for_cheap_pages() -> None:
40 assert 1 <= settings.heavy_page_concurrency < settings.db_pool_size + settings.db_max_overflow
41
42
43 def _app_holding(release: asyncio.Event, entered: list[str]): # type: ignore[no-untyped-def]
44 async def app(scope: Scope, receive: Receive, send: Send) -> None:
45 entered.append(scope["path"])
46 await release.wait()
47 await send({"type": "http.response.start", "status": 200, "headers": [(b"content-type", b"text/plain")]})
48 await send({"type": "http.response.body", "body": b"ok"})
49 return app
50
51
52 @pytest.mark.asyncio
53 async def test_excess_heavy_requests_get_a_fast_503_and_never_reach_the_app(monkeypatch: pytest.MonkeyPatch) -> None:
54 monkeypatch.setattr(settings, "heavy_page_concurrency", 1)
55 monkeypatch.setattr(settings, "heavy_page_wait_seconds", 0.2)
56 release, entered = asyncio.Event(), []
57 gate = HeavyPageAdmissionMiddleware(_app_holding(release, entered))
58 async with AsyncClient(transport=ASGITransport(app=gate), base_url="https://musehub.ai") as client:
59 first = asyncio.create_task(client.get("/gabriel/muse/blob/dev/a.py"))
60 while not entered:
61 await asyncio.sleep(0.01)
62 busy_html = await client.get("/gabriel/muse/blame/dev/a.py", headers={"accept": "text/html"})
63 busy_json = await client.get("/gabriel/muse/symbols?q=x")
64 cheap = asyncio.create_task(client.get("/gabriel/muse/issues")) # not gated: reaches the app while the slot is taken
65 while len(entered) < 2:
66 await asyncio.sleep(0.01)
67 release.set()
68 assert (await first).status_code == 200 and (await cheap).status_code == 200
69 assert entered == ["/gabriel/muse/blob/dev/a.py", "/gabriel/muse/issues"], "un-admitted requests must not reach the app (no DB work)"
70 assert busy_html.status_code == 503 and busy_html.headers["retry-after"] == str(RETRY_AFTER_SECONDS)
71 assert "text/html" in busy_html.headers["content-type"] and "reload" in busy_html.text
72 assert busy_json.status_code == 503 and busy_json.json() == {"detail": "busy: try again shortly"}
73
74
75 @pytest.mark.asyncio
76 async def test_slot_is_released_after_the_response_and_after_an_error(monkeypatch: pytest.MonkeyPatch) -> None:
77 monkeypatch.setattr(settings, "heavy_page_concurrency", 1)
78 monkeypatch.setattr(settings, "heavy_page_wait_seconds", 0.2)
79
80 async def boom(scope: Scope, receive: Receive, send: Send) -> None:
81 if scope["path"].endswith("/boom"):
82 raise RuntimeError("render failed")
83 await send({"type": "http.response.start", "status": 200, "headers": []})
84 await send({"type": "http.response.body", "body": b""})
85
86 gate = HeavyPageAdmissionMiddleware(boom)
87 async with AsyncClient(transport=ASGITransport(app=gate, raise_app_exceptions=False), base_url="https://musehub.ai") as client:
88 assert (await client.get("/gabriel/muse/blob/boom")).status_code == 500
89 for _ in range(3):
90 assert (await client.get("/gabriel/muse/blob/ok")).status_code == 200
91
92
93 @pytest.mark.asyncio
94 async def test_staging_robots_disallows_everything_and_pages_carry_noindex(client: AsyncClient) -> None:
95 staging = await client.get("/robots.txt", headers={"host": "staging.musehub.ai"})
96 assert staging.status_code == 200
97 assert staging.text.splitlines()[-2:] == ["User-agent: *", "Disallow: /"]
98 page = await client.get("/robots.txt", headers={"host": "staging.musehub.ai"})
99 assert page.headers.get("x-robots-tag") == "noindex, nofollow"
100 canonical = await client.get("/robots.txt", headers={"host": "musehub.ai"})
101 assert "x-robots-tag" not in canonical.headers
102
103
104 @pytest.mark.asyncio
105 async def test_canonical_robots_repeats_the_crawl_trap_disallows_in_every_group(client: AsyncClient) -> None:
106 text = (await client.get("/robots.txt", headers={"host": "musehub.ai"})).text
107 groups = text.count("User-agent:")
108 for rule in ("Disallow: /*/raw/", "Disallow: /*/blame/", "Disallow: /*/symbols?", "Disallow: /*/intel/*?", "Disallow: /musehub/oembed"):
109 assert text.count(rule) == groups, f"{rule!r} must be in all {groups} groups (a named group replaces the * group)"
110 assert "Disallow: /\n" not in text
111 for group in text.split("User-agent:")[1:]: # first-match crawlers: every Disallow precedes the group's Allow
112 lines = [ln for ln in group.splitlines() if ln.startswith(("Allow:", "Disallow:"))]
113 if "Allow: /" in lines:
114 assert all(ln.startswith("Disallow:") for ln in lines[: lines.index("Allow: /")]) and lines.index("Allow: /") == len(lines) - 1
115
116
117 @pytest.mark.asyncio
118 async def test_a_gate_503_still_carries_the_security_headers(monkeypatch: pytest.MonkeyPatch, client: AsyncClient) -> None:
119 """The gate sits inside SecurityHeaders: its 503 gets nosniff/X-Frame-Options (and staging's X-Robots-Tag) like any response."""
120 monkeypatch.setattr(settings, "heavy_page_wait_seconds", 0.0)
121 monkeypatch.setattr(settings, "heavy_page_concurrency", 1)
122 import musehub.middleware.heavy_pages as hp
123
124 taken = asyncio.Semaphore(1)
125 await taken.acquire() # the only slot is taken: a gated request must be refused (proves the gate is installed in the app, too)
126 monkeypatch.setattr(hp.HeavyPageAdmissionMiddleware, "_semaphore", lambda self: taken)
127 resp = await client.get("/gabriel/muse/blob/dev/a.py", headers={"host": "staging.musehub.ai", "accept": "text/html"})
128 assert resp.status_code == 503
129 assert resp.headers.get("x-content-type-options") == "nosniff" and resp.headers.get("x-robots-tag") == "noindex, nofollow"
130
131
132 def test_generated_symbol_search_links_are_nofollow() -> None:
133 import pathlib
134 import re
135
136 root = pathlib.Path(__file__).resolve().parent.parent / "musehub" / "templates"
137 for tpl in root.rglob("*.html"):
138 for tag in re.findall(r"<a\s[^>]*?/symbols\?q=[^>]*>", tpl.read_text()):
139 assert 'rel="nofollow"' in tag, (tpl.name, tag[:120])
140
141
142 def test_pages_advertise_no_oembed_endpoint() -> None:
143 """There is no oEmbed route: the discovery link sent every crawler that followed it to a failing repo lookup (staging/prod 500s)."""
144 import pathlib
145
146 base = (pathlib.Path(__file__).resolve().parent.parent / "musehub" / "templates" / "musehub" / "base.html").read_text()
147 assert "oembed" not in base.lower()
148
149
150
151 def test_gated_pages_queue_before_the_busy_page() -> None:
152 """The busy page is a last resort: a gated request waits (holding no DB connection) well before giving up."""
153 assert settings.heavy_page_wait_seconds >= 10
File History 4 commits
sha256:69676bc7346817557178c2c769cf545a7bf8e8a25ff63c862d267619b6d5a7d6 fix(web): never gate navigation pages; gated pages queue up… Opus 5.5 minor ⚠ 2 days ago
sha256:013ac5cc92862a516c303d63d7bb46532118568e6049bc6890aa2e41f3223529 fix(web): stop advertising an oEmbed endpoint that does not… Opus 5.5 minor ⚠ 2 days ago
sha256:32b8dc1897e4aab3cd9c94fad9d11d513470c095fdd289e2bd2b8f57eaae0be6 fix(web): review of b9b1ef31 — gate the remaining heavy pag… Opus 5.5 patch 2 days ago
sha256:b9b1ef31dc867066f6954ca33840604c51c6e168c27db004176357466b815374 fix(web): crawler resilience — admit heavy pages before the… Opus 5.5 minor ⚠ 2 days ago