gabriel / musehub public
test_musehub_sitemap.py python
302 lines 10.7 KB
Raw
sha256:9590cee1e0ccd6c76528f005b95d634d80f5019f0dcb7c371e149adc31d1fb65 refactor: enforce gRPC framing on all MWP wire traffic Sonnet 4.6 minor ⚠ breaking 156 days ago
1 """Tests for the MuseHub sitemap.xml and robots.txt endpoints.
2
3 Covers acceptance criteria:
4 - test_sitemap_returns_xml — GET /sitemap.xml returns 200 with XML content-type
5 - test_sitemap_contains_static_pages — static explore/trending/topics URLs are always present
6 - test_sitemap_contains_public_repo — a seeded public repo appears in the sitemap
7 - test_sitemap_excludes_private_repo — private repos do NOT appear in the sitemap
8 - test_sitemap_contains_user_profile — seeded user profile URL appears in sitemap
9 - test_sitemap_contains_topic_urls — repo tags generate /topics/{tag} entries
10 - test_sitemap_contains_release_url — a release URL appears for repos with releases
11 - test_sitemap_xml_well_formed — sitemap can be parsed as valid XML
12 - test_sitemap_loc_uses_request_host — loc entries use the base URL from the request
13 - test_robots_txt_returns_plain_text — GET /robots.txt returns 200 text/plain
14 - test_robots_txt_allows_musehub_ui — Allow: / is present
15 - test_robots_txt_disallows_settings — settings path is disallowed
16 - test_robots_txt_disallows_api — /api/ directory is disallowed
17 - test_robots_txt_contains_sitemap_url — Sitemap: directive points to /sitemap.xml
18 - test_robots_txt_names_known_agents — known AI bots appear with explicit Allow
19 - test_robots_txt_no_auth_required — endpoint is accessible without authentication
20 - test_sitemap_no_auth_required — sitemap is accessible without authentication
21 """
22 from __future__ import annotations
23
24 import pytest
25 from httpx import AsyncClient
26 from sqlalchemy.ext.asyncio import AsyncSession
27 from xml.etree import ElementTree as ET
28
29 from musehub.db.musehub_models import (
30 MusehubIdentity,
31 MusehubRelease,
32 MusehubRepo,
33 )
34
35
36 # ---------------------------------------------------------------------------
37 # Helpers
38 # ---------------------------------------------------------------------------
39
40
41 async def _make_public_repo(
42 db_session: AsyncSession,
43 *,
44 owner: str = "sitemap-user",
45 slug: str = "sitemap-repo",
46 tags: list[str] | None = None,
47 visibility: str = "public",
48 ) -> MusehubRepo:
49 """Seed a repo and return the ORM object."""
50 repo = MusehubRepo(
51 name=slug,
52 owner=owner,
53 slug=slug,
54 visibility=visibility,
55 owner_user_id="sitemap-user-id",
56 description="test repo for sitemap",
57 tags=tags or [],
58 )
59 db_session.add(repo)
60 await db_session.commit()
61 await db_session.refresh(repo)
62 return repo
63
64
65 async def _make_profile(
66 db_session: AsyncSession,
67 *,
68 username: str = "sitemap-user",
69 user_id: str = "sitemap-user-id",
70 ) -> MusehubIdentity:
71 """Seed a user identity and return the ORM object."""
72 identity = MusehubIdentity(
73 identity_id=user_id,
74 handle=username,
75 identity_type="human",
76 )
77 db_session.add(identity)
78 await db_session.commit()
79 await db_session.refresh(identity)
80 return identity
81
82
83 async def _make_release(
84 db_session: AsyncSession,
85 repo_id: str,
86 *,
87 tag: str = "v1.0",
88 ) -> MusehubRelease:
89 """Seed a release and return the ORM object."""
90 release = MusehubRelease(
91 repo_id=repo_id,
92 tag=tag,
93 title=f"Release {tag}",
94 body="",
95 author="sitemap-user",
96 )
97 db_session.add(release)
98 await db_session.commit()
99 await db_session.refresh(release)
100 return release
101
102
103 # ---------------------------------------------------------------------------
104 # Sitemap tests
105 # ---------------------------------------------------------------------------
106
107
108 async def test_sitemap_returns_xml(client: AsyncClient, db_session: AsyncSession) -> None:
109 """GET /sitemap.xml returns 200 with an XML content-type."""
110 response = await client.get("/sitemap.xml")
111 assert response.status_code == 200
112 assert "xml" in response.headers["content-type"]
113
114
115 async def test_sitemap_contains_static_pages(
116 client: AsyncClient, db_session: AsyncSession
117 ) -> None:
118 """Static explore and topics pages are always included in the sitemap."""
119 response = await client.get("/sitemap.xml")
120 assert response.status_code == 200
121 body = response.text
122 assert "/explore" in body
123 assert "/topics" in body
124
125
126 async def test_sitemap_contains_public_repo(
127 client: AsyncClient, db_session: AsyncSession
128 ) -> None:
129 """A seeded public repo's UI URL appears in the sitemap."""
130 await _make_public_repo(db_session, owner="artist", slug="cool-track")
131 response = await client.get("/sitemap.xml")
132 assert response.status_code == 200
133 body = response.text
134 assert "/artist/cool-track" in body
135
136
137 async def test_sitemap_excludes_private_repo(
138 client: AsyncClient, db_session: AsyncSession
139 ) -> None:
140 """Private repos must not appear anywhere in the sitemap."""
141 await _make_public_repo(db_session, owner="secretuser", slug="hidden-project", visibility="private")
142 response = await client.get("/sitemap.xml")
143 assert response.status_code == 200
144 body = response.text
145 assert "hidden-project" not in body
146 assert "secretuser" not in body
147
148
149 async def test_sitemap_contains_user_profile(
150 client: AsyncClient, db_session: AsyncSession
151 ) -> None:
152 """A seeded user profile generates a /users/{username} entry."""
153 await _make_profile(db_session, username="jazzmaster", user_id="jazzmaster-uid")
154 response = await client.get("/sitemap.xml")
155 assert response.status_code == 200
156 assert "/users/jazzmaster" in response.text
157
158
159 async def test_sitemap_contains_topic_urls(
160 client: AsyncClient, db_session: AsyncSession
161 ) -> None:
162 """Tags on public repos generate /topics/{tag} entries."""
163 await _make_public_repo(db_session, owner="producer", slug="beats", tags=["lo-fi", "jazz"])
164 response = await client.get("/sitemap.xml")
165 assert response.status_code == 200
166 body = response.text
167 assert "/topics/lo-fi" in body
168 assert "/topics/jazz" in body
169
170
171 async def test_sitemap_contains_release_url(
172 client: AsyncClient, db_session: AsyncSession
173 ) -> None:
174 """A release on a public repo generates a /releases/{tag} sitemap entry."""
175 repo = await _make_public_repo(db_session, owner="bandname", slug="debut-album")
176 await _make_release(db_session, repo.repo_id, tag="v1.0")
177 response = await client.get("/sitemap.xml")
178 assert response.status_code == 200
179 assert "/bandname/debut-album/releases/v1.0" in response.text
180
181
182 async def test_sitemap_xml_well_formed(
183 client: AsyncClient, db_session: AsyncSession
184 ) -> None:
185 """The sitemap response must be parseable as valid XML."""
186 response = await client.get("/sitemap.xml")
187 assert response.status_code == 200
188 # This raises if the document is not well-formed XML.
189 root = ET.fromstring(response.content)
190 assert root.tag.endswith("urlset")
191
192
193 async def test_sitemap_loc_uses_request_host(
194 client: AsyncClient, db_session: AsyncSession
195 ) -> None:
196 """loc entries in the sitemap use the base URL from the incoming request."""
197 await _make_public_repo(db_session, owner="testowner", slug="testrepo")
198 response = await client.get("/sitemap.xml")
199 assert response.status_code == 200
200 # The test client uses base_url="http://test" — every loc must start with http://test.
201 body = response.text
202 assert "<loc>http://test" in body
203
204
205 async def test_sitemap_no_auth_required(
206 client: AsyncClient, db_session: AsyncSession
207 ) -> None:
208 """Sitemap endpoint must be accessible without authentication (crawlers don't authenticate)."""
209 response = await client.get("/sitemap.xml")
210 assert response.status_code != 401
211 assert response.status_code == 200
212
213
214 async def test_sitemap_repo_commits_page_included(
215 client: AsyncClient, db_session: AsyncSession
216 ) -> None:
217 """Each public repo's /commits page also appears in the sitemap."""
218 await _make_public_repo(db_session, owner="composer", slug="symphony-no1")
219 response = await client.get("/sitemap.xml")
220 assert response.status_code == 200
221 assert "/composer/symphony-no1/commits" in response.text
222
223
224 async def test_sitemap_repo_issues_page_included(
225 client: AsyncClient, db_session: AsyncSession
226 ) -> None:
227 """Each public repo's /issues page also appears in the sitemap."""
228 await _make_public_repo(db_session, owner="composer", slug="symphony-no2")
229 response = await client.get("/sitemap.xml")
230 assert response.status_code == 200
231 assert "/composer/symphony-no2/issues" in response.text
232
233
234 # ---------------------------------------------------------------------------
235 # Robots.txt tests
236 # ---------------------------------------------------------------------------
237
238
239 async def test_robots_txt_returns_plain_text(
240 client: AsyncClient, db_session: AsyncSession
241 ) -> None:
242 """GET /robots.txt returns 200 with text/plain content-type."""
243 response = await client.get("/robots.txt")
244 assert response.status_code == 200
245 assert "text/plain" in response.headers["content-type"]
246
247
248 async def test_robots_txt_allows_musehub_ui(
249 client: AsyncClient, db_session: AsyncSession
250 ) -> None:
251 """Allow: / is present for all crawlers."""
252 response = await client.get("/robots.txt")
253 assert response.status_code == 200
254 assert "Allow: /" in response.text
255
256
257 async def test_robots_txt_disallows_settings(
258 client: AsyncClient, db_session: AsyncSession
259 ) -> None:
260 """Settings paths are disallowed to prevent indexing of private user config pages."""
261 response = await client.get("/robots.txt")
262 assert response.status_code == 200
263 assert "Disallow: /*/settings" in response.text
264
265
266 async def test_robots_txt_disallows_api(
267 client: AsyncClient, db_session: AsyncSession
268 ) -> None:
269 """API paths are disallowed — crawlers should use the sitemap, not the REST API."""
270 response = await client.get("/robots.txt")
271 assert response.status_code == 200
272 assert "Disallow: /api/" in response.text
273
274
275 async def test_robots_txt_contains_sitemap_url(
276 client: AsyncClient, db_session: AsyncSession
277 ) -> None:
278 """Sitemap: directive is present and points to /sitemap.xml."""
279 response = await client.get("/robots.txt")
280 assert response.status_code == 200
281 assert "Sitemap:" in response.text
282 assert "sitemap.xml" in response.text
283
284
285 async def test_robots_txt_names_known_agents(
286 client: AsyncClient, db_session: AsyncSession
287 ) -> None:
288 """Known AI discovery bots (GPTBot, ClaudeBot, etc.) appear with explicit Allow."""
289 response = await client.get("/robots.txt")
290 assert response.status_code == 200
291 body = response.text
292 for bot in ("GPTBot", "ClaudeBot", "Googlebot", "CursorBot"):
293 assert bot in body
294
295
296 async def test_robots_txt_no_auth_required(
297 client: AsyncClient, db_session: AsyncSession
298 ) -> None:
299 """robots.txt must be accessible without authentication."""
300 response = await client.get("/robots.txt")
301 assert response.status_code != 401
302 assert response.status_code == 200
File History 1 commit
sha256:9590cee1e0ccd6c76528f005b95d634d80f5019f0dcb7c371e149adc31d1fb65 refactor: enforce gRPC framing on all MWP wire traffic Sonnet 4.6 minor ⚠ 156 days ago