gabriel / musehub public
test_object_store.py python
876 lines 38.8 KB
Raw
sha256:a10adeeb7a0169cb9900f9806ed7a973047258abb6283724fe55e8eb68ff3f0a init: musehub initial commit Human 171 days ago
1 """Tests for the Object Store — Section 3 of test-coverage-checklist.md.
2
3 Coverage layers
4 ───────────────
5 Unit — LocalBackend._path sanitisation, traversal guard, _write idempotency,
6 put/get/exists/delete async wrappers, uri_for absolute-path contract,
7 _detect_file_type and _content_type helpers.
8 Integration — Service layer (musehub_repository): list_objects, get_object_row,
9 get_object_by_path with a real in-memory DB.
10 E2E — HTTP handlers: list objects, get object content, blob meta,
11 public vs private visibility, 404/410/401 error paths.
12 Stress — 50-object fan-out push then list; 20-repo isolation fan-out.
13 Data — Content fidelity (byte-for-byte round-trip), _write idempotency
14 prevents overwrites, disk_path absolute-path regression
15 (sha256: colon sanitisation + objects_dir root must be present).
16 Security — _path traversal rejection (repo_id and object_id), cross-repo
17 object isolation, private-repo auth gate on every endpoint.
18 Performance — LocalBackend put/get latency budgets, list_objects at 100+ rows.
19 """
20 from __future__ import annotations
21
22 import hashlib
23 import os
24 import time
25 import uuid
26 from pathlib import Path
27
28 import pytest
29 import pytest_asyncio
30 from httpx import AsyncClient
31 from sqlalchemy.ext.asyncio import AsyncSession
32
33 from musehub.config import settings
34 from musehub.db import musehub_models as db
35 from musehub.storage.backends import LocalBackend
36 from tests.factories import create_repo
37 from musehub.muse_contracts.json_types import JSONObject, StrDict
38
39
40 # ─────────────────────────────────────────────────────────────────────────────
41 # Helpers
42 # ─────────────────────────────────────────────────────────────────────────────
43
44 def _sha256_id(data: bytes) -> str:
45 return "sha256:" + hashlib.sha256(data).hexdigest()
46
47
48 async def _insert_object(
49 session: AsyncSession,
50 repo_id: str,
51 object_id: str,
52 path: str,
53 disk_path: str,
54 size: int = 0,
55 ) -> db.MusehubObject:
56 obj = db.MusehubObject(
57 object_id=object_id,
58 repo_id=repo_id,
59 path=path,
60 size_bytes=size,
61 disk_path=disk_path,
62 storage_uri=f"local://{disk_path}",
63 )
64 session.add(obj)
65 await session.commit()
66 await session.refresh(obj)
67 return obj
68
69
70 # ─────────────────────────────────────────────────────────────────────────────
71 # Layer 1 — Unit: LocalBackend
72 # ─────────────────────────────────────────────────────────────────────────────
73
74 class TestLocalBackendPath:
75 """_path() must produce the correct file location and reject traversals."""
76
77 def test_safe_path_is_under_root(self, tmp_path: Path) -> None:
78 backend = LocalBackend(objects_dir=str(tmp_path))
79 p = backend._path("myrepo", "sha256:abc")
80 assert str(p).startswith(str(tmp_path))
81
82 def test_colon_in_object_id_sanitised(self, tmp_path: Path) -> None:
83 backend = LocalBackend(objects_dir=str(tmp_path))
84 p = backend._path("myrepo", "sha256:abc123")
85 assert ":" not in p.name
86 assert p.name == "sha256_abc123"
87
88 def test_slash_in_object_id_sanitised(self, tmp_path: Path) -> None:
89 backend = LocalBackend(objects_dir=str(tmp_path))
90 p = backend._path("myrepo", "a/b/c")
91 assert "/" not in p.name
92 assert p.name == "a_b_c"
93
94 def test_traversal_via_repo_id_raises(self, tmp_path: Path) -> None:
95 backend = LocalBackend(objects_dir=str(tmp_path))
96 with pytest.raises(ValueError, match="traversal"):
97 backend._path("../../etc/passwd", "object")
98
99 def test_traversal_via_dotdot_repo_id_raises(self, tmp_path: Path) -> None:
100 backend = LocalBackend(objects_dir=str(tmp_path))
101 with pytest.raises(ValueError, match="traversal"):
102 backend._path("../secret", "obj")
103
104 def test_normal_path_does_not_raise(self, tmp_path: Path) -> None:
105 backend = LocalBackend(objects_dir=str(tmp_path))
106 p = backend._path("repo-123", "sha256:deadbeef")
107 assert p.parent.parent == tmp_path
108
109 def test_uri_for_embeds_absolute_path(self, tmp_path: Path) -> None:
110 """uri_for must return local://<absolute_path> so disk_path is correct."""
111 backend = LocalBackend(objects_dir=str(tmp_path))
112 uri = backend.uri_for("myrepo", "sha256:cafe")
113 assert uri.startswith("local:///")
114 # Stripping the scheme gives the real absolute disk path
115 disk_path = uri.replace("local://", "")
116 assert Path(disk_path).is_absolute()
117 # The path must contain the sanitised object_id, not the raw one
118 assert "sha256_cafe" in disk_path
119 assert "sha256:cafe" not in disk_path
120
121
122 class TestLocalBackendWrite:
123 """_write() is idempotent — second call must not overwrite existing content."""
124
125 def test_write_creates_file(self, tmp_path: Path) -> None:
126 backend = LocalBackend(objects_dir=str(tmp_path))
127 p = backend._path("repo", "obj1")
128 backend._write(p, b"hello")
129 assert p.exists()
130 assert p.read_bytes() == b"hello"
131
132 def test_write_is_idempotent(self, tmp_path: Path) -> None:
133 backend = LocalBackend(objects_dir=str(tmp_path))
134 p = backend._path("repo", "obj2")
135 backend._write(p, b"original")
136 backend._write(p, b"overwrite-attempt")
137 # Content must not change
138 assert p.read_bytes() == b"original"
139
140 def test_write_creates_parent_dirs(self, tmp_path: Path) -> None:
141 backend = LocalBackend(objects_dir=str(tmp_path))
142 p = tmp_path / "deep" / "nested" / "dir" / "file.bin"
143 p.parent.mkdir(parents=True)
144 backend._write(p, b"data")
145 assert p.read_bytes() == b"data"
146
147
148 class TestLocalBackendAsyncOps:
149 """put/get/exists/delete async interface."""
150
151 @pytest.mark.asyncio
152 async def test_put_returns_uri_and_file_exists(self, tmp_path: Path) -> None:
153 backend = LocalBackend(objects_dir=str(tmp_path))
154 uri = await backend.put("repo-a", "sha256:aaa", b"content-a")
155 assert uri.startswith("local://")
156 disk = uri.replace("local://", "")
157 assert Path(disk).exists()
158 assert Path(disk).read_bytes() == b"content-a"
159
160 @pytest.mark.asyncio
161 async def test_get_returns_none_for_missing(self, tmp_path: Path) -> None:
162 backend = LocalBackend(objects_dir=str(tmp_path))
163 result = await backend.get("repo-a", "sha256:missing")
164 assert result is None
165
166 @pytest.mark.asyncio
167 async def test_get_returns_correct_bytes(self, tmp_path: Path) -> None:
168 backend = LocalBackend(objects_dir=str(tmp_path))
169 data = b"\x00\x01\x02\x03" * 1024
170 await backend.put("repo-b", "sha256:bbb", data)
171 result = await backend.get("repo-b", "sha256:bbb")
172 assert result == data
173
174 @pytest.mark.asyncio
175 async def test_exists_false_before_put(self, tmp_path: Path) -> None:
176 backend = LocalBackend(objects_dir=str(tmp_path))
177 assert not await backend.exists("repo-c", "sha256:ccc")
178
179 @pytest.mark.asyncio
180 async def test_exists_true_after_put(self, tmp_path: Path) -> None:
181 backend = LocalBackend(objects_dir=str(tmp_path))
182 await backend.put("repo-c", "sha256:ccc", b"x")
183 assert await backend.exists("repo-c", "sha256:ccc")
184
185 @pytest.mark.asyncio
186 async def test_delete_removes_file(self, tmp_path: Path) -> None:
187 backend = LocalBackend(objects_dir=str(tmp_path))
188 await backend.put("repo-d", "sha256:ddd", b"y")
189 await backend.delete("repo-d", "sha256:ddd")
190 assert not await backend.exists("repo-d", "sha256:ddd")
191
192 @pytest.mark.asyncio
193 async def test_delete_noop_on_missing(self, tmp_path: Path) -> None:
194 backend = LocalBackend(objects_dir=str(tmp_path))
195 # Should not raise
196 await backend.delete("repo-d", "sha256:nonexistent")
197
198 @pytest.mark.asyncio
199 async def test_separate_repos_isolated(self, tmp_path: Path) -> None:
200 backend = LocalBackend(objects_dir=str(tmp_path))
201 await backend.put("repo-e", "sha256:fff", b"for-repo-e")
202 assert not await backend.exists("repo-f", "sha256:fff")
203
204
205 # ─────────────────────────────────────────────────────────────────────────────
206 # Layer 1 (cont.) — Unit: file-type detection helpers
207 # ─────────────────────────────────────────────────────────────────────────────
208
209 class TestDetectFileType:
210 """_detect_file_type covers image, json, xml, and other."""
211
212 def test_json_is_json(self) -> None:
213 from musehub.api.routes.musehub.objects import _detect_file_type
214 assert _detect_file_type("meta.json") == "json"
215
216 def test_png_is_image(self) -> None:
217 from musehub.api.routes.musehub.objects import _detect_file_type
218 assert _detect_file_type("cover.png") == "image"
219
220 def test_webp_is_image(self) -> None:
221 from musehub.api.routes.musehub.objects import _detect_file_type
222 assert _detect_file_type("thumb.webp") == "image"
223
224 def test_jpeg_is_image(self) -> None:
225 from musehub.api.routes.musehub.objects import _detect_file_type
226 assert _detect_file_type("photo.jpeg") == "image"
227
228 def test_xml_is_xml(self) -> None:
229 from musehub.api.routes.musehub.objects import _detect_file_type
230 assert _detect_file_type("score.xml") == "xml"
231
232 def test_unknown_ext_is_other(self) -> None:
233 from musehub.api.routes.musehub.objects import _detect_file_type
234 assert _detect_file_type("file.xyz") == "other"
235
236 def test_no_ext_is_other(self) -> None:
237 from musehub.api.routes.musehub.objects import _detect_file_type
238 assert _detect_file_type("Makefile") == "other"
239
240
241 class TestContentType:
242 """_content_type returns correct MIME type."""
243
244 def test_png_content_type(self) -> None:
245 from musehub.api.routes.musehub.objects import _content_type
246 ct = _content_type("cover.png")
247 assert "png" in ct.lower() or "image" in ct.lower()
248
249 def test_json_content_type(self) -> None:
250 from musehub.api.routes.musehub.objects import _content_type
251 assert "json" in _content_type("data.json")
252
253 def test_webp_content_type(self) -> None:
254 from musehub.api.routes.musehub.objects import _content_type
255 assert "webp" in _content_type("roll.webp")
256
257
258 # ─────────────────────────────────────────────────────────────────────────────
259 # Layer 2 — Integration: service layer
260 # ─────────────────────────────────────────────────────────────────────────────
261
262 class TestServiceListObjects:
263 @pytest.mark.asyncio
264 async def test_empty_repo_returns_empty_list(self, db_session: AsyncSession) -> None:
265 from musehub.services import musehub_repository
266 repo = await create_repo(db_session, visibility="public")
267 result = await musehub_repository.list_objects(db_session, repo.repo_id)
268 assert result == []
269
270 @pytest.mark.asyncio
271 async def test_returns_all_objects_sorted_by_path(
272 self, db_session: AsyncSession, tmp_path: Path
273 ) -> None:
274 from musehub.services import musehub_repository
275 repo = await create_repo(db_session, visibility="public")
276 for name in ("z.bin", "a.bin", "m.bin"):
277 p = tmp_path / name
278 p.write_bytes(b"data")
279 await _insert_object(db_session, repo.repo_id, f"sha256:{name}", name, str(p))
280
281 result = await musehub_repository.list_objects(db_session, repo.repo_id)
282 paths = [r.path for r in result]
283 assert paths == sorted(paths)
284
285 @pytest.mark.asyncio
286 async def test_isolated_to_repo(self, db_session: AsyncSession, tmp_path: Path) -> None:
287 from musehub.services import musehub_repository
288 repo_a = await create_repo(db_session, slug="repo-a-svc", visibility="public")
289 repo_b = await create_repo(db_session, slug="repo-b-svc", visibility="public")
290 p = tmp_path / "obj.bin"
291 p.write_bytes(b"x")
292 await _insert_object(db_session, repo_a.repo_id, "sha256:svc-a", "a.bin", str(p))
293
294 result = await musehub_repository.list_objects(db_session, repo_b.repo_id)
295 assert result == []
296
297
298 class TestServiceGetObjectRow:
299 @pytest.mark.asyncio
300 async def test_returns_row_when_found(
301 self, db_session: AsyncSession, tmp_path: Path
302 ) -> None:
303 from musehub.services import musehub_repository
304 repo = await create_repo(db_session, visibility="public")
305 p = tmp_path / "file.bin"
306 p.write_bytes(b"bytes")
307 await _insert_object(db_session, repo.repo_id, "sha256:rowtest", "file.bin", str(p))
308
309 row = await musehub_repository.get_object_row(db_session, repo.repo_id, "sha256:rowtest")
310 assert row is not None
311 assert row.object_id == "sha256:rowtest"
312
313 @pytest.mark.asyncio
314 async def test_returns_none_when_missing(self, db_session: AsyncSession) -> None:
315 from musehub.services import musehub_repository
316 repo = await create_repo(db_session, visibility="public")
317 row = await musehub_repository.get_object_row(db_session, repo.repo_id, "sha256:ghost")
318 assert row is None
319
320 @pytest.mark.asyncio
321 async def test_wrong_repo_returns_none(
322 self, db_session: AsyncSession, tmp_path: Path
323 ) -> None:
324 from musehub.services import musehub_repository
325 repo_a = await create_repo(db_session, slug="row-a", visibility="public")
326 repo_b = await create_repo(db_session, slug="row-b", visibility="public")
327 p = tmp_path / "f.bin"
328 p.write_bytes(b"x")
329 await _insert_object(db_session, repo_a.repo_id, "sha256:xrepo", "f.bin", str(p))
330
331 row = await musehub_repository.get_object_row(db_session, repo_b.repo_id, "sha256:xrepo")
332 assert row is None
333
334
335 class TestServiceGetObjectByPath:
336 @pytest.mark.asyncio
337 async def test_returns_most_recent_for_path(
338 self, db_session: AsyncSession, tmp_path: Path
339 ) -> None:
340 from musehub.services import musehub_repository
341 import asyncio
342 repo = await create_repo(db_session, visibility="public")
343
344 p1 = tmp_path / "v1.bin"
345 p1.write_bytes(b"v1")
346 await _insert_object(db_session, repo.repo_id, "sha256:v1", "track.bin", str(p1))
347 await asyncio.sleep(0.01) # ensure distinct created_at
348
349 p2 = tmp_path / "v2.bin"
350 p2.write_bytes(b"v2")
351 await _insert_object(db_session, repo.repo_id, "sha256:v2", "track.bin", str(p2))
352
353 row = await musehub_repository.get_object_by_path(db_session, repo.repo_id, "track.bin")
354 assert row is not None
355 assert row.object_id == "sha256:v2"
356
357 @pytest.mark.asyncio
358 async def test_returns_none_for_missing_path(self, db_session: AsyncSession) -> None:
359 from musehub.services import musehub_repository
360 repo = await create_repo(db_session, visibility="public")
361 row = await musehub_repository.get_object_by_path(db_session, repo.repo_id, "ghost.bin")
362 assert row is None
363
364
365 # ─────────────────────────────────────────────────────────────────────────────
366 # Layer 3 — E2E: HTTP endpoints
367 # ─────────────────────────────────────────────────────────────────────────────
368
369 class TestListObjectsEndpoint:
370 @pytest.mark.asyncio
371 async def test_public_repo_empty_list(
372 self, client: AsyncClient, db_session: AsyncSession
373 ) -> None:
374 repo = await create_repo(db_session, slug="e2e-list-empty", visibility="public")
375 resp = await client.get(f"/api/repos/{repo.repo_id}/objects")
376 assert resp.status_code == 200
377 assert resp.json()["objects"] == []
378
379 @pytest.mark.asyncio
380 async def test_public_repo_returns_metadata(
381 self, client: AsyncClient, db_session: AsyncSession, tmp_path: Path
382 ) -> None:
383 repo = await create_repo(db_session, slug="e2e-list-meta", visibility="public")
384 p = tmp_path / "t.bin"
385 p.write_bytes(b"data")
386 await _insert_object(db_session, repo.repo_id, "sha256:list1", "t.bin", str(p), size=4)
387
388 resp = await client.get(f"/api/repos/{repo.repo_id}/objects")
389 assert resp.status_code == 200
390 objects = resp.json()["objects"]
391 assert len(objects) == 1
392 assert objects[0]["path"] == "t.bin"
393 assert objects[0]["sizeBytes"] == 4
394
395 @pytest.mark.asyncio
396 async def test_unknown_repo_returns_404(
397 self, client: AsyncClient, db_session: AsyncSession
398 ) -> None:
399 resp = await client.get(f"/api/repos/{uuid.uuid4()}/objects")
400 assert resp.status_code == 404
401
402 @pytest.mark.asyncio
403 async def test_private_repo_without_auth_returns_401(
404 self, client: AsyncClient, db_session: AsyncSession
405 ) -> None:
406 repo = await create_repo(db_session, slug="e2e-list-priv", visibility="private")
407 resp = await client.get(f"/api/repos/{repo.repo_id}/objects")
408 assert resp.status_code == 401
409
410 @pytest.mark.asyncio
411 async def test_private_repo_with_auth_returns_200(
412 self,
413 client: AsyncClient,
414 db_session: AsyncSession,
415 auth_headers: StrDict,
416 tmp_path: Path,
417 ) -> None:
418 repo = await create_repo(db_session, slug="e2e-list-priv-auth", visibility="private")
419 resp = await client.get(
420 f"/api/repos/{repo.repo_id}/objects", headers=auth_headers
421 )
422 assert resp.status_code == 200
423
424
425 class TestGetObjectContentEndpoint:
426 @pytest.mark.asyncio
427 async def test_returns_file_bytes(
428 self, client: AsyncClient, db_session: AsyncSession, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
429 ) -> None:
430 monkeypatch.setattr(settings, "musehub_objects_dir", str(tmp_path))
431 repo = await create_repo(db_session, slug="e2e-content-ok", visibility="public")
432 data = b"\x00\x01\x02DATA-content"
433 p = tmp_path / "song.bin"
434 p.write_bytes(data)
435 obj_id = "sha256:content-ok"
436 await _insert_object(db_session, repo.repo_id, obj_id, "song.bin", str(p), len(data))
437
438 resp = await client.get(f"/api/repos/{repo.repo_id}/objects/{obj_id}/content")
439 assert resp.status_code == 200
440 assert resp.content == data
441
442 @pytest.mark.asyncio
443 async def test_unknown_object_returns_404(
444 self, client: AsyncClient, db_session: AsyncSession
445 ) -> None:
446 repo = await create_repo(db_session, slug="e2e-content-404obj", visibility="public")
447 resp = await client.get(f"/api/repos/{repo.repo_id}/objects/sha256:ghost/content")
448 assert resp.status_code == 404
449
450 @pytest.mark.asyncio
451 async def test_unknown_repo_returns_404(
452 self, client: AsyncClient, db_session: AsyncSession
453 ) -> None:
454 resp = await client.get(f"/api/repos/{uuid.uuid4()}/objects/sha256:x/content")
455 assert resp.status_code == 404
456
457 @pytest.mark.asyncio
458 async def test_missing_disk_file_returns_410(
459 self, client: AsyncClient, db_session: AsyncSession, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
460 ) -> None:
461 monkeypatch.setattr(settings, "musehub_objects_dir", str(tmp_path))
462 repo = await create_repo(db_session, slug="e2e-content-410", visibility="public")
463 # Point disk_path at a non-existent file inside the storage root
464 gone_path = str(tmp_path / "gone.bin")
465 await _insert_object(db_session, repo.repo_id, "sha256:gone", "gone.bin", gone_path)
466
467 resp = await client.get(f"/api/repos/{repo.repo_id}/objects/sha256:gone/content")
468 assert resp.status_code == 410
469
470 @pytest.mark.asyncio
471 async def test_private_repo_without_auth_returns_401(
472 self, client: AsyncClient, db_session: AsyncSession, tmp_path: Path
473 ) -> None:
474 repo = await create_repo(db_session, slug="e2e-content-priv", visibility="private")
475 p = tmp_path / "secret.bin"
476 p.write_bytes(b"secret")
477 await _insert_object(db_session, repo.repo_id, "sha256:priv1", "secret.bin", str(p))
478
479 resp = await client.get(f"/api/repos/{repo.repo_id}/objects/sha256:priv1/content")
480 assert resp.status_code == 401
481
482
483 class TestGetBlobMetaEndpoint:
484 @pytest.mark.asyncio
485 async def test_returns_blob_metadata(
486 self, client: AsyncClient, db_session: AsyncSession, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
487 ) -> None:
488 monkeypatch.setattr(settings, "musehub_objects_dir", str(tmp_path))
489 repo = await create_repo(db_session, slug="e2e-blob-meta", visibility="public")
490 p = tmp_path / "score.json"
491 p.write_text('{"notes": []}')
492 await _insert_object(
493 db_session, repo.repo_id, "sha256:blobmeta1", "score.json", str(p), size=13
494 )
495
496 resp = await client.get(f"/api/repos/{repo.repo_id}/blob/main/score.json")
497 assert resp.status_code == 200
498 body = resp.json()
499 assert body["path"] == "score.json"
500 assert body["sizeBytes"] == 13
501 assert body["fileType"] == "json"
502 # JSON files under 256 KB get content_text embedded
503 assert body["contentText"] is not None
504
505 @pytest.mark.asyncio
506 async def test_unknown_path_returns_404(
507 self, client: AsyncClient, db_session: AsyncSession
508 ) -> None:
509 repo = await create_repo(db_session, slug="e2e-blob-404", visibility="public")
510 resp = await client.get(f"/api/repos/{repo.repo_id}/blob/main/ghost.bin")
511 assert resp.status_code == 404
512
513 @pytest.mark.asyncio
514 async def test_unknown_repo_returns_404(
515 self, client: AsyncClient, db_session: AsyncSession
516 ) -> None:
517 resp = await client.get(f"/api/repos/{uuid.uuid4()}/blob/main/any.bin")
518 assert resp.status_code == 404
519
520 @pytest.mark.asyncio
521 async def test_private_repo_without_auth_returns_401(
522 self, client: AsyncClient, db_session: AsyncSession
523 ) -> None:
524 repo = await create_repo(db_session, slug="e2e-blob-priv", visibility="private")
525 resp = await client.get(f"/api/repos/{repo.repo_id}/blob/main/any.bin")
526 assert resp.status_code == 401
527
528 @pytest.mark.asyncio
529 async def test_binary_file_has_no_content_text(
530 self, client: AsyncClient, db_session: AsyncSession, tmp_path: Path
531 ) -> None:
532 """Non-text files (type 'other') must not get content_text embedded."""
533 repo = await create_repo(db_session, slug="e2e-blob-bin", visibility="public")
534 p = tmp_path / "artifact.bin"
535 p.write_bytes(b"\x00\x01\x02\x03")
536 await _insert_object(
537 db_session, repo.repo_id, "sha256:blobbin", "artifact.bin", str(p), size=4
538 )
539
540 resp = await client.get(f"/api/repos/{repo.repo_id}/blob/main/artifact.bin")
541 assert resp.status_code == 200
542 assert resp.json()["contentText"] is None
543
544
545 # ─────────────────────────────────────────────────────────────────────────────
546 # Layer 4 — Stress
547 # ─────────────────────────────────────────────────────────────────────────────
548
549 def _wire_push(content: bytes, oid: str, path: str = "artifact.bin") -> JSONObject:
550 """Build a minimal wire push payload dict (raw msgpack-serialisable)."""
551 return {
552 "bundle": {
553 "commits": [],
554 "snapshots": [],
555 "objects": [{"object_id": oid, "path": path, "content": content}],
556 },
557 "branch": "main",
558 "force": False,
559 }
560
561
562 class TestObjectStoreStress:
563 @pytest.mark.asyncio
564 async def test_50_objects_push_then_list_all_present(
565 self, client: AsyncClient, db_session: AsyncSession, wire_headers: StrDict
566 ) -> None:
567 """Push 50 objects via the wire protocol; list must return all 50."""
568 import msgpack
569
570 repo = await create_repo(db_session, slug="stress-50obj", owner="test-user-wire",
571 owner_user_id="wire-test-user-id", visibility="public")
572
573 OBJECT_COUNT = 50
574 objects = []
575 for i in range(OBJECT_COUNT):
576 content = f"DATA-{i:04d}".encode() * 128
577 oid = _sha256_id(content)
578 objects.append({"object_id": oid, "path": f"artifact_{i:04d}.bin", "content": content})
579
580 payload = {"bundle": {"commits": [], "snapshots": [], "objects": objects},
581 "branch": "main", "force": False}
582 push_resp = await client.post(
583 f"/{repo.owner}/{repo.slug}/push",
584 content=msgpack.packb(payload),
585 headers=wire_headers,
586 )
587 assert push_resp.status_code == 200
588
589 list_resp = await client.get(f"/api/repos/{repo.repo_id}/objects")
590 assert list_resp.status_code == 200
591 assert len(list_resp.json()["objects"]) == OBJECT_COUNT
592
593 @pytest.mark.asyncio
594 async def test_20_repo_fan_out_isolated(
595 self, client: AsyncClient, db_session: AsyncSession, wire_headers: StrDict
596 ) -> None:
597 """20 repos each pushed one unique object; listing each repo returns exactly 1."""
598 import msgpack
599
600 REPO_COUNT = 20
601 repos = []
602 for i in range(REPO_COUNT):
603 repo = await create_repo(db_session, slug=f"fanout-{i:02d}",
604 owner="test-user-wire",
605 owner_user_id="wire-test-user-id",
606 visibility="public")
607 repos.append(repo)
608 content = f"unique-{i}".encode()
609 oid = _sha256_id(content)
610 resp = await client.post(
611 f"/{repo.owner}/{repo.slug}/push",
612 content=msgpack.packb(_wire_push(content, oid)),
613 headers=wire_headers,
614 )
615 assert resp.status_code == 200
616
617 for repo in repos:
618 resp = await client.get(f"/api/repos/{repo.repo_id}/objects")
619 assert resp.status_code == 200
620 assert len(resp.json()["objects"]) == 1
621
622
623 # ─────────────────────────────────────────────────────────────────────────────
624 # Layer 5 — Data Integrity
625 # ─────────────────────────────────────────────────────────────────────────────
626
627 class TestDataIntegrity:
628 @pytest.mark.asyncio
629 async def test_disk_path_points_to_real_file_after_wire_push(
630 self, client: AsyncClient, db_session: AsyncSession, wire_headers: StrDict
631 ) -> None:
632 """Regression: disk_path must be an absolute path with sanitised colon.
633
634 Before the uri_for fix, disk_path stored "repo-id/sha256:abc" — a relative
635 path with a literal colon — so get_object_content always returned 410 Gone.
636 """
637 import msgpack
638
639 content = b"DATA-regression-test" * 10
640 oid = _sha256_id(content) # e.g. "sha256:<hex>"
641 repo = await create_repo(db_session, slug="diskpath-regression",
642 owner="test-user-wire",
643 owner_user_id="wire-test-user-id",
644 visibility="public")
645
646 push_resp = await client.post(
647 f"/{repo.owner}/{repo.slug}/push",
648 content=msgpack.packb(_wire_push(content, oid, "artifact.bin")),
649 headers=wire_headers,
650 )
651 assert push_resp.status_code == 200
652
653 # Verify the stored disk_path is absolute and has no colon in the filename
654 from musehub.services import musehub_repository
655 row = await musehub_repository.get_object_row(db_session, repo.repo_id, oid)
656 assert row is not None
657 assert Path(row.disk_path).is_absolute(), (
658 f"disk_path is not absolute: {row.disk_path!r}"
659 )
660 assert ":" not in Path(row.disk_path).name, (
661 f"disk_path filename still has colon: {row.disk_path!r}"
662 )
663 assert Path(row.disk_path).exists(), (
664 f"disk_path does not exist on disk: {row.disk_path!r}"
665 )
666
667 @pytest.mark.asyncio
668 async def test_content_round_trip_via_http(
669 self, client: AsyncClient, db_session: AsyncSession, wire_headers: StrDict
670 ) -> None:
671 """Content retrieved via HTTP must match pushed bytes exactly."""
672 import msgpack
673
674 content = os.urandom(512)
675 oid = _sha256_id(content)
676 repo = await create_repo(db_session, slug="roundtrip-test",
677 owner="test-user-wire",
678 owner_user_id="wire-test-user-id",
679 visibility="public")
680
681 push_resp = await client.post(
682 f"/{repo.owner}/{repo.slug}/push",
683 content=msgpack.packb(_wire_push(content, oid)),
684 headers=wire_headers,
685 )
686 assert push_resp.status_code == 200
687
688 content_resp = await client.get(
689 f"/api/repos/{repo.repo_id}/objects/{oid}/content"
690 )
691 assert content_resp.status_code == 200
692 assert content_resp.content == content
693
694 @pytest.mark.asyncio
695 async def test_idempotent_push_stores_object_once(
696 self, client: AsyncClient, db_session: AsyncSession, wire_headers: StrDict
697 ) -> None:
698 """Pushing the same object twice must not duplicate DB rows."""
699 import msgpack
700 from sqlalchemy import select
701
702 content = b"idempotent-content"
703 oid = _sha256_id(content)
704 repo = await create_repo(db_session, slug="idempotent-obj",
705 owner="test-user-wire",
706 owner_user_id="wire-test-user-id",
707 visibility="public")
708
709 payload = msgpack.packb(_wire_push(content, oid))
710 for _ in range(2):
711 r = await client.post(
712 f"/{repo.owner}/{repo.slug}/push",
713 content=payload, headers=wire_headers,
714 )
715 assert r.status_code == 200
716
717 from sqlalchemy import func
718 count_stmt = (
719 select(func.count())
720 .select_from(db.MusehubObject)
721 .where(db.MusehubObject.object_id == oid)
722 )
723 count = (await db_session.execute(count_stmt)).scalar_one()
724 assert count == 1
725
726 @pytest.mark.asyncio
727 async def test_local_backend_put_idempotent_does_not_overwrite(
728 self, tmp_path: Path
729 ) -> None:
730 """_write: second put with different content must not overwrite the first."""
731 backend = LocalBackend(objects_dir=str(tmp_path))
732 await backend.put("repo", "sha256:idem", b"original")
733 await backend.put("repo", "sha256:idem", b"changed")
734 result = await backend.get("repo", "sha256:idem")
735 assert result == b"original"
736
737
738 # ─────────────────────────────────────────────────────────────────────────────
739 # Layer 6 — Security
740 # ─────────────────────────────────────────────────────────────────────────────
741
742 class TestObjectStoreSecurity:
743 def test_traversal_via_repo_id_raises_value_error(self, tmp_path: Path) -> None:
744 backend = LocalBackend(objects_dir=str(tmp_path))
745 with pytest.raises(ValueError, match="traversal"):
746 backend._path("../../etc", "obj")
747
748 def test_traversal_via_object_id_colon_is_sanitised(self, tmp_path: Path) -> None:
749 """Colons in object_id are sanitised to underscores, not interpreted as path."""
750 backend = LocalBackend(objects_dir=str(tmp_path))
751 p = backend._path("repo", "sha256:../../../../etc/passwd")
752 # After sanitisation the entire object_id is flattened to one filename component
753 assert p.parent == tmp_path / "repo"
754
755 def test_traversal_via_repo_id_double_slash(self, tmp_path: Path) -> None:
756 backend = LocalBackend(objects_dir=str(tmp_path))
757 with pytest.raises(ValueError, match="traversal"):
758 backend._path("/../../../etc", "obj")
759
760 @pytest.mark.asyncio
761 async def test_private_repo_list_objects_requires_auth(
762 self, client: AsyncClient, db_session: AsyncSession
763 ) -> None:
764 repo = await create_repo(db_session, slug="sec-priv-list", visibility="private")
765 resp = await client.get(f"/api/repos/{repo.repo_id}/objects")
766 assert resp.status_code == 401
767
768 @pytest.mark.asyncio
769 async def test_private_repo_content_requires_auth(
770 self, client: AsyncClient, db_session: AsyncSession, tmp_path: Path
771 ) -> None:
772 repo = await create_repo(db_session, slug="sec-priv-content", visibility="private")
773 p = tmp_path / "x.bin"
774 p.write_bytes(b"x")
775 await _insert_object(db_session, repo.repo_id, "sha256:sec1", "x.bin", str(p))
776
777 resp = await client.get(f"/api/repos/{repo.repo_id}/objects/sha256:sec1/content")
778 assert resp.status_code == 401
779
780 @pytest.mark.asyncio
781 async def test_cross_repo_object_not_visible_in_other_repo(
782 self, client: AsyncClient, db_session: AsyncSession, tmp_path: Path
783 ) -> None:
784 repo_a = await create_repo(db_session, slug="sec-cross-a", visibility="public")
785 repo_b = await create_repo(db_session, slug="sec-cross-b", visibility="public")
786 p = tmp_path / "a.bin"
787 p.write_bytes(b"a")
788 await _insert_object(db_session, repo_a.repo_id, "sha256:cross1", "a.bin", str(p))
789
790 # Object in repo_a must not appear in repo_b listing
791 resp = await client.get(f"/api/repos/{repo_b.repo_id}/objects")
792 assert resp.status_code == 200
793 assert resp.json()["objects"] == []
794
795 # Direct content fetch via repo_b must return 404
796 resp = await client.get(
797 f"/api/repos/{repo_b.repo_id}/objects/sha256:cross1/content"
798 )
799 assert resp.status_code == 404
800
801
802 # ─────────────────────────────────────────────────────────────────────────────
803 # Layer 7 — Performance
804 # ─────────────────────────────────────────────────────────────────────────────
805
806 class TestObjectStorePerformance:
807 @pytest.mark.asyncio
808 async def test_local_backend_put_1kb_under_50ms(self, tmp_path: Path) -> None:
809 backend = LocalBackend(objects_dir=str(tmp_path))
810 data = b"x" * 1024
811 times = []
812 for i in range(50):
813 oid = f"sha256:perf{i:04d}"
814 t0 = time.perf_counter()
815 await backend.put(f"repo", oid, data)
816 times.append(time.perf_counter() - t0)
817 median_ms = sorted(times)[len(times) // 2] * 1000
818 assert median_ms < 50, f"put median {median_ms:.1f}ms exceeded 50ms budget"
819
820 @pytest.mark.asyncio
821 async def test_local_backend_get_1kb_under_10ms(self, tmp_path: Path) -> None:
822 backend = LocalBackend(objects_dir=str(tmp_path))
823 data = b"y" * 1024
824 await backend.put("repo", "sha256:getperf", data)
825 times = []
826 for _ in range(100):
827 t0 = time.perf_counter()
828 _ = await backend.get("repo", "sha256:getperf")
829 times.append(time.perf_counter() - t0)
830 median_ms = sorted(times)[len(times) // 2] * 1000
831 assert median_ms < 10, f"get median {median_ms:.1f}ms exceeded 10ms budget"
832
833 @pytest.mark.asyncio
834 async def test_list_objects_100_rows_under_200ms(
835 self, client: AsyncClient, db_session: AsyncSession, tmp_path: Path
836 ) -> None:
837 repo = await create_repo(db_session, slug="perf-list-100", visibility="public")
838 for i in range(100):
839 p = tmp_path / f"f{i:03d}.bin"
840 p.write_bytes(b"m")
841 await _insert_object(
842 db_session, repo.repo_id, f"sha256:perf{i:03d}", f"f{i:03d}.bin", str(p), 1
843 )
844 # warm-up
845 await client.get(f"/api/repos/{repo.repo_id}/objects")
846
847 times = []
848 for _ in range(10):
849 t0 = time.perf_counter()
850 resp = await client.get(f"/api/repos/{repo.repo_id}/objects")
851 times.append(time.perf_counter() - t0)
852 assert resp.status_code == 200
853 assert len(resp.json()["objects"]) == 100
854 median_ms = sorted(times)[len(times) // 2] * 1000
855 assert median_ms < 200, f"list_objects median {median_ms:.1f}ms exceeded 200ms"
856
857 @pytest.mark.asyncio
858 async def test_get_object_content_1mb_under_500ms(
859 self, client: AsyncClient, db_session: AsyncSession, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
860 ) -> None:
861 monkeypatch.setattr(settings, "musehub_objects_dir", str(tmp_path))
862 repo = await create_repo(db_session, slug="perf-content-1mb", visibility="public")
863 data = os.urandom(1024 * 1024)
864 p = tmp_path / "big.bin"
865 p.write_bytes(data)
866 oid = "sha256:perf1mb"
867 await _insert_object(db_session, repo.repo_id, oid, "big.bin", str(p), len(data))
868
869 # warm-up
870 await client.get(f"/api/repos/{repo.repo_id}/objects/{oid}/content")
871
872 t0 = time.perf_counter()
873 resp = await client.get(f"/api/repos/{repo.repo_id}/objects/{oid}/content")
874 elapsed_ms = (time.perf_counter() - t0) * 1000
875 assert resp.status_code == 200
876 assert elapsed_ms < 500, f"1 MB content serve took {elapsed_ms:.1f}ms > 500ms"
File History 1 commit
sha256:a10adeeb7a0169cb9900f9806ed7a973047258abb6283724fe55e8eb68ff3f0a init: musehub initial commit Human 171 days ago