test_storage_backends.py
python
sha256:763eb2cb8675073b84c19345b27586d2ed939a9aee97c5479b69f502f1a70eff
fix(tests): update test suite to match current implementation
Sonnet 4.6
patch
119 days ago
| 1 | """Section 33 — Storage Backends: 7-layer test suite. |
| 2 | |
| 3 | Covers: |
| 4 | musehub/storage/backends.py — StorageBackend protocol, LocalBackend, S3Backend, get_backend, decode_b64 |
| 5 | |
| 6 | Key behaviour under test: |
| 7 | LocalBackend: |
| 8 | - put/get/exists/delete round-trip (per-repo layout) |
| 9 | - _path requires repo_root — raises ValueError without it |
| 10 | - _path(oid, repo_root) produces objects/sha256/<2hex>/<62hex> layout |
| 11 | - put is idempotent (second write to same path is a no-op) |
| 12 | - uri_for returns "local://" prefix |
| 13 | - missing object → get returns None, exists returns False, delete is a no-op |
| 14 | |
| 15 | S3Backend: |
| 16 | - _key preserves colons (colons are valid in S3/R2 keys) |
| 17 | - uri_for returns "s3://<bucket>/..." prefix |
| 18 | - put/get/exists/delete delegated to boto3 client (tested via mock) |
| 19 | - credential/boto3 errors on get/exists → return None/False (no raise) |
| 20 | - _s3_put always calls put_object (idempotent — HeadObject check removed; deduplication is upstream) |
| 21 | |
| 22 | get_backend: |
| 23 | - returns LocalBackend when aws_s3_asset_bucket is unset |
| 24 | - returns S3Backend when aws_s3_asset_bucket is set |
| 25 | |
| 26 | decode_b64: |
| 27 | - decodes standard base64 |
| 28 | - handles missing padding (1, 2, 3 missing "=" chars) |
| 29 | """ |
| 30 | from __future__ import annotations |
| 31 | |
| 32 | import asyncio |
| 33 | import secrets |
| 34 | import time |
| 35 | from pathlib import Path |
| 36 | from types import SimpleNamespace |
| 37 | from typing import TypedDict |
| 38 | from unittest.mock import MagicMock, patch |
| 39 | |
| 40 | import pytest |
| 41 | |
| 42 | from musehub.storage.backends import LocalBackend, S3Backend, decode_b64, get_backend |
| 43 | from muse.core.types import long_id |
| 44 | |
| 45 | |
| 46 | # ── helpers ─────────────────────────────────────────────────────────────────── |
| 47 | |
| 48 | |
| 49 | class _S3GetResponse(TypedDict): |
| 50 | Body: MagicMock |
| 51 | |
| 52 | |
| 53 | def _uid() -> str: |
| 54 | return secrets.token_hex(16) |
| 55 | |
| 56 | |
| 57 | def _backend() -> LocalBackend: |
| 58 | return LocalBackend() |
| 59 | |
| 60 | |
| 61 | def _repo_root(tmp_path: Path) -> Path: |
| 62 | return tmp_path / "repos" / "gabriel" / "test-repo" |
| 63 | |
| 64 | |
| 65 | # ═══════════════════════════════════════════════════════════════════════════════ |
| 66 | # Layer 1 — Unit (pure logic, no filesystem I/O) |
| 67 | # ═══════════════════════════════════════════════════════════════════════════════ |
| 68 | |
| 69 | |
| 70 | class TestUnitLocalBackend: |
| 71 | def test_path_requires_repo_root(self, tmp_path: Path) -> None: |
| 72 | b = _backend() |
| 73 | with pytest.raises(ValueError, match="repo_root"): |
| 74 | b._path(long_id("a" * 64), repo_root=None) |
| 75 | |
| 76 | def test_path_with_repo_root_uses_algo_shard_layout(self, tmp_path: Path) -> None: |
| 77 | b = _backend() |
| 78 | repo_root = _repo_root(tmp_path) |
| 79 | hex64 = "ab" + "c" * 62 |
| 80 | oid = long_id(hex64) |
| 81 | p = b._path(oid, repo_root=repo_root) |
| 82 | expected = repo_root / "objects" / "sha256" / "ab" / ("c" * 62) |
| 83 | assert p == expected |
| 84 | |
| 85 | def test_uri_for_has_local_prefix(self, tmp_path: Path) -> None: |
| 86 | b = _backend() |
| 87 | repo_root = _repo_root(tmp_path) |
| 88 | oid = long_id("a" * 64) |
| 89 | uri = b.uri_for(oid, repo_root=repo_root) |
| 90 | assert uri.startswith("local://") |
| 91 | |
| 92 | def test_uri_for_contains_obj(self, tmp_path: Path) -> None: |
| 93 | b = _backend() |
| 94 | repo_root = _repo_root(tmp_path) |
| 95 | oid = long_id("a" * 64) |
| 96 | uri = b.uri_for(oid, repo_root=repo_root) |
| 97 | assert "sha256" in uri |
| 98 | |
| 99 | |
| 100 | class TestUnitS3Backend: |
| 101 | def test_key_basic(self) -> None: |
| 102 | b = S3Backend(bucket="test-bucket", region="us-east-1") |
| 103 | key = b._key("abc123") |
| 104 | assert "abc123" in key |
| 105 | |
| 106 | def test_key_preserves_colon(self) -> None: |
| 107 | b = S3Backend(bucket="b", region="us-east-1") |
| 108 | key = b._key("sha256:deadbeef") |
| 109 | assert "sha256:deadbeef" in key |
| 110 | |
| 111 | def test_uri_for_s3_prefix(self) -> None: |
| 112 | b = S3Backend(bucket="my-bucket", region="us-east-1") |
| 113 | uri = b.uri_for("obj1") |
| 114 | assert uri.startswith("s3://my-bucket/") |
| 115 | |
| 116 | def test_uri_for_preserves_colon_in_key(self) -> None: |
| 117 | b = S3Backend(bucket="b", region="us-east-1") |
| 118 | uri = b.uri_for("sha256:abc") |
| 119 | assert "sha256:abc" in uri |
| 120 | |
| 121 | |
| 122 | class TestUnitDecodeb64: |
| 123 | def test_standard_base64(self) -> None: |
| 124 | import base64 |
| 125 | data = b"hello world" |
| 126 | encoded = base64.b64encode(data).decode() |
| 127 | assert decode_b64(encoded) == data |
| 128 | |
| 129 | def test_missing_one_padding(self) -> None: |
| 130 | import base64 |
| 131 | data = b"hi" |
| 132 | encoded = base64.b64encode(data).decode().rstrip("=") |
| 133 | assert decode_b64(encoded) == data |
| 134 | |
| 135 | def test_missing_two_padding(self) -> None: |
| 136 | import base64 |
| 137 | data = b"h" |
| 138 | encoded = base64.b64encode(data).decode().rstrip("=") |
| 139 | assert decode_b64(encoded) == data |
| 140 | |
| 141 | def test_already_padded(self) -> None: |
| 142 | import base64 |
| 143 | data = b"test" |
| 144 | encoded = base64.b64encode(data).decode() |
| 145 | assert decode_b64(encoded) == data |
| 146 | |
| 147 | def test_empty_string(self) -> None: |
| 148 | assert decode_b64("") == b"" |
| 149 | |
| 150 | |
| 151 | class TestUnitGetBackend: |
| 152 | def test_get_backend_returns_local_when_no_s3_bucket(self) -> None: |
| 153 | with patch("musehub.storage.backends.settings") as mock_settings: |
| 154 | mock_settings.r2_bucket = None |
| 155 | mock_settings.aws_s3_asset_bucket = None |
| 156 | result = get_backend() |
| 157 | assert isinstance(result, LocalBackend) |
| 158 | |
| 159 | def test_get_backend_returns_s3_when_bucket_set(self) -> None: |
| 160 | with patch("musehub.storage.backends.settings") as mock_settings: |
| 161 | mock_settings.aws_s3_asset_bucket = "my-bucket" |
| 162 | mock_settings.aws_region = "us-east-1" |
| 163 | mock_settings.r2_bucket = None |
| 164 | result = get_backend() |
| 165 | assert isinstance(result, S3Backend) |
| 166 | |
| 167 | def test_get_backend_with_owner_slug_returns_bound_local(self, tmp_path: Path) -> None: |
| 168 | """get_backend(owner, slug) returns a LocalBackend pre-bound to that repo_root.""" |
| 169 | with patch("musehub.storage.backends.settings") as mock_settings: |
| 170 | mock_settings.r2_bucket = None |
| 171 | mock_settings.aws_s3_asset_bucket = None |
| 172 | mock_settings.musehub_repos_dir = str(tmp_path) |
| 173 | result = get_backend("gabriel", "myrepo") |
| 174 | assert isinstance(result, LocalBackend) |
| 175 | assert result._repo_root is not None |
| 176 | |
| 177 | async def test_get_backend_bound_local_get_needs_no_repo_root_kwarg(self, tmp_path: Path) -> None: |
| 178 | """A backend from get_backend(owner, slug) satisfies storage.get(oid) with no extra args.""" |
| 179 | with patch("musehub.storage.backends.settings") as mock_settings: |
| 180 | mock_settings.r2_bucket = None |
| 181 | mock_settings.aws_s3_asset_bucket = None |
| 182 | mock_settings.musehub_repos_dir = str(tmp_path) |
| 183 | backend = get_backend("gabriel", "myrepo") |
| 184 | |
| 185 | oid = long_id("a" * 64) |
| 186 | await backend.put(oid, b"hello") |
| 187 | result = await backend.get(oid) |
| 188 | assert result == b"hello" |
| 189 | |
| 190 | async def test_get_backend_bound_local_exists_needs_no_repo_root_kwarg(self, tmp_path: Path) -> None: |
| 191 | with patch("musehub.storage.backends.settings") as mock_settings: |
| 192 | mock_settings.r2_bucket = None |
| 193 | mock_settings.aws_s3_asset_bucket = None |
| 194 | mock_settings.musehub_repos_dir = str(tmp_path) |
| 195 | backend = get_backend("gabriel", "myrepo") |
| 196 | |
| 197 | oid = long_id("b" * 64) |
| 198 | assert await backend.exists(oid) is False |
| 199 | await backend.put(oid, b"data") |
| 200 | assert await backend.exists(oid) is True |
| 201 | |
| 202 | def test_get_backend_no_args_local_returns_unbound(self) -> None: |
| 203 | """get_backend() with no owner/slug still returns LocalBackend — unbound.""" |
| 204 | with patch("musehub.storage.backends.settings") as mock_settings: |
| 205 | mock_settings.r2_bucket = None |
| 206 | mock_settings.aws_s3_asset_bucket = None |
| 207 | result = get_backend() |
| 208 | assert isinstance(result, LocalBackend) |
| 209 | assert result._repo_root is None |
| 210 | |
| 211 | |
| 212 | class TestUnitLocalBackendBoundInit: |
| 213 | """LocalBackend(repo_root=path) binds repo_root at construction; methods need no kwarg.""" |
| 214 | |
| 215 | def test_bound_backend_stores_repo_root(self, tmp_path: Path) -> None: |
| 216 | root = tmp_path / "gabriel" / "myrepo" |
| 217 | b = LocalBackend(repo_root=root) |
| 218 | assert b._repo_root == root |
| 219 | |
| 220 | def test_unbound_backend_has_none_repo_root(self) -> None: |
| 221 | b = LocalBackend() |
| 222 | assert b._repo_root is None |
| 223 | |
| 224 | async def test_bound_put_and_get_no_kwarg(self, tmp_path: Path) -> None: |
| 225 | root = tmp_path / "gabriel" / "myrepo" |
| 226 | b = LocalBackend(repo_root=root) |
| 227 | oid = long_id("c" * 64) |
| 228 | await b.put(oid, b"bound put") |
| 229 | result = await b.get(oid) |
| 230 | assert result == b"bound put" |
| 231 | |
| 232 | async def test_bound_exists_no_kwarg(self, tmp_path: Path) -> None: |
| 233 | root = tmp_path / "gabriel" / "myrepo" |
| 234 | b = LocalBackend(repo_root=root) |
| 235 | oid = long_id("d" * 64) |
| 236 | assert await b.exists(oid) is False |
| 237 | await b.put(oid, b"x") |
| 238 | assert await b.exists(oid) is True |
| 239 | |
| 240 | async def test_bound_delete_no_kwarg(self, tmp_path: Path) -> None: |
| 241 | root = tmp_path / "gabriel" / "myrepo" |
| 242 | b = LocalBackend(repo_root=root) |
| 243 | oid = long_id("e" * 64) |
| 244 | await b.put(oid, b"y") |
| 245 | await b.delete(oid) |
| 246 | assert await b.exists(oid) is False |
| 247 | |
| 248 | async def test_kwarg_overrides_bound_repo_root(self, tmp_path: Path) -> None: |
| 249 | """Explicit repo_root kwarg at call time takes precedence over bound root.""" |
| 250 | root_a = tmp_path / "a" / "repo" |
| 251 | root_b = tmp_path / "b" / "repo" |
| 252 | b = LocalBackend(repo_root=root_a) |
| 253 | oid = long_id("f" * 64) |
| 254 | await b.put(oid, b"in-b", repo_root=root_b) |
| 255 | assert await b.exists(oid, repo_root=root_b) is True |
| 256 | assert await b.exists(oid, repo_root=root_a) is False |
| 257 | assert await b.get(oid) is None # root_a doesn't have it |
| 258 | assert await b.get(oid, repo_root=root_b) == b"in-b" |
| 259 | |
| 260 | async def test_unbound_get_without_kwarg_raises(self) -> None: |
| 261 | b = LocalBackend() |
| 262 | oid = long_id("0" * 64) |
| 263 | with pytest.raises(ValueError, match="repo_root"): |
| 264 | await b.get(oid) |
| 265 | |
| 266 | |
| 267 | # ═══════════════════════════════════════════════════════════════════════════════ |
| 268 | # Layer 2 — Integration (real filesystem I/O via tmp_path) |
| 269 | # ═══════════════════════════════════════════════════════════════════════════════ |
| 270 | |
| 271 | |
| 272 | class TestIntegrationLocalBackend: |
| 273 | async def test_put_and_get_round_trip(self, tmp_path: Path) -> None: |
| 274 | b = _backend() |
| 275 | repo_root = _repo_root(tmp_path) |
| 276 | data = b"hello storage" |
| 277 | oid = long_id("a" * 64) |
| 278 | uri = await b.put(oid, data, repo_root=repo_root) |
| 279 | assert uri.startswith("local://") |
| 280 | result = await b.get(oid, repo_root=repo_root) |
| 281 | assert result == data |
| 282 | |
| 283 | async def test_get_missing_returns_none(self, tmp_path: Path) -> None: |
| 284 | b = _backend() |
| 285 | repo_root = _repo_root(tmp_path) |
| 286 | oid = long_id("b" * 64) |
| 287 | result = await b.get(oid, repo_root=repo_root) |
| 288 | assert result is None |
| 289 | |
| 290 | async def test_exists_after_put(self, tmp_path: Path) -> None: |
| 291 | b = _backend() |
| 292 | repo_root = _repo_root(tmp_path) |
| 293 | oid = long_id("c" * 64) |
| 294 | assert await b.exists(oid, repo_root=repo_root) is False |
| 295 | await b.put(oid, b"data", repo_root=repo_root) |
| 296 | assert await b.exists(oid, repo_root=repo_root) is True |
| 297 | |
| 298 | async def test_exists_missing_returns_false(self, tmp_path: Path) -> None: |
| 299 | b = _backend() |
| 300 | repo_root = _repo_root(tmp_path) |
| 301 | oid = long_id("d" * 64) |
| 302 | assert await b.exists(oid, repo_root=repo_root) is False |
| 303 | |
| 304 | async def test_delete_removes_object(self, tmp_path: Path) -> None: |
| 305 | b = _backend() |
| 306 | repo_root = _repo_root(tmp_path) |
| 307 | oid = long_id("e" * 64) |
| 308 | await b.put(oid, b"bye", repo_root=repo_root) |
| 309 | assert await b.exists(oid, repo_root=repo_root) is True |
| 310 | await b.delete(oid, repo_root=repo_root) |
| 311 | assert await b.exists(oid, repo_root=repo_root) is False |
| 312 | |
| 313 | async def test_delete_nonexistent_is_noop(self, tmp_path: Path) -> None: |
| 314 | b = _backend() |
| 315 | repo_root = _repo_root(tmp_path) |
| 316 | oid = long_id("f" * 64) |
| 317 | await b.delete(oid, repo_root=repo_root) |
| 318 | |
| 319 | async def test_put_idempotent_same_bytes(self, tmp_path: Path) -> None: |
| 320 | """Putting the same bytes twice is a no-op — content-addressed dedup.""" |
| 321 | b = _backend() |
| 322 | repo_root = _repo_root(tmp_path) |
| 323 | oid = long_id("0" * 64) |
| 324 | await b.put(oid, b"content", repo_root=repo_root) |
| 325 | await b.put(oid, b"content", repo_root=repo_root) |
| 326 | result = await b.get(oid, repo_root=repo_root) |
| 327 | assert result == b"content" |
| 328 | |
| 329 | async def test_put_creates_parent_dirs(self, tmp_path: Path) -> None: |
| 330 | b = _backend() |
| 331 | repo_root = _repo_root(tmp_path) |
| 332 | oid = long_id("1" * 64) |
| 333 | await b.put(oid, b"data", repo_root=repo_root) |
| 334 | assert await b.exists(oid, repo_root=repo_root) is True |
| 335 | |
| 336 | async def test_sha256_oid_round_trip(self, tmp_path: Path) -> None: |
| 337 | b = _backend() |
| 338 | repo_root = _repo_root(tmp_path) |
| 339 | oid = long_id("ab" + "c" * 62) |
| 340 | data = b"sha256 content" |
| 341 | await b.put(oid, data, repo_root=repo_root) |
| 342 | result = await b.get(oid, repo_root=repo_root) |
| 343 | assert result == data |
| 344 | |
| 345 | async def test_per_repo_isolation(self, tmp_path: Path) -> None: |
| 346 | """Objects in different repos are isolated — same oid, different repo_root.""" |
| 347 | b = _backend() |
| 348 | repo_a = tmp_path / "repos" / "alice" / "repo-a" |
| 349 | repo_b = tmp_path / "repos" / "bob" / "repo-b" |
| 350 | oid = long_id("2" * 64) |
| 351 | await b.put(oid, b"alice data", repo_root=repo_a) |
| 352 | assert await b.exists(oid, repo_root=repo_b) is False |
| 353 | assert await b.get(oid, repo_root=repo_a) == b"alice data" |
| 354 | |
| 355 | |
| 356 | class TestIntegrationS3BackendMocked: |
| 357 | """S3Backend with a mock boto3 client — no real AWS calls.""" |
| 358 | |
| 359 | def _mock_s3_backend(self, *, head_raises: bool = False) -> S3Backend: |
| 360 | b = S3Backend(bucket="test-bucket", region="us-east-1") |
| 361 | mock_client = MagicMock() |
| 362 | if head_raises: |
| 363 | mock_client.head_object.side_effect = Exception("NoSuchKey") |
| 364 | b._client = mock_client |
| 365 | return b |
| 366 | |
| 367 | async def test_put_calls_put_object_directly(self) -> None: |
| 368 | """put() is idempotent — no HeadObject check, just put_object unconditionally.""" |
| 369 | b = self._mock_s3_backend() |
| 370 | await b.put("obj1", b"data") |
| 371 | b._client.put_object.assert_called_once() |
| 372 | b._client.head_object.assert_not_called() |
| 373 | |
| 374 | async def test_put_is_idempotent(self) -> None: |
| 375 | """Uploading the same object twice both succeed — put_object called twice.""" |
| 376 | b = self._mock_s3_backend() |
| 377 | await b.put("obj1", b"data") |
| 378 | await b.put("obj1", b"data") |
| 379 | assert b._client.put_object.call_count == 2 |
| 380 | |
| 381 | async def test_get_returns_body_bytes(self) -> None: |
| 382 | b = self._mock_s3_backend() |
| 383 | mock_body = MagicMock() |
| 384 | mock_body.read.return_value = b"s3 content" |
| 385 | b._client.get_object.return_value = {"Body": mock_body} |
| 386 | result = await b.get("obj1") |
| 387 | assert result == b"s3 content" |
| 388 | |
| 389 | async def test_get_returns_none_on_error(self) -> None: |
| 390 | b = self._mock_s3_backend() |
| 391 | b._client.get_object.side_effect = Exception("NoSuchKey") |
| 392 | result = await b.get("obj1") |
| 393 | assert result is None |
| 394 | |
| 395 | async def test_exists_returns_true_when_head_succeeds(self) -> None: |
| 396 | b = self._mock_s3_backend() |
| 397 | b._client.head_object.return_value = {} |
| 398 | assert await b.exists("obj1") is True |
| 399 | |
| 400 | async def test_exists_returns_false_on_error(self) -> None: |
| 401 | b = self._mock_s3_backend(head_raises=True) |
| 402 | assert await b.exists("obj1") is False |
| 403 | |
| 404 | async def test_delete_calls_delete_object(self) -> None: |
| 405 | b = self._mock_s3_backend() |
| 406 | await b.delete("obj1") |
| 407 | b._client.delete_object.assert_called_once() |
| 408 | |
| 409 | async def test_get_batch_returns_all_found_objects(self) -> None: |
| 410 | """get_batch maps every object_id to its bytes when S3 finds all of them.""" |
| 411 | b = self._mock_s3_backend() |
| 412 | |
| 413 | def _get_object(Bucket: str, Key: str) -> _S3GetResponse: |
| 414 | mock_body = MagicMock() |
| 415 | mock_body.read.return_value = Key.encode() |
| 416 | return {"Body": mock_body} |
| 417 | |
| 418 | b._client.get_object.side_effect = _get_object |
| 419 | result = await b.get_batch(["oid-a", "oid-b", "oid-c"]) |
| 420 | assert set(result.keys()) == {"oid-a", "oid-b", "oid-c"} |
| 421 | |
| 422 | async def test_get_batch_omits_missing_objects(self) -> None: |
| 423 | """get_batch omits object_ids where S3 raises (NoSuchKey etc.).""" |
| 424 | b = self._mock_s3_backend() |
| 425 | b._client.get_object.side_effect = Exception("NoSuchKey") |
| 426 | result = await b.get_batch(["oid-1", "oid-2"]) |
| 427 | assert result == {} |
| 428 | |
| 429 | async def test_get_batch_is_parallel_not_sequential(self) -> None: |
| 430 | """N parallel S3 gets must complete in ~1× per-object latency, not N×.""" |
| 431 | import asyncio |
| 432 | |
| 433 | DELAY = 0.04 |
| 434 | N = 6 |
| 435 | |
| 436 | b = S3Backend(bucket="test-bucket", region="us-east-1") |
| 437 | |
| 438 | async def _slow_get(oid: str) -> bytes | None: |
| 439 | await asyncio.sleep(DELAY) |
| 440 | return oid.encode() |
| 441 | |
| 442 | setattr(b, "get", _slow_get) |
| 443 | |
| 444 | obj_ids = [f"oid-{i}" for i in range(N)] |
| 445 | start = time.perf_counter() |
| 446 | result = await b.get_batch(obj_ids) |
| 447 | elapsed = time.perf_counter() - start |
| 448 | |
| 449 | assert len(result) == N |
| 450 | assert elapsed < DELAY * 2.5, ( |
| 451 | f"get_batch took {elapsed * 1000:.0f} ms; expected ≈{DELAY * 1000:.0f} ms " |
| 452 | f"(parallel). Sequential would take ≥{DELAY * N * 1000:.0f} ms. " |
| 453 | "S3Backend.get_batch must use asyncio.gather, not the base-class loop." |
| 454 | ) |
| 455 | |
| 456 | |
| 457 | # ═══════════════════════════════════════════════════════════════════════════════ |
| 458 | # Layer 3 — End-to-End (LocalBackend as the full stack backend) |
| 459 | # ═══════════════════════════════════════════════════════════════════════════════ |
| 460 | |
| 461 | |
| 462 | class TestE2ELocalBackend: |
| 463 | async def test_full_lifecycle(self, tmp_path: Path) -> None: |
| 464 | """put → exists(True) → get → delete → exists(False) → get(None).""" |
| 465 | b = _backend() |
| 466 | repo_root = _repo_root(tmp_path) |
| 467 | obj_id = long_id("3" * 64) |
| 468 | data = b"end-to-end content" |
| 469 | |
| 470 | uri = await b.put(obj_id, data, repo_root=repo_root) |
| 471 | assert uri.startswith("local://") |
| 472 | |
| 473 | assert await b.exists(obj_id, repo_root=repo_root) is True |
| 474 | assert await b.get(obj_id, repo_root=repo_root) == data |
| 475 | |
| 476 | await b.delete(obj_id, repo_root=repo_root) |
| 477 | assert await b.exists(obj_id, repo_root=repo_root) is False |
| 478 | assert await b.get(obj_id, repo_root=repo_root) is None |
| 479 | |
| 480 | async def test_binary_data_preserved(self, tmp_path: Path) -> None: |
| 481 | b = _backend() |
| 482 | repo_root = _repo_root(tmp_path) |
| 483 | oid = long_id("4" * 64) |
| 484 | data = bytes(range(256)) |
| 485 | await b.put(oid, data, repo_root=repo_root) |
| 486 | result = await b.get(oid, repo_root=repo_root) |
| 487 | assert result == data |
| 488 | |
| 489 | async def test_large_object(self, tmp_path: Path) -> None: |
| 490 | b = _backend() |
| 491 | repo_root = _repo_root(tmp_path) |
| 492 | oid = long_id("5" * 64) |
| 493 | data = b"x" * (4 * 1024 * 1024) |
| 494 | await b.put(oid, data, repo_root=repo_root) |
| 495 | result = await b.get(oid, repo_root=repo_root) |
| 496 | assert result == data |
| 497 | |
| 498 | async def test_uri_resolves_to_disk_path(self, tmp_path: Path) -> None: |
| 499 | b = _backend() |
| 500 | repo_root = _repo_root(tmp_path) |
| 501 | oid = long_id("6" * 64) |
| 502 | data = b"uri content" |
| 503 | uri = await b.put(oid, data, repo_root=repo_root) |
| 504 | disk_path = Path(uri.replace("local://", "")) |
| 505 | assert disk_path.exists() |
| 506 | assert disk_path.read_bytes() == data |
| 507 | |
| 508 | |
| 509 | # ═══════════════════════════════════════════════════════════════════════════════ |
| 510 | # Layer 4 — Stress |
| 511 | # ═══════════════════════════════════════════════════════════════════════════════ |
| 512 | |
| 513 | |
| 514 | class TestStressLocalBackend: |
| 515 | async def test_write_many_objects(self, tmp_path: Path) -> None: |
| 516 | b = _backend() |
| 517 | repo_root = _repo_root(tmp_path) |
| 518 | obj_ids = [long_id(str(i).zfill(64)) for i in range(100)] |
| 519 | for oid in obj_ids: |
| 520 | await b.put(oid, oid.encode(), repo_root=repo_root) |
| 521 | for oid in obj_ids: |
| 522 | assert await b.exists(oid, repo_root=repo_root) is True |
| 523 | |
| 524 | async def test_concurrent_writes_different_objects(self, tmp_path: Path) -> None: |
| 525 | b = _backend() |
| 526 | repo_root = _repo_root(tmp_path) |
| 527 | obj_ids = [long_id(str(i).zfill(64)) for i in range(20)] |
| 528 | await asyncio.gather(*[b.put(oid, oid.encode(), repo_root=repo_root) for oid in obj_ids]) |
| 529 | for oid in obj_ids: |
| 530 | assert await b.get(oid, repo_root=repo_root) == oid.encode() |
| 531 | |
| 532 | async def test_concurrent_reads(self, tmp_path: Path) -> None: |
| 533 | b = _backend() |
| 534 | repo_root = _repo_root(tmp_path) |
| 535 | oid = long_id("7" * 64) |
| 536 | await b.put(oid, b"shared content", repo_root=repo_root) |
| 537 | results = await asyncio.gather(*[b.get(oid, repo_root=repo_root) for _ in range(20)]) |
| 538 | assert all(r == b"shared content" for r in results) |
| 539 | |
| 540 | |
| 541 | # ═══════════════════════════════════════════════════════════════════════════════ |
| 542 | # Layer 5 — Data Integrity |
| 543 | # ═══════════════════════════════════════════════════════════════════════════════ |
| 544 | |
| 545 | |
| 546 | class TestDataIntegrityLocalBackend: |
| 547 | async def test_put_returns_deterministic_uri(self, tmp_path: Path) -> None: |
| 548 | b = _backend() |
| 549 | repo_root = _repo_root(tmp_path) |
| 550 | oid = long_id("8" * 64) |
| 551 | uri1 = await b.put(oid, b"data", repo_root=repo_root) |
| 552 | uri2 = b.uri_for(oid, repo_root=repo_root) |
| 553 | assert uri1 == uri2 |
| 554 | |
| 555 | async def test_data_not_corrupted(self, tmp_path: Path) -> None: |
| 556 | b = _backend() |
| 557 | repo_root = _repo_root(tmp_path) |
| 558 | oid = long_id("9" * 64) |
| 559 | payload = b"\x00\x01\x02\x03" * 1000 |
| 560 | await b.put(oid, payload, repo_root=repo_root) |
| 561 | result = await b.get(oid, repo_root=repo_root) |
| 562 | assert result == payload |
| 563 | |
| 564 | async def test_put_is_idempotent_for_identical_bytes(self, tmp_path: Path) -> None: |
| 565 | b = _backend() |
| 566 | repo_root = _repo_root(tmp_path) |
| 567 | oid = long_id("a" * 64) |
| 568 | await b.put(oid, b"original", repo_root=repo_root) |
| 569 | await b.put(oid, b"original", repo_root=repo_root) |
| 570 | result = await b.get(oid, repo_root=repo_root) |
| 571 | assert result == b"original" |
| 572 | |
| 573 | async def test_put_repairs_corrupted_file(self, tmp_path: Path) -> None: |
| 574 | """put() must overwrite an existing file whose bytes differ from the new data.""" |
| 575 | import zlib |
| 576 | b = _backend() |
| 577 | repo_root = _repo_root(tmp_path) |
| 578 | raw = b"# real raw content" |
| 579 | corrupt = zlib.compress(raw) |
| 580 | oid = long_id("b" * 64) |
| 581 | |
| 582 | path = b._path(oid, repo_root=repo_root) |
| 583 | path.parent.mkdir(parents=True, exist_ok=True) |
| 584 | path.write_bytes(corrupt) |
| 585 | import stat |
| 586 | path.chmod(stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) |
| 587 | |
| 588 | await b.put(oid, raw, repo_root=repo_root) |
| 589 | |
| 590 | result = await b.get(oid, repo_root=repo_root) |
| 591 | assert result == raw, ( |
| 592 | f"expected raw content but got {result[:20]!r} — " |
| 593 | "LocalBackend.put() failed to repair the corrupted file" |
| 594 | ) |
| 595 | |
| 596 | async def test_delete_only_removes_target(self, tmp_path: Path) -> None: |
| 597 | b = _backend() |
| 598 | repo_root = _repo_root(tmp_path) |
| 599 | oid_keep = long_id("c" * 64) |
| 600 | oid_remove = long_id("d" * 64) |
| 601 | await b.put(oid_keep, b"keep", repo_root=repo_root) |
| 602 | await b.put(oid_remove, b"remove", repo_root=repo_root) |
| 603 | await b.delete(oid_remove, repo_root=repo_root) |
| 604 | assert await b.get(oid_keep, repo_root=repo_root) == b"keep" |
| 605 | assert await b.get(oid_remove, repo_root=repo_root) is None |
| 606 | |
| 607 | async def test_empty_bytes_stored_and_retrieved(self, tmp_path: Path) -> None: |
| 608 | b = _backend() |
| 609 | repo_root = _repo_root(tmp_path) |
| 610 | oid = long_id("e" * 64) |
| 611 | await b.put(oid, b"", repo_root=repo_root) |
| 612 | result = await b.get(oid, repo_root=repo_root) |
| 613 | assert result == b"" |
| 614 | |
| 615 | |
| 616 | # ═══════════════════════════════════════════════════════════════════════════════ |
| 617 | # Layer 6 — Security |
| 618 | # ═══════════════════════════════════════════════════════════════════════════════ |
| 619 | |
| 620 | |
| 621 | class TestSecurityLocalBackend: |
| 622 | def test_s3_key_preserves_colons(self) -> None: |
| 623 | b = S3Backend(bucket="b", region="us-east-1") |
| 624 | key = b._key("sha256:evil:colons") |
| 625 | assert "sha256:evil:colons" in key |
| 626 | |
| 627 | async def test_put_without_repo_root_raises(self, tmp_path: Path) -> None: |
| 628 | b = _backend() |
| 629 | oid = long_id("f" * 64) |
| 630 | with pytest.raises(ValueError, match="repo_root"): |
| 631 | await b.put(oid, b"data", repo_root=None) |
| 632 | |
| 633 | async def test_get_without_repo_root_raises(self, tmp_path: Path) -> None: |
| 634 | b = _backend() |
| 635 | oid = long_id("0" * 64) |
| 636 | with pytest.raises(ValueError, match="repo_root"): |
| 637 | await b.get(oid, repo_root=None) |
| 638 | |
| 639 | async def test_exists_without_repo_root_raises(self, tmp_path: Path) -> None: |
| 640 | b = _backend() |
| 641 | oid = long_id("1" * 64) |
| 642 | with pytest.raises(ValueError, match="repo_root"): |
| 643 | await b.exists(oid, repo_root=None) |
| 644 | |
| 645 | |
| 646 | # ═══════════════════════════════════════════════════════════════════════════════ |
| 647 | # Layer 7 — Performance |
| 648 | # ═══════════════════════════════════════════════════════════════════════════════ |
| 649 | |
| 650 | |
| 651 | class TestPerformanceLocalBackend: |
| 652 | async def test_put_latency(self, tmp_path: Path) -> None: |
| 653 | b = _backend() |
| 654 | repo_root = _repo_root(tmp_path) |
| 655 | oid = long_id("2" * 64) |
| 656 | data = b"perf-test-payload" * 100 |
| 657 | start = time.perf_counter() |
| 658 | await b.put(oid, data, repo_root=repo_root) |
| 659 | elapsed = time.perf_counter() - start |
| 660 | assert elapsed < 0.5 |
| 661 | |
| 662 | async def test_get_latency(self, tmp_path: Path) -> None: |
| 663 | b = _backend() |
| 664 | repo_root = _repo_root(tmp_path) |
| 665 | oid = long_id("3" * 64) |
| 666 | data = b"x" * (512 * 1024) |
| 667 | await b.put(oid, data, repo_root=repo_root) |
| 668 | |
| 669 | start = time.perf_counter() |
| 670 | result = await b.get(oid, repo_root=repo_root) |
| 671 | elapsed = time.perf_counter() - start |
| 672 | |
| 673 | assert result == data |
| 674 | assert elapsed < 0.5 |
| 675 | |
| 676 | async def test_exists_latency(self, tmp_path: Path) -> None: |
| 677 | b = _backend() |
| 678 | repo_root = _repo_root(tmp_path) |
| 679 | oid = long_id("4" * 64) |
| 680 | await b.put(oid, b"data", repo_root=repo_root) |
| 681 | |
| 682 | start = time.perf_counter() |
| 683 | for _ in range(50): |
| 684 | await b.exists(oid, repo_root=repo_root) |
| 685 | elapsed = time.perf_counter() - start |
| 686 | |
| 687 | assert elapsed < 1.0 |
| 688 | |
| 689 | async def test_50_sequential_puts_under_budget(self, tmp_path: Path) -> None: |
| 690 | b = _backend() |
| 691 | repo_root = _repo_root(tmp_path) |
| 692 | data = b"payload" * 100 |
| 693 | |
| 694 | start = time.perf_counter() |
| 695 | for i in range(50): |
| 696 | oid = long_id(str(i).zfill(64)) |
| 697 | await b.put(oid, data, repo_root=repo_root) |
| 698 | elapsed = time.perf_counter() - start |
| 699 | |
| 700 | assert elapsed < 2.0 |
| 701 | |
| 702 | |
| 703 | # ═══════════════════════════════════════════════════════════════════════════════ |
| 704 | # Layer 8 — Atomic Write Safety (Phase 8 crash-safety fix) |
| 705 | # ═══════════════════════════════════════════════════════════════════════════════ |
| 706 | |
| 707 | |
| 708 | class TestAtomicWrite: |
| 709 | """_write() uses mkstemp → fsync → os.replace — no partial files visible to readers.""" |
| 710 | |
| 711 | async def test_write_leaves_complete_file(self, tmp_path: Path) -> None: |
| 712 | b = _backend() |
| 713 | repo_root = _repo_root(tmp_path) |
| 714 | oid = long_id("5" * 64) |
| 715 | data = b"atomic content" |
| 716 | await b.put(oid, data, repo_root=repo_root) |
| 717 | path = b._path(oid, repo_root=repo_root) |
| 718 | assert path.exists() |
| 719 | assert path.read_bytes() == data |
| 720 | |
| 721 | async def test_write_no_tmp_file_remains_after_success(self, tmp_path: Path) -> None: |
| 722 | b = _backend() |
| 723 | repo_root = _repo_root(tmp_path) |
| 724 | oid = long_id("6" * 64) |
| 725 | await b.put(oid, b"data", repo_root=repo_root) |
| 726 | objects_dir = repo_root / "objects" |
| 727 | tmp_files = list(objects_dir.rglob("tmp*")) |
| 728 | assert tmp_files == [], f"stray temp files: {tmp_files}" |
| 729 | |
| 730 | async def test_write_file_is_immutable_after_put(self, tmp_path: Path) -> None: |
| 731 | import stat as _stat |
| 732 | b = _backend() |
| 733 | repo_root = _repo_root(tmp_path) |
| 734 | oid = long_id("7" * 64) |
| 735 | await b.put(oid, b"readonly data", repo_root=repo_root) |
| 736 | path = b._path(oid, repo_root=repo_root) |
| 737 | mode = _stat.S_IMODE(path.stat().st_mode) |
| 738 | assert mode == 0o444, f"expected 0o444 but got {oct(mode)}" |
| 739 | |
| 740 | async def test_write_tmp_file_cleaned_up_on_fsync_failure(self, tmp_path: Path) -> None: |
| 741 | import os as _os |
| 742 | b = _backend() |
| 743 | repo_root = _repo_root(tmp_path) |
| 744 | oid = long_id("8" * 64) |
| 745 | |
| 746 | def _failing_fsync(fd: int) -> None: |
| 747 | raise OSError("simulated fsync failure") |
| 748 | |
| 749 | import unittest.mock as _mock |
| 750 | objects_dir = repo_root / "objects" |
| 751 | objects_dir.mkdir(parents=True, exist_ok=True) |
| 752 | |
| 753 | with _mock.patch("os.fsync", side_effect=_failing_fsync): |
| 754 | with pytest.raises(OSError, match="simulated fsync failure"): |
| 755 | await b.put(oid, b"never written", repo_root=repo_root) |
| 756 | |
| 757 | tmp_files = list(objects_dir.rglob("tmp*")) |
| 758 | assert tmp_files == [], f"orphan temp files after fsync failure: {tmp_files}" |
| 759 | |
| 760 | async def test_write_idempotent_identical_bytes_skips_io(self, tmp_path: Path) -> None: |
| 761 | import time as _time |
| 762 | b = _backend() |
| 763 | repo_root = _repo_root(tmp_path) |
| 764 | oid = long_id("9" * 64) |
| 765 | data = b"idempotent" |
| 766 | await b.put(oid, data, repo_root=repo_root) |
| 767 | path = b._path(oid, repo_root=repo_root) |
| 768 | mtime_before = path.stat().st_mtime_ns |
| 769 | _time.sleep(0.01) |
| 770 | await b.put(oid, data, repo_root=repo_root) |
| 771 | mtime_after = path.stat().st_mtime_ns |
| 772 | assert mtime_before == mtime_after, "second put with identical bytes must not touch the file" |
| 773 | |
| 774 | async def test_write_repairs_corrupted_file_atomically(self, tmp_path: Path) -> None: |
| 775 | import stat as _stat |
| 776 | b = _backend() |
| 777 | repo_root = _repo_root(tmp_path) |
| 778 | raw = b"correct content" |
| 779 | corrupt = b"wrong zlib garbage" |
| 780 | oid = long_id("aa" + "b" * 62) |
| 781 | |
| 782 | path = b._path(oid, repo_root=repo_root) |
| 783 | path.parent.mkdir(parents=True, exist_ok=True) |
| 784 | path.write_bytes(corrupt) |
| 785 | path.chmod(_stat.S_IRUSR | _stat.S_IRGRP | _stat.S_IROTH) |
| 786 | |
| 787 | await b.put(oid, raw, repo_root=repo_root) |
| 788 | assert await b.get(oid, repo_root=repo_root) == raw |
| 789 | |
| 790 | async def test_write_concurrent_same_object_safe(self, tmp_path: Path) -> None: |
| 791 | import asyncio as _asyncio |
| 792 | b = _backend() |
| 793 | repo_root = _repo_root(tmp_path) |
| 794 | oid = long_id("bb" + "c" * 62) |
| 795 | data = b"concurrent data" |
| 796 | results = await _asyncio.gather(*[b.put(oid, data, repo_root=repo_root) for _ in range(10)]) |
| 797 | assert all(r.startswith("local://") for r in results) |
| 798 | assert await b.get(oid, repo_root=repo_root) == data |
| 799 | |
| 800 | |
| 801 | # ═══════════════════════════════════════════════════════════════════════════════ |
| 802 | # read_object_bytes — unified adapter |
| 803 | # ═══════════════════════════════════════════════════════════════════════════════ |
| 804 | |
| 805 | |
| 806 | class TestUnitReadObjectBytes: |
| 807 | """read_object_bytes(obj) must handle every storage case without callers |
| 808 | knowing which backend is active.""" |
| 809 | |
| 810 | def _obj(self, **kwargs): |
| 811 | """Build a minimal MusehubObject-like namespace.""" |
| 812 | defaults = dict( |
| 813 | object_id=long_id("aa" * 32), |
| 814 | content_cache=None, |
| 815 | disk_path="", |
| 816 | storage_uri="", |
| 817 | ) |
| 818 | defaults.update(kwargs) |
| 819 | return SimpleNamespace(**defaults) |
| 820 | |
| 821 | async def test_returns_content_cache_when_present(self) -> None: |
| 822 | from musehub.storage.backends import read_object_bytes |
| 823 | obj = self._obj(content_cache=b"cached") |
| 824 | assert await read_object_bytes(obj) == b"cached" |
| 825 | |
| 826 | async def test_content_cache_takes_priority_over_disk(self, tmp_path: Path) -> None: |
| 827 | from musehub.storage.backends import read_object_bytes |
| 828 | disk_file = tmp_path / "obj" |
| 829 | disk_file.write_bytes(b"disk content") |
| 830 | obj = self._obj(content_cache=b"cached", disk_path=str(disk_file)) |
| 831 | assert await read_object_bytes(obj) == b"cached" |
| 832 | |
| 833 | async def test_reads_local_disk_path(self, tmp_path: Path) -> None: |
| 834 | from musehub.storage.backends import read_object_bytes |
| 835 | disk_file = tmp_path / "obj" |
| 836 | disk_file.write_bytes(b"from disk") |
| 837 | obj = self._obj(disk_path=str(disk_file)) |
| 838 | assert await read_object_bytes(obj) == b"from disk" |
| 839 | |
| 840 | async def test_reads_local_uri_prefix(self, tmp_path: Path) -> None: |
| 841 | from musehub.storage.backends import read_object_bytes |
| 842 | disk_file = tmp_path / "obj" |
| 843 | disk_file.write_bytes(b"from local uri") |
| 844 | obj = self._obj(disk_path=f"local://{disk_file}") |
| 845 | assert await read_object_bytes(obj) == b"from local uri" |
| 846 | |
| 847 | async def test_reads_from_s3_when_s3_uri(self) -> None: |
| 848 | from musehub.storage.backends import read_object_bytes |
| 849 | obj = self._obj(disk_path="s3://my-bucket/objects/sha256_abc123") |
| 850 | mock_backend = MagicMock() |
| 851 | |
| 852 | async def _fake_get(oid, **kw): |
| 853 | return b"s3 bytes" |
| 854 | |
| 855 | mock_backend.get = _fake_get |
| 856 | with patch("musehub.storage.backends.get_backend", return_value=mock_backend): |
| 857 | result = await read_object_bytes(obj) |
| 858 | assert result == b"s3 bytes" |
| 859 | |
| 860 | async def test_missing_disk_file_returns_none(self, tmp_path: Path) -> None: |
| 861 | from musehub.storage.backends import read_object_bytes |
| 862 | obj = self._obj(disk_path=str(tmp_path / "does_not_exist")) |
| 863 | assert await read_object_bytes(obj) is None |
| 864 | |
| 865 | async def test_no_path_no_cache_returns_none(self) -> None: |
| 866 | from musehub.storage.backends import read_object_bytes |
| 867 | obj = self._obj() |
| 868 | assert await read_object_bytes(obj) is None |
File History
1 commit
sha256:763eb2cb8675073b84c19345b27586d2ed939a9aee97c5479b69f502f1a70eff
fix(tests): update test suite to match current implementation
Sonnet 4.6
patch
119 days ago