test_object_store_section52.py
python
sha256:a10adeeb7a0169cb9900f9806ed7a973047258abb6283724fe55e8eb68ff3f0a
init: musehub initial commit
Human
171 days ago
| 1 | """Tests for checklist section 5.2 — Object store. |
| 2 | |
| 3 | Covers: |
| 4 | - Object files are immutable after write (LocalBackend chmod 0o444) |
| 5 | - Integrity scan detects missing objects |
| 6 | - Integrity scan detects hash-mismatched objects |
| 7 | - Integrity scan passes on clean objects |
| 8 | - Per-repo storage quota enforced in wire push |
| 9 | - Soft-delete sets deleted_at without removing the file |
| 10 | - Hard-delete reaper removes only objects past retention window |
| 11 | """ |
| 12 | from __future__ import annotations |
| 13 | |
| 14 | import hashlib |
| 15 | import stat |
| 16 | import tempfile |
| 17 | from pathlib import Path |
| 18 | from unittest.mock import AsyncMock, MagicMock, patch |
| 19 | |
| 20 | import pytest |
| 21 | from sqlalchemy.ext.asyncio import AsyncSession |
| 22 | |
| 23 | from musehub.db.musehub_models import MusehubObject as _MusehubObject |
| 24 | from tests.factories import create_repo |
| 25 | |
| 26 | |
| 27 | # --------------------------------------------------------------------------- |
| 28 | # Helpers |
| 29 | # --------------------------------------------------------------------------- |
| 30 | |
| 31 | def _sha256_id(data: bytes) -> str: |
| 32 | return "sha256:" + hashlib.sha256(data).hexdigest() |
| 33 | |
| 34 | |
| 35 | async def _add_object( |
| 36 | session: AsyncSession, |
| 37 | repo_id: str, |
| 38 | data: bytes, |
| 39 | *, |
| 40 | object_id: str | None = None, |
| 41 | ) -> _MusehubObject: |
| 42 | from musehub.db import musehub_models as db |
| 43 | |
| 44 | oid = object_id or _sha256_id(data) |
| 45 | obj = db.MusehubObject( |
| 46 | object_id=oid, |
| 47 | repo_id=repo_id, |
| 48 | path="test.bin", |
| 49 | size_bytes=len(data), |
| 50 | disk_path=f"/tmp/{oid.replace(':', '_')}", |
| 51 | storage_uri=f"local:///tmp/{oid.replace(':', '_')}", |
| 52 | ) |
| 53 | session.add(obj) |
| 54 | await session.flush() |
| 55 | return obj |
| 56 | |
| 57 | |
| 58 | # --------------------------------------------------------------------------- |
| 59 | # Immutability — LocalBackend sets file to read-only after write |
| 60 | # --------------------------------------------------------------------------- |
| 61 | |
| 62 | def test_local_backend_write_sets_readonly() -> None: |
| 63 | """LocalBackend._write must chmod the file to 0o444 after first write.""" |
| 64 | from musehub.storage.backends import LocalBackend |
| 65 | |
| 66 | with tempfile.TemporaryDirectory() as tmpdir: |
| 67 | backend = LocalBackend(objects_dir=tmpdir) |
| 68 | path = Path(tmpdir) / "repo1" / "sha256_abc" |
| 69 | path.parent.mkdir(parents=True, exist_ok=True) |
| 70 | backend._write(path, b"hello") |
| 71 | |
| 72 | mode = path.stat().st_mode |
| 73 | # No write bits for owner, group, or other. |
| 74 | assert not (mode & stat.S_IWUSR), "owner write bit must be cleared" |
| 75 | assert not (mode & stat.S_IWGRP), "group write bit must be cleared" |
| 76 | assert not (mode & stat.S_IWOTH), "other write bit must be cleared" |
| 77 | # Read bits must be set. |
| 78 | assert mode & stat.S_IRUSR |
| 79 | |
| 80 | |
| 81 | def test_local_backend_write_skips_existing_file() -> None: |
| 82 | """LocalBackend._write must not overwrite an existing file.""" |
| 83 | from musehub.storage.backends import LocalBackend |
| 84 | |
| 85 | with tempfile.TemporaryDirectory() as tmpdir: |
| 86 | backend = LocalBackend(objects_dir=tmpdir) |
| 87 | path = Path(tmpdir) / "repo1" / "sha256_abc" |
| 88 | path.parent.mkdir(parents=True, exist_ok=True) |
| 89 | path.write_bytes(b"original") |
| 90 | # Re-chmod to writable so the skip-check can be tested without OS errors. |
| 91 | path.chmod(0o644) |
| 92 | |
| 93 | backend._write(path, b"overwrite-attempt") |
| 94 | assert path.read_bytes() == b"original", "existing file must not be overwritten" |
| 95 | |
| 96 | |
| 97 | # --------------------------------------------------------------------------- |
| 98 | # Integrity scan — clean |
| 99 | # --------------------------------------------------------------------------- |
| 100 | |
| 101 | @pytest.mark.anyio |
| 102 | async def test_integrity_scan_clean(db_session: AsyncSession) -> None: |
| 103 | """scan_object_integrity must return ok=True when all sampled objects are valid.""" |
| 104 | from musehub.maintenance.object_integrity import scan_object_integrity |
| 105 | |
| 106 | repo = await create_repo(db_session, slug="integrity-clean", owner="testuser") |
| 107 | data = b"test content integrity" |
| 108 | oid = _sha256_id(data) |
| 109 | await _add_object(db_session, repo.repo_id, data, object_id=oid) |
| 110 | await db_session.commit() |
| 111 | |
| 112 | # Backend that returns the correct content. |
| 113 | backend = AsyncMock() |
| 114 | backend.get = AsyncMock(return_value=data) |
| 115 | |
| 116 | result = await scan_object_integrity(db_session, backend, sample_size=10) |
| 117 | assert result.ok |
| 118 | assert result.mismatch_count == 0 |
| 119 | assert result.sampled >= 1 |
| 120 | |
| 121 | |
| 122 | # --------------------------------------------------------------------------- |
| 123 | # Integrity scan — missing object |
| 124 | # --------------------------------------------------------------------------- |
| 125 | |
| 126 | @pytest.mark.anyio |
| 127 | async def test_integrity_scan_detects_missing_object(db_session: AsyncSession) -> None: |
| 128 | """scan_object_integrity must flag objects whose backing file is absent.""" |
| 129 | from musehub.maintenance.object_integrity import scan_object_integrity |
| 130 | |
| 131 | repo = await create_repo(db_session, slug="integrity-missing", owner="testuser") |
| 132 | data = b"missing object data" |
| 133 | oid = _sha256_id(data) |
| 134 | await _add_object(db_session, repo.repo_id, data, object_id=oid) |
| 135 | await db_session.commit() |
| 136 | |
| 137 | backend = AsyncMock() |
| 138 | backend.get = AsyncMock(return_value=None) # simulates missing file |
| 139 | |
| 140 | result = await scan_object_integrity(db_session, backend, sample_size=10) |
| 141 | assert not result.ok |
| 142 | mismatch_ids = [m.object_id for m in result.mismatches] |
| 143 | assert oid in mismatch_ids |
| 144 | reasons = {m.reason for m in result.mismatches} |
| 145 | assert "missing" in reasons |
| 146 | |
| 147 | |
| 148 | # --------------------------------------------------------------------------- |
| 149 | # Integrity scan — hash mismatch |
| 150 | # --------------------------------------------------------------------------- |
| 151 | |
| 152 | @pytest.mark.anyio |
| 153 | async def test_integrity_scan_detects_hash_mismatch(db_session: AsyncSession) -> None: |
| 154 | """scan_object_integrity must flag objects whose content no longer matches their ID.""" |
| 155 | from musehub.maintenance.object_integrity import scan_object_integrity |
| 156 | |
| 157 | repo = await create_repo(db_session, slug="integrity-mismatch", owner="testuser") |
| 158 | data = b"original content" |
| 159 | oid = _sha256_id(data) |
| 160 | await _add_object(db_session, repo.repo_id, data, object_id=oid) |
| 161 | await db_session.commit() |
| 162 | |
| 163 | corrupted = b"corrupted content XYZ" |
| 164 | backend = AsyncMock() |
| 165 | backend.get = AsyncMock(return_value=corrupted) |
| 166 | |
| 167 | result = await scan_object_integrity(db_session, backend, sample_size=10) |
| 168 | assert not result.ok |
| 169 | mismatch_ids = [m.object_id for m in result.mismatches] |
| 170 | assert oid in mismatch_ids |
| 171 | reasons = {m.reason for m in result.mismatches} |
| 172 | assert "hash_mismatch" in reasons |
| 173 | |
| 174 | |
| 175 | # --------------------------------------------------------------------------- |
| 176 | # Soft-delete |
| 177 | # --------------------------------------------------------------------------- |
| 178 | |
| 179 | @pytest.mark.anyio |
| 180 | async def test_soft_delete_sets_deleted_at(db_session: AsyncSession) -> None: |
| 181 | """soft_delete_object must set deleted_at without removing the DB row.""" |
| 182 | from musehub.maintenance.object_integrity import soft_delete_object |
| 183 | from musehub.db import musehub_models as db_models |
| 184 | |
| 185 | repo = await create_repo(db_session, slug="soft-delete-test", owner="testuser") |
| 186 | data = b"soft delete me" |
| 187 | oid = _sha256_id(data) |
| 188 | await _add_object(db_session, repo.repo_id, data, object_id=oid) |
| 189 | await db_session.commit() |
| 190 | |
| 191 | found = await soft_delete_object(db_session, oid) |
| 192 | assert found is True |
| 193 | |
| 194 | row = await db_session.get(db_models.MusehubObject, oid) |
| 195 | assert row is not None, "row must still exist after soft delete" |
| 196 | assert row.deleted_at is not None, "deleted_at must be set" |
| 197 | |
| 198 | |
| 199 | @pytest.mark.anyio |
| 200 | async def test_soft_delete_unknown_object_returns_false(db_session: AsyncSession) -> None: |
| 201 | from musehub.maintenance.object_integrity import soft_delete_object |
| 202 | |
| 203 | found = await soft_delete_object(db_session, "sha256:" + "f" * 64) |
| 204 | assert found is False |
| 205 | |
| 206 | |
| 207 | # --------------------------------------------------------------------------- |
| 208 | # Hard-delete reaper |
| 209 | # --------------------------------------------------------------------------- |
| 210 | |
| 211 | @pytest.mark.anyio |
| 212 | async def test_reaper_removes_objects_past_retention(db_session: AsyncSession) -> None: |
| 213 | """reap_deleted_objects must hard-delete objects whose deleted_at exceeds retention.""" |
| 214 | from datetime import datetime, timedelta, timezone |
| 215 | from musehub.maintenance.object_integrity import reap_deleted_objects, soft_delete_object |
| 216 | from musehub.db import musehub_models as db_models |
| 217 | |
| 218 | repo = await create_repo(db_session, slug="reap-old", owner="testuser") |
| 219 | data = b"old deleted object" |
| 220 | oid = _sha256_id(data) |
| 221 | await _add_object(db_session, repo.repo_id, data, object_id=oid) |
| 222 | await db_session.commit() |
| 223 | |
| 224 | # Soft-delete the object, then backdate deleted_at past the retention window. |
| 225 | await soft_delete_object(db_session, oid) |
| 226 | await db_session.flush() |
| 227 | row = await db_session.get(db_models.MusehubObject, oid) |
| 228 | assert row is not None |
| 229 | row.deleted_at = datetime.now(tz=timezone.utc) - timedelta(days=31) |
| 230 | await db_session.commit() |
| 231 | |
| 232 | backend = AsyncMock() |
| 233 | backend.delete = AsyncMock() |
| 234 | |
| 235 | reaped = await reap_deleted_objects(db_session, backend, retention_days=30) |
| 236 | assert reaped == 1 |
| 237 | backend.delete.assert_called_once() |
| 238 | |
| 239 | # Row must be gone from DB. |
| 240 | gone = await db_session.get(db_models.MusehubObject, oid) |
| 241 | assert gone is None, "hard-deleted object must be removed from DB" |
| 242 | |
| 243 | |
| 244 | @pytest.mark.anyio |
| 245 | async def test_reaper_spares_objects_within_retention(db_session: AsyncSession) -> None: |
| 246 | """reap_deleted_objects must NOT remove objects soft-deleted within retention window.""" |
| 247 | from datetime import datetime, timedelta, timezone |
| 248 | from musehub.maintenance.object_integrity import reap_deleted_objects, soft_delete_object |
| 249 | from musehub.db import musehub_models as db_models |
| 250 | |
| 251 | repo = await create_repo(db_session, slug="reap-new", owner="testuser") |
| 252 | data = b"recently deleted object" |
| 253 | oid = _sha256_id(data) |
| 254 | await _add_object(db_session, repo.repo_id, data, object_id=oid) |
| 255 | await db_session.commit() |
| 256 | |
| 257 | await soft_delete_object(db_session, oid) |
| 258 | await db_session.commit() |
| 259 | |
| 260 | # deleted_at is NOW — well within the 30-day window. |
| 261 | backend = AsyncMock() |
| 262 | backend.delete = AsyncMock() |
| 263 | |
| 264 | reaped = await reap_deleted_objects(db_session, backend, retention_days=30) |
| 265 | assert reaped == 0 |
| 266 | backend.delete.assert_not_called() |
| 267 | |
| 268 | still_there = await db_session.get(db_models.MusehubObject, oid) |
| 269 | assert still_there is not None |
| 270 | |
| 271 | |
| 272 | # --------------------------------------------------------------------------- |
| 273 | # Per-repo quota — wire push |
| 274 | # --------------------------------------------------------------------------- |
| 275 | |
| 276 | @pytest.mark.anyio |
| 277 | async def test_wire_push_rejects_when_repo_quota_exceeded(db_session: AsyncSession) -> None: |
| 278 | """wire_push must return ok=False when the push would exceed per_repo_quota_bytes.""" |
| 279 | from musehub.services.musehub_wire import wire_push |
| 280 | from musehub.models.wire import WirePushRequest, WireBundle, WireObject |
| 281 | |
| 282 | repo = await create_repo(db_session, slug="quota-wire", owner="testuser") |
| 283 | await db_session.commit() |
| 284 | |
| 285 | big_content = b"x" * 100 # 100 bytes per object |
| 286 | |
| 287 | bundle = WireBundle( |
| 288 | objects=[ |
| 289 | WireObject( |
| 290 | object_id=_sha256_id(big_content), |
| 291 | path="big.bin", |
| 292 | content=big_content, |
| 293 | ) |
| 294 | ], |
| 295 | commits=[], |
| 296 | snapshots=[], |
| 297 | ) |
| 298 | req = WirePushRequest(repo_id=repo.repo_id, branch="main", bundle=bundle) |
| 299 | |
| 300 | # Patch quota to 50 bytes — incoming 100 bytes will exceed it. |
| 301 | with patch("musehub.services.musehub_wire.settings") as mock_settings: |
| 302 | mock_settings.per_repo_quota_bytes = 50 |
| 303 | result = await wire_push(db_session, repo.repo_id, req, pusher_id="testuser") |
| 304 | |
| 305 | assert result.ok is False |
| 306 | assert "quota" in result.message.lower() |
File History
1 commit
sha256:a10adeeb7a0169cb9900f9806ed7a973047258abb6283724fe55e8eb68ff3f0a
init: musehub initial commit
Human
171 days ago