gabriel / musehub public
test_object_store.py python
313 lines 12.0 KB
Raw
sha256:9590cee1e0ccd6c76528f005b95d634d80f5019f0dcb7c371e149adc31d1fb65 refactor: enforce gRPC framing on all MWP wire traffic Sonnet 4.6 minor ⚠ breaking 156 days ago
1 """Tests for checklist section 5.2 — Object store.
2
3 Covers:
4 - Object files are immutable after write (LocalBackend chmod 0o444)
5 - Integrity scan detects missing objects
6 - Integrity scan detects hash-mismatched objects
7 - Integrity scan passes on clean objects
8 - Per-repo storage quota enforced in wire push
9 - Soft-delete sets deleted_at without removing the file
10 - Hard-delete reaper removes only objects past retention window
11 """
12 from __future__ import annotations
13
14 import hashlib
15 import stat
16 import tempfile
17 from pathlib import Path
18 from unittest.mock import AsyncMock, MagicMock, patch
19
20 import pytest
21 from sqlalchemy.ext.asyncio import AsyncSession
22
23 from musehub.db.musehub_models import MusehubObject as _MusehubObject
24 from tests.factories import create_repo
25
26
27 # ---------------------------------------------------------------------------
28 # Helpers
29 # ---------------------------------------------------------------------------
30
31 def _sha256_id(data: bytes) -> str:
32 return "sha256:" + hashlib.sha256(data).hexdigest()
33
34
35 async def _add_object(
36 session: AsyncSession,
37 repo_id: str,
38 data: bytes,
39 *,
40 object_id: str | None = None,
41 ) -> _MusehubObject:
42 from musehub.db import musehub_models as db
43
44 oid = object_id or _sha256_id(data)
45 obj = db.MusehubObject(
46 object_id=oid,
47 path="test.bin",
48 size_bytes=len(data),
49 disk_path=f"/tmp/{oid.replace(':', '_')}",
50 storage_uri=f"local:///tmp/{oid.replace(':', '_')}",
51 )
52 session.add(obj)
53 session.add(db.MusehubObjectRef(repo_id=repo_id, object_id=oid))
54 await session.flush()
55 return obj
56
57
58 # ---------------------------------------------------------------------------
59 # Immutability — LocalBackend sets file to read-only after write
60 # ---------------------------------------------------------------------------
61
62 def test_local_backend_write_sets_readonly() -> None:
63 """LocalBackend._write must chmod the file to 0o444 after first write."""
64 from musehub.storage.backends import LocalBackend
65
66 with tempfile.TemporaryDirectory() as tmpdir:
67 backend = LocalBackend(objects_dir=tmpdir)
68 path = Path(tmpdir) / "repo1" / "sha256_abc"
69 path.parent.mkdir(parents=True, exist_ok=True)
70 backend._write(path, b"hello")
71
72 mode = path.stat().st_mode
73 # No write bits for owner, group, or other.
74 assert not (mode & stat.S_IWUSR), "owner write bit must be cleared"
75 assert not (mode & stat.S_IWGRP), "group write bit must be cleared"
76 assert not (mode & stat.S_IWOTH), "other write bit must be cleared"
77 # Read bits must be set.
78 assert mode & stat.S_IRUSR
79
80
81 def test_local_backend_write_idempotent_for_same_content() -> None:
82 """LocalBackend._write is a no-op when the file already contains identical bytes."""
83 from musehub.storage.backends import LocalBackend
84
85 with tempfile.TemporaryDirectory() as tmpdir:
86 backend = LocalBackend(objects_dir=tmpdir)
87 path = Path(tmpdir) / "repo1" / "sha256_abc"
88 path.parent.mkdir(parents=True, exist_ok=True)
89 backend._write(path, b"original")
90 mtime_before = path.stat().st_mtime_ns
91
92 # Writing the exact same bytes must not change the file.
93 backend._write(path, b"original")
94 assert path.read_bytes() == b"original", "content must be unchanged"
95
96
97 def test_local_backend_write_repairs_corrupt_file() -> None:
98 """LocalBackend._write overwrites when the stored bytes differ (self-heal / repair)."""
99 from musehub.storage.backends import LocalBackend
100
101 with tempfile.TemporaryDirectory() as tmpdir:
102 backend = LocalBackend(objects_dir=tmpdir)
103 path = Path(tmpdir) / "repo1" / "sha256_abc"
104 path.parent.mkdir(parents=True, exist_ok=True)
105 path.write_bytes(b"corrupt")
106 path.chmod(0o644)
107
108 backend._write(path, b"correct")
109 assert path.read_bytes() == b"correct", "corrupt file must be replaced"
110
111
112 # ---------------------------------------------------------------------------
113 # Integrity scan — clean
114 # ---------------------------------------------------------------------------
115
116 async def test_integrity_scan_clean(db_session: AsyncSession) -> None:
117 """scan_object_integrity must return ok=True when all sampled objects are valid."""
118 from musehub.maintenance.object_integrity import scan_object_integrity
119
120 repo = await create_repo(db_session, slug="integrity-clean", owner="testuser")
121 data = b"test content integrity"
122 oid = _sha256_id(data)
123 await _add_object(db_session, repo.repo_id, data, object_id=oid)
124 await db_session.commit()
125
126 # Backend that returns the correct content.
127 backend = AsyncMock()
128 backend.get = AsyncMock(return_value=data)
129
130 result = await scan_object_integrity(db_session, backend, sample_size=10)
131 assert result.ok
132 assert result.mismatch_count == 0
133 assert result.sampled >= 1
134
135
136 # ---------------------------------------------------------------------------
137 # Integrity scan — missing object
138 # ---------------------------------------------------------------------------
139
140 async def test_integrity_scan_detects_missing_object(db_session: AsyncSession) -> None:
141 """scan_object_integrity must flag objects whose backing file is absent."""
142 from musehub.maintenance.object_integrity import scan_object_integrity
143
144 repo = await create_repo(db_session, slug="integrity-missing", owner="testuser")
145 data = b"missing object data"
146 oid = _sha256_id(data)
147 await _add_object(db_session, repo.repo_id, data, object_id=oid)
148 await db_session.commit()
149
150 backend = AsyncMock()
151 backend.get = AsyncMock(return_value=None) # simulates missing file
152
153 result = await scan_object_integrity(db_session, backend, sample_size=10)
154 assert not result.ok
155 mismatch_ids = [m.object_id for m in result.mismatches]
156 assert oid in mismatch_ids
157 reasons = {m.reason for m in result.mismatches}
158 assert "missing" in reasons
159
160
161 # ---------------------------------------------------------------------------
162 # Integrity scan — hash mismatch
163 # ---------------------------------------------------------------------------
164
165 async def test_integrity_scan_detects_hash_mismatch(db_session: AsyncSession) -> None:
166 """scan_object_integrity must flag objects whose content no longer matches their ID."""
167 from musehub.maintenance.object_integrity import scan_object_integrity
168
169 repo = await create_repo(db_session, slug="integrity-mismatch", owner="testuser")
170 data = b"original content"
171 oid = _sha256_id(data)
172 await _add_object(db_session, repo.repo_id, data, object_id=oid)
173 await db_session.commit()
174
175 corrupted = b"corrupted content XYZ"
176 backend = AsyncMock()
177 backend.get = AsyncMock(return_value=corrupted)
178
179 result = await scan_object_integrity(db_session, backend, sample_size=10)
180 assert not result.ok
181 mismatch_ids = [m.object_id for m in result.mismatches]
182 assert oid in mismatch_ids
183 reasons = {m.reason for m in result.mismatches}
184 assert "hash_mismatch" in reasons
185
186
187 # ---------------------------------------------------------------------------
188 # Soft-delete
189 # ---------------------------------------------------------------------------
190
191 async def test_soft_delete_sets_deleted_at(db_session: AsyncSession) -> None:
192 """soft_delete_object must set deleted_at without removing the DB row."""
193 from musehub.maintenance.object_integrity import soft_delete_object
194 from musehub.db import musehub_models as db_models
195
196 repo = await create_repo(db_session, slug="soft-delete-test", owner="testuser")
197 data = b"soft delete me"
198 oid = _sha256_id(data)
199 await _add_object(db_session, repo.repo_id, data, object_id=oid)
200 await db_session.commit()
201
202 found = await soft_delete_object(db_session, oid)
203 assert found is True
204
205 row = await db_session.get(db_models.MusehubObject, oid)
206 assert row is not None, "row must still exist after soft delete"
207 assert row.deleted_at is not None, "deleted_at must be set"
208
209
210 async def test_soft_delete_unknown_object_returns_false(db_session: AsyncSession) -> None:
211 from musehub.maintenance.object_integrity import soft_delete_object
212
213 found = await soft_delete_object(db_session, "sha256:" + "f" * 64)
214 assert found is False
215
216
217 # ---------------------------------------------------------------------------
218 # Hard-delete reaper
219 # ---------------------------------------------------------------------------
220
221 async def test_reaper_removes_objects_past_retention(db_session: AsyncSession) -> None:
222 """reap_deleted_objects must hard-delete objects whose deleted_at exceeds retention."""
223 from datetime import datetime, timedelta, timezone
224 from musehub.maintenance.object_integrity import reap_deleted_objects, soft_delete_object
225 from musehub.db import musehub_models as db_models
226
227 repo = await create_repo(db_session, slug="reap-old", owner="testuser")
228 data = b"old deleted object"
229 oid = _sha256_id(data)
230 await _add_object(db_session, repo.repo_id, data, object_id=oid)
231 await db_session.commit()
232
233 # Soft-delete the object, then backdate deleted_at past the retention window.
234 await soft_delete_object(db_session, oid)
235 await db_session.flush()
236 row = await db_session.get(db_models.MusehubObject, oid)
237 assert row is not None
238 row.deleted_at = datetime.now(tz=timezone.utc) - timedelta(days=31)
239 await db_session.commit()
240
241 backend = AsyncMock()
242 backend.delete = AsyncMock()
243
244 reaped = await reap_deleted_objects(db_session, backend, retention_days=30)
245 assert reaped == 1
246 backend.delete.assert_called_once()
247
248 # Row must be gone from DB.
249 gone = await db_session.get(db_models.MusehubObject, oid)
250 assert gone is None, "hard-deleted object must be removed from DB"
251
252
253 async def test_reaper_spares_objects_within_retention(db_session: AsyncSession) -> None:
254 """reap_deleted_objects must NOT remove objects soft-deleted within retention window."""
255 from datetime import datetime, timedelta, timezone
256 from musehub.maintenance.object_integrity import reap_deleted_objects, soft_delete_object
257 from musehub.db import musehub_models as db_models
258
259 repo = await create_repo(db_session, slug="reap-new", owner="testuser")
260 data = b"recently deleted object"
261 oid = _sha256_id(data)
262 await _add_object(db_session, repo.repo_id, data, object_id=oid)
263 await db_session.commit()
264
265 await soft_delete_object(db_session, oid)
266 await db_session.commit()
267
268 # deleted_at is NOW — well within the 30-day window.
269 backend = AsyncMock()
270 backend.delete = AsyncMock()
271
272 reaped = await reap_deleted_objects(db_session, backend, retention_days=30)
273 assert reaped == 0
274 backend.delete.assert_not_called()
275
276 still_there = await db_session.get(db_models.MusehubObject, oid)
277 assert still_there is not None
278
279
280 # ---------------------------------------------------------------------------
281 # Per-repo quota — wire push
282 # ---------------------------------------------------------------------------
283
284 async def test_wire_push_rejects_when_repo_quota_exceeded(db_session: AsyncSession) -> None:
285 """wire_push must return ok=False when the push would exceed per_repo_quota_bytes."""
286 from musehub.services.musehub_wire import wire_push
287 from musehub.models.wire import WirePushRequest, WireBundle, WireObject
288
289 repo = await create_repo(db_session, slug="quota-wire", owner="testuser")
290 await db_session.commit()
291
292 big_content = b"x" * 100 # 100 bytes per object
293
294 bundle = WireBundle(
295 objects=[
296 WireObject(
297 object_id=_sha256_id(big_content),
298 path="big.bin",
299 content=big_content,
300 )
301 ],
302 commits=[],
303 snapshots=[],
304 )
305 req = WirePushRequest(repo_id=repo.repo_id, branch="main", bundle=bundle)
306
307 # Patch quota to 50 bytes — incoming 100 bytes will exceed it.
308 with patch("musehub.services.musehub_wire.settings") as mock_settings:
309 mock_settings.per_repo_quota_bytes = 50
310 result = await wire_push(db_session, repo.repo_id, req, pusher_id="testuser")
311
312 assert result.ok is False
313 assert "quota" in result.message.lower()
File History 1 commit
sha256:9590cee1e0ccd6c76528f005b95d634d80f5019f0dcb7c371e149adc31d1fb65 refactor: enforce gRPC framing on all MWP wire traffic Sonnet 4.6 minor ⚠ 156 days ago