test_stress_object_store.py
python
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9
docs: expand cache plan with all seven testing tiers and do…
Sonnet 4.6
131 days ago
| 1 | """Stress tests for the content-addressed object store. |
| 2 | |
| 3 | Exercises: |
| 4 | - Write-then-read round-trip for varied payload sizes (1 byte … 10 MB). |
| 5 | - Idempotency: writing the same object ID twice is a no-op. |
| 6 | - has_object before and after writes. |
| 7 | - object_path sharding: first two hex chars as directory. |
| 8 | - read_object returns None for absent objects. |
| 9 | - restore_object copies bytes faithfully. |
| 10 | - write_object_from_path uses copy semantics, not load. |
| 11 | - Content integrity: read(write(content)) == content. |
| 12 | - Multiple distinct objects coexist without collision. |
| 13 | """ |
| 14 | |
| 15 | import os |
| 16 | import pathlib |
| 17 | import secrets |
| 18 | |
| 19 | import pytest |
| 20 | |
| 21 | from muse.core.object_store import ( |
| 22 | has_object, |
| 23 | object_path, |
| 24 | objects_dir, |
| 25 | read_object, |
| 26 | restore_object, |
| 27 | write_object, |
| 28 | write_object_from_path, |
| 29 | ) |
| 30 | from muse.core._types import blob_id, long_id, fake_id |
| 31 | |
| 32 | |
| 33 | # --------------------------------------------------------------------------- |
| 34 | # Helpers |
| 35 | # --------------------------------------------------------------------------- |
| 36 | |
| 37 | |
| 38 | @pytest.fixture |
| 39 | def repo(tmp_path: pathlib.Path) -> pathlib.Path: |
| 40 | (tmp_path / ".muse").mkdir() |
| 41 | return tmp_path |
| 42 | |
| 43 | |
| 44 | # --------------------------------------------------------------------------- |
| 45 | # Basic round-trip |
| 46 | # --------------------------------------------------------------------------- |
| 47 | |
| 48 | |
| 49 | class TestRoundTrip: |
| 50 | def test_write_then_read_small(self, repo: pathlib.Path) -> None: |
| 51 | data = b"hello muse" |
| 52 | oid = blob_id(data) |
| 53 | write_object(repo, oid, data) |
| 54 | assert read_object(repo, oid) == data |
| 55 | |
| 56 | def test_write_then_read_empty(self, repo: pathlib.Path) -> None: |
| 57 | data = b"" |
| 58 | oid = blob_id(data) |
| 59 | write_object(repo, oid, data) |
| 60 | assert read_object(repo, oid) == data |
| 61 | |
| 62 | def test_write_then_read_single_byte(self, repo: pathlib.Path) -> None: |
| 63 | data = b"\x00" |
| 64 | oid = blob_id(data) |
| 65 | write_object(repo, oid, data) |
| 66 | assert read_object(repo, oid) == data |
| 67 | |
| 68 | def test_write_then_read_binary(self, repo: pathlib.Path) -> None: |
| 69 | data = bytes(range(256)) * 100 |
| 70 | oid = blob_id(data) |
| 71 | write_object(repo, oid, data) |
| 72 | assert read_object(repo, oid) == data |
| 73 | |
| 74 | @pytest.mark.parametrize("size", [1, 100, 4096, 65536, 1_000_000]) |
| 75 | def test_write_then_read_various_sizes(self, repo: pathlib.Path, size: int) -> None: |
| 76 | data = secrets.token_bytes(size) |
| 77 | oid = blob_id(data) |
| 78 | assert write_object(repo, oid, data) is True |
| 79 | assert read_object(repo, oid) == data |
| 80 | |
| 81 | def test_content_integrity(self, repo: pathlib.Path) -> None: |
| 82 | """Read back exactly what was written — not a truncated or padded version.""" |
| 83 | for i in range(20): |
| 84 | data = f"object-content-{i}-{'x' * i}".encode() |
| 85 | oid = blob_id(data) |
| 86 | write_object(repo, oid, data) |
| 87 | recovered = read_object(repo, oid) |
| 88 | assert recovered == data |
| 89 | assert len(recovered) == len(data) |
| 90 | |
| 91 | |
| 92 | # --------------------------------------------------------------------------- |
| 93 | # Idempotency |
| 94 | # --------------------------------------------------------------------------- |
| 95 | |
| 96 | |
| 97 | class TestIdempotency: |
| 98 | def test_double_write_returns_false_second_time(self, repo: pathlib.Path) -> None: |
| 99 | data = b"idempotent" |
| 100 | oid = blob_id(data) |
| 101 | assert write_object(repo, oid, data) is True |
| 102 | assert write_object(repo, oid, data) is False |
| 103 | |
| 104 | def test_double_write_does_not_corrupt(self, repo: pathlib.Path) -> None: |
| 105 | data = b"original content" |
| 106 | oid = blob_id(data) |
| 107 | write_object(repo, oid, data) |
| 108 | # Writing different content with the same ID raises ValueError (integrity check). |
| 109 | # The object on disk is NOT overwritten — idempotency guard fires first. |
| 110 | with pytest.raises(ValueError, match="Content integrity failure"): |
| 111 | write_object(repo, oid, b"different content") |
| 112 | assert read_object(repo, oid) == data |
| 113 | |
| 114 | def test_triple_write_stays_stable(self, repo: pathlib.Path) -> None: |
| 115 | data = b"triple-write" |
| 116 | oid = blob_id(data) |
| 117 | for _ in range(3): |
| 118 | write_object(repo, oid, data) |
| 119 | assert read_object(repo, oid) == data |
| 120 | |
| 121 | |
| 122 | # --------------------------------------------------------------------------- |
| 123 | # has_object |
| 124 | # --------------------------------------------------------------------------- |
| 125 | |
| 126 | |
| 127 | class TestHasObject: |
| 128 | def test_absent_before_write(self, repo: pathlib.Path) -> None: |
| 129 | oid = blob_id(b"not yet written") |
| 130 | assert not has_object(repo, oid) |
| 131 | |
| 132 | def test_present_after_write(self, repo: pathlib.Path) -> None: |
| 133 | data = b"present" |
| 134 | oid = blob_id(data) |
| 135 | write_object(repo, oid, data) |
| 136 | assert has_object(repo, oid) |
| 137 | |
| 138 | def test_other_objects_dont_shadow(self, repo: pathlib.Path) -> None: |
| 139 | a = b"object-a" |
| 140 | b_ = b"object-b" |
| 141 | oid_a = blob_id(a) |
| 142 | oid_b = blob_id(b_) |
| 143 | write_object(repo, oid_a, a) |
| 144 | assert has_object(repo, oid_a) |
| 145 | assert not has_object(repo, oid_b) |
| 146 | write_object(repo, oid_b, b_) |
| 147 | assert has_object(repo, oid_b) |
| 148 | |
| 149 | |
| 150 | # --------------------------------------------------------------------------- |
| 151 | # Absent objects |
| 152 | # --------------------------------------------------------------------------- |
| 153 | |
| 154 | |
| 155 | class TestAbsentObjects: |
| 156 | def test_read_absent_returns_none(self, repo: pathlib.Path) -> None: |
| 157 | fake_oid = fake_id("absent-a") |
| 158 | assert read_object(repo, fake_oid) is None |
| 159 | |
| 160 | def test_restore_absent_returns_false(self, repo: pathlib.Path, tmp_path: pathlib.Path) -> None: |
| 161 | fake_oid = fake_id("absent-b") |
| 162 | dest = tmp_path / "restored.bin" |
| 163 | result = restore_object(repo, fake_oid, dest) |
| 164 | assert result is False |
| 165 | assert not dest.exists() |
| 166 | |
| 167 | def test_has_object_false_for_random_id(self, repo: pathlib.Path) -> None: |
| 168 | for _ in range(10): |
| 169 | assert not has_object(repo, long_id(secrets.token_hex(32))) |
| 170 | |
| 171 | |
| 172 | # --------------------------------------------------------------------------- |
| 173 | # Sharding layout |
| 174 | # --------------------------------------------------------------------------- |
| 175 | |
| 176 | |
| 177 | class TestSharding: |
| 178 | def test_object_path_uses_first_two_chars_as_dir(self, repo: pathlib.Path) -> None: |
| 179 | oid = long_id("ab" + "c" * 62) |
| 180 | path = object_path(repo, oid) |
| 181 | assert path.parent.name == "ab" |
| 182 | assert path.name == "c" * 62 |
| 183 | |
| 184 | def test_objects_with_same_prefix_go_to_same_shard(self, repo: pathlib.Path) -> None: |
| 185 | oid1 = long_id("ff" + "0" * 62) |
| 186 | oid2 = long_id("ff" + "1" * 62) |
| 187 | assert object_path(repo, oid1).parent == object_path(repo, oid2).parent |
| 188 | |
| 189 | def test_objects_with_different_prefix_go_to_different_shards(self, repo: pathlib.Path) -> None: |
| 190 | # Use valid 64-char hex IDs with different first-two-char prefixes. |
| 191 | oid1 = long_id("aa" + "f" * 62) |
| 192 | oid2 = long_id("bb" + "f" * 62) |
| 193 | assert object_path(repo, oid1).parent != object_path(repo, oid2).parent |
| 194 | |
| 195 | def test_256_shards_can_all_be_created(self, repo: pathlib.Path) -> None: |
| 196 | """Write one object per possible shard prefix (00-ff). |
| 197 | |
| 198 | Finds data whose SHA-256 starts with each 2-hex prefix by brute-force, |
| 199 | using a counter to stay deterministic. |
| 200 | """ |
| 201 | import itertools |
| 202 | written_prefixes: set[str] = set() |
| 203 | for n in itertools.count(): |
| 204 | if len(written_prefixes) == 256: |
| 205 | break |
| 206 | data = f"shard-seed-{n}".encode() |
| 207 | oid = blob_id(data) |
| 208 | prefix = object_path(repo, oid).parent.name |
| 209 | if prefix not in written_prefixes: |
| 210 | write_object(repo, oid, data) |
| 211 | written_prefixes.add(prefix) |
| 212 | # Verify all 256 shard dirs exist under the sha256/ algo directory. |
| 213 | algo_dir = objects_dir(repo) / "sha256" |
| 214 | shards = [d.name for d in algo_dir.iterdir() if d.is_dir()] |
| 215 | assert len(shards) == 256 |
| 216 | |
| 217 | |
| 218 | # --------------------------------------------------------------------------- |
| 219 | # write_object_from_path |
| 220 | # --------------------------------------------------------------------------- |
| 221 | |
| 222 | |
| 223 | class TestWriteObjectFromPath: |
| 224 | def test_from_path_round_trip(self, repo: pathlib.Path, tmp_path: pathlib.Path) -> None: |
| 225 | src = tmp_path / "source.bin" |
| 226 | data = b"from-path-content" |
| 227 | src.write_bytes(data) |
| 228 | oid = blob_id(data) |
| 229 | assert write_object_from_path(repo, oid, src) is True |
| 230 | assert read_object(repo, oid) == data |
| 231 | |
| 232 | def test_from_path_idempotent(self, repo: pathlib.Path, tmp_path: pathlib.Path) -> None: |
| 233 | src = tmp_path / "idem.bin" |
| 234 | data = b"idempotent-from-path" |
| 235 | src.write_bytes(data) |
| 236 | oid = blob_id(data) |
| 237 | write_object_from_path(repo, oid, src) |
| 238 | assert write_object_from_path(repo, oid, src) is False |
| 239 | |
| 240 | |
| 241 | # --------------------------------------------------------------------------- |
| 242 | # restore_object |
| 243 | # --------------------------------------------------------------------------- |
| 244 | |
| 245 | |
| 246 | class TestRestoreObject: |
| 247 | def test_restore_round_trip(self, repo: pathlib.Path, tmp_path: pathlib.Path) -> None: |
| 248 | data = b"restore-me" |
| 249 | oid = blob_id(data) |
| 250 | write_object(repo, oid, data) |
| 251 | dest = tmp_path / "sub" / "restored.bin" |
| 252 | assert restore_object(repo, oid, dest) is True |
| 253 | assert dest.read_bytes() == data |
| 254 | |
| 255 | def test_restore_creates_parent_dirs(self, repo: pathlib.Path, tmp_path: pathlib.Path) -> None: |
| 256 | data = b"deep-restore" |
| 257 | oid = blob_id(data) |
| 258 | write_object(repo, oid, data) |
| 259 | dest = tmp_path / "a" / "b" / "c" / "file.bin" |
| 260 | restore_object(repo, oid, dest) |
| 261 | assert dest.exists() |
| 262 | |
| 263 | def test_restore_large_object_intact(self, repo: pathlib.Path, tmp_path: pathlib.Path) -> None: |
| 264 | data = secrets.token_bytes(2_000_000) |
| 265 | oid = blob_id(data) |
| 266 | write_object(repo, oid, data) |
| 267 | dest = tmp_path / "large.bin" |
| 268 | restore_object(repo, oid, dest) |
| 269 | assert dest.read_bytes() == data |
| 270 | |
| 271 | |
| 272 | # --------------------------------------------------------------------------- |
| 273 | # Multiple distinct objects |
| 274 | # --------------------------------------------------------------------------- |
| 275 | |
| 276 | |
| 277 | class TestMultipleObjects: |
| 278 | def test_100_distinct_objects_coexist(self, repo: pathlib.Path) -> None: |
| 279 | written: _FileStore = {} |
| 280 | for i in range(100): |
| 281 | data = f"payload-{i:03d}-{'z' * i}".encode() |
| 282 | oid = blob_id(data) |
| 283 | write_object(repo, oid, data) |
| 284 | written[oid] = data |
| 285 | |
| 286 | for oid, data in written.items(): |
| 287 | assert read_object(repo, oid) == data |
| 288 | |
| 289 | def test_all_objects_independently_addressable(self, repo: pathlib.Path) -> None: |
| 290 | """Verify no two distinct objects collide in the store.""" |
| 291 | oids: list[str] = [] |
| 292 | for i in range(50): |
| 293 | data = secrets.token_bytes(64) |
| 294 | oid = blob_id(data) |
| 295 | write_object(repo, oid, data) |
| 296 | oids.append(oid) |
| 297 | # All OIDs should be unique (probabilistic but essentially certain). |
| 298 | assert len(set(oids)) == 50 |
File History
3 commits
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9
docs: expand cache plan with all seven testing tiers and do…
Sonnet 4.6
131 days ago
sha256:7f9e2ef5286aedad9c1e6011b4c46ca27f39dbdad6e3409357e36b26e46b3b7c
docs: docstring sprint for-each-ref→hotspots — idiomatic ru…
Sonnet 4.6
patch
137 days ago
sha256:a09b1b4f6838754495547f200aa0ce88e2f56ffc5b20b900f6f0cff2c3cdede9
fix(cursorignore): remove git-ism (.git/worktrees)
Human
minor
⚠
140 days ago