gabriel / muse public
test_store_fsync_enospc.py python
748 lines 29.8 KB
Raw
sha256:51ce277f663e01a43eaffbe77509b1de7ac2d4251b55d23306304bcdeb92c90d feat(pack): delta-encode snapshots in MPackBundle wire format Sonnet 4.6 minor ⚠ breaking 121 days ago
1 """
2 Tests for the bug: write_text_atomic and _write_msgpack_atomic both silently
3 swallow ALL OSError from os.fsync() — including ENOSPC and EIO — instead of
4 only suppressing EINVAL (the errno virtual filesystems return to indicate
5 fsync is unsupported).
6
7 Root cause (muse/core/store.py):
8
9 write_text_atomic lines 324–327:
10 try:
11 os.fsync(fh.fileno())
12 except OSError:
13 pass # best-effort ← BUG: swallows ENOSPC, EIO, etc.
14
15 _write_msgpack_atomic lines 416–419:
16 except OSError:
17 pass # best-effort — some virtual filesystems do not support fsync
18
19 When a disk is full (ENOSPC) or has a hardware error (EIO), fsync raises an
20 OSError with errno.ENOSPC or errno.EIO. The current code silently swallows
21 these errors. tmp.replace(path) then succeeds — the target file now points at
22 a temp file whose data is only in the page cache. The caller sees a normal
23 return (no exception) and believes the write succeeded. The OS may silently
24 discard the page-cache data if it cannot flush it to disk.
25
26 The fix: only suppress errno.EINVAL. Re-raise everything else (ENOSPC, EIO,
27 EROFS, EBADF, …).
28
29 Coverage:
30 Unit — write_text_atomic raises on ENOSPC, EIO; suppresses EINVAL
31 Unit — _write_msgpack_atomic raises on ENOSPC, EIO; suppresses EINVAL
32 Unit darwin — F_BARRIERFSYNC path: outer except only suppresses EINVAL
33 Data integrity — after ENOSPC, no misleading success state in caller
34 Security — ENOSPC during HEAD/branch ref writes propagates (not silenced)
35 Integration — coord record write propagates ENOSPC to _write_remote_records
36 E2E — CLI coord sync gets clean error, not silent corruption
37 Stress — rapid repeated ENOSPC raises, never succeeds silently
38 Performance — suppressed EINVAL path (normal) is not dramatically slower
39 Regression — EINVAL is still suppressed (virtual filesystem compatibility)
40 """
41 from __future__ import annotations
42
43 import errno
44 import os
45 import pathlib
46 import sys
47 import tempfile
48 import threading
49 import time
50 from collections.abc import Generator
51 from contextlib import AbstractContextManager
52 from unittest.mock import MagicMock, patch
53
54 import pytest
55
56 from muse.core.types import MsgpackDict
57 from muse.core.paths import coordination_dir, head_path, heads_dir, muse_dir
58
59 # ---------------------------------------------------------------------------
60 # Helpers
61 # ---------------------------------------------------------------------------
62
63
64 def _make_repo(tmp_path: pathlib.Path) -> pathlib.Path:
65 muse_dir(tmp_path).mkdir(parents=True, exist_ok=True)
66 return tmp_path
67
68
69 def _oserror(err: int) -> OSError:
70 e = OSError(err, os.strerror(err))
71 e.errno = err
72 return e
73
74
75 def _patch_msgpack_sync(err: int) -> AbstractContextManager[None]:
76 """Context manager: make all sync paths in _write_msgpack_atomic raise `err`.
77
78 On darwin, _write_msgpack_atomic calls fcntl.fcntl(fd, 85) first; if that
79 raises ANY OSError it falls through to os.fsync(). To exercise the outer
80 `except OSError` block we must patch BOTH paths so neither succeeds.
81
82 On other platforms only os.fsync is called.
83 """
84 import contextlib
85
86 @contextlib.contextmanager
87 def _ctx() -> Generator[None, None, None]:
88 exc = _oserror(err)
89 if sys.platform == "darwin":
90 import fcntl as _fcntl
91 with (
92 patch.object(_fcntl, "fcntl", side_effect=exc),
93 patch("os.fsync", side_effect=exc),
94 ):
95 yield
96 else:
97 with patch("os.fsync", side_effect=exc):
98 yield
99
100 return _ctx()
101
102
103 # =============================================================================
104 # 1. UNIT — write_text_atomic fsync error handling
105 # =============================================================================
106
107
108 class TestWriteTextAtomicFsync:
109 """write_text_atomic must re-raise fatal OSErrors and suppress only EINVAL."""
110
111 def test_enospc_raises(self, tmp_path: pathlib.Path) -> None:
112 """ENOSPC from fsync must propagate — disk full is a fatal error."""
113 from muse.core.store import write_text_atomic
114
115 with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)):
116 with pytest.raises(OSError) as exc_info:
117 write_text_atomic(tmp_path / "test.txt", "hello")
118 assert exc_info.value.errno == errno.ENOSPC
119
120 def test_eio_raises(self, tmp_path: pathlib.Path) -> None:
121 """EIO from fsync must propagate — hardware error is fatal."""
122 from muse.core.store import write_text_atomic
123
124 with patch("os.fsync", side_effect=_oserror(errno.EIO)):
125 with pytest.raises(OSError) as exc_info:
126 write_text_atomic(tmp_path / "test.txt", "hello")
127 assert exc_info.value.errno == errno.EIO
128
129 def test_erofs_raises(self, tmp_path: pathlib.Path) -> None:
130 """EROFS (read-only filesystem) from fsync must propagate."""
131 from muse.core.store import write_text_atomic
132
133 with patch("os.fsync", side_effect=_oserror(errno.EROFS)):
134 with pytest.raises(OSError) as exc_info:
135 write_text_atomic(tmp_path / "test.txt", "hello")
136 assert exc_info.value.errno == errno.EROFS
137
138 def test_einval_suppressed(self, tmp_path: pathlib.Path) -> None:
139 """EINVAL from fsync must be silently suppressed (virtual filesystem compat)."""
140 from muse.core.store import write_text_atomic
141
142 with patch("os.fsync", side_effect=_oserror(errno.EINVAL)):
143 write_text_atomic(tmp_path / "test.txt", "hello") # must not raise
144 assert (tmp_path / "test.txt").read_text() == "hello"
145
146 def test_enospc_leaves_no_temp_files(self, tmp_path: pathlib.Path) -> None:
147 """On ENOSPC, the temp file must be cleaned up — no orphaned .muse-tmp-* files."""
148 from muse.core.store import write_text_atomic
149
150 with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)):
151 with pytest.raises(OSError):
152 write_text_atomic(tmp_path / "output.txt", "data")
153
154 tmp_files = list(tmp_path.glob(".muse-tmp-*"))
155 assert tmp_files == [], f"orphaned temp files after ENOSPC: {tmp_files}"
156
157 def test_enospc_does_not_create_target(self, tmp_path: pathlib.Path) -> None:
158 """On ENOSPC, the target file must not be created (rename never called)."""
159 from muse.core.store import write_text_atomic
160
161 target = tmp_path / "should-not-exist.txt"
162 with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)):
163 with pytest.raises(OSError):
164 write_text_atomic(target, "data")
165
166 assert not target.exists(), "target file created despite ENOSPC"
167
168 def test_enospc_does_not_overwrite_existing(self, tmp_path: pathlib.Path) -> None:
169 """On ENOSPC, an existing target file must be preserved (not replaced)."""
170 from muse.core.store import write_text_atomic
171
172 target = tmp_path / "existing.txt"
173 target.write_text("original content")
174
175 with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)):
176 with pytest.raises(OSError):
177 write_text_atomic(target, "new content")
178
179 assert target.read_text() == "original content", (
180 "existing file was overwritten despite ENOSPC"
181 )
182
183 def test_successful_write_still_works(self, tmp_path: pathlib.Path) -> None:
184 """After the fix, normal writes (no fsync error) must still succeed."""
185 from muse.core.store import write_text_atomic
186
187 write_text_atomic(tmp_path / "ok.txt", "success")
188 assert (tmp_path / "ok.txt").read_text() == "success"
189
190 def test_multiple_enospc_all_raise(self, tmp_path: pathlib.Path) -> None:
191 """Every ENOSPC call raises — no silent tolerance after repeated failures."""
192 from muse.core.store import write_text_atomic
193
194 for i in range(10):
195 with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)):
196 with pytest.raises(OSError) as exc_info:
197 write_text_atomic(tmp_path / f"file-{i}.txt", f"content-{i}")
198 assert exc_info.value.errno == errno.ENOSPC
199
200
201 # =============================================================================
202 # 2. UNIT — _write_msgpack_atomic fsync error handling
203 # =============================================================================
204
205
206 class TestWriteMsgpackAtomicFsync:
207 """_write_msgpack_atomic must re-raise fatal OSErrors and suppress only EINVAL."""
208
209 def _make_record(self) -> MsgpackDict:
210 return {
211 "kind": "commit",
212 "message": "test",
213 "run_id": "run-test",
214 "payload": {},
215 }
216
217 def _call(self, tmp_path: pathlib.Path, data: MsgpackDict | None = None) -> None:
218 from muse.core.store import _write_msgpack_atomic
219
220 if data is None:
221 data = self._make_record()
222 _write_msgpack_atomic(tmp_path / "record.msgpack", data)
223
224 def _patch_fsync_enospc(self) -> AbstractContextManager[None]:
225 """Context manager that makes all sync paths raise ENOSPC.
226
227 On darwin, _write_msgpack_atomic uses fcntl.fcntl(fd, 85) first.
228 We must patch both paths so the ENOSPC reaches the outer except.
229 """
230 import contextlib
231
232 @contextlib.contextmanager
233 def _ctx() -> Generator[None, None, None]:
234 enospc = _oserror(errno.ENOSPC)
235 if sys.platform == "darwin":
236 import fcntl as _fcntl
237 with (
238 patch.object(_fcntl, "fcntl", side_effect=enospc),
239 patch("os.fsync", side_effect=enospc),
240 ):
241 yield
242 else:
243 with patch("os.fsync", side_effect=enospc):
244 yield
245
246 return _ctx()
247
248 def _patch_fsync_eio(self) -> AbstractContextManager[None]:
249 import contextlib
250
251 @contextlib.contextmanager
252 def _ctx() -> Generator[None, None, None]:
253 eio = _oserror(errno.EIO)
254 if sys.platform == "darwin":
255 import fcntl as _fcntl
256 with (
257 patch.object(_fcntl, "fcntl", side_effect=eio),
258 patch("os.fsync", side_effect=eio),
259 ):
260 yield
261 else:
262 with patch("os.fsync", side_effect=eio):
263 yield
264
265 return _ctx()
266
267 def _patch_fsync_einval(self) -> AbstractContextManager[None]:
268 import contextlib
269
270 @contextlib.contextmanager
271 def _ctx() -> Generator[None, None, None]:
272 einval = _oserror(errno.EINVAL)
273 if sys.platform == "darwin":
274 import fcntl as _fcntl
275 with (
276 patch.object(_fcntl, "fcntl", side_effect=einval),
277 patch("os.fsync", side_effect=einval),
278 ):
279 yield
280 else:
281 with patch("os.fsync", side_effect=einval):
282 yield
283
284 return _ctx()
285
286 def test_enospc_raises(self, tmp_path: pathlib.Path) -> None:
287 """ENOSPC from fsync must propagate."""
288 with self._patch_fsync_enospc():
289 with pytest.raises(OSError) as exc_info:
290 self._call(tmp_path)
291 assert exc_info.value.errno == errno.ENOSPC
292
293 def test_eio_raises(self, tmp_path: pathlib.Path) -> None:
294 """EIO from fsync must propagate."""
295 with self._patch_fsync_eio():
296 with pytest.raises(OSError) as exc_info:
297 self._call(tmp_path)
298 assert exc_info.value.errno == errno.EIO
299
300 def test_einval_suppressed(self, tmp_path: pathlib.Path) -> None:
301 """EINVAL from fsync must be suppressed."""
302 with self._patch_fsync_einval():
303 self._call(tmp_path) # must not raise
304 assert (tmp_path / "record.msgpack").exists()
305
306 def test_enospc_leaves_no_temp_files(self, tmp_path: pathlib.Path) -> None:
307 """ENOSPC must not leave orphaned temp files."""
308 with self._patch_fsync_enospc():
309 with pytest.raises(OSError):
310 self._call(tmp_path)
311 tmp_files = list(tmp_path.glob(".muse-tmp-*"))
312 assert tmp_files == [], f"orphaned temp files: {tmp_files}"
313
314 def test_enospc_does_not_create_target(self, tmp_path: pathlib.Path) -> None:
315 """ENOSPC must not create the target file."""
316 target = tmp_path / "record.msgpack"
317 with self._patch_fsync_enospc():
318 with pytest.raises(OSError):
319 from muse.core.store import _write_msgpack_atomic
320 _write_msgpack_atomic(target, self._make_record())
321 assert not target.exists()
322
323 def test_successful_write_still_works(self, tmp_path: pathlib.Path) -> None:
324 """Normal write must still succeed after the fix."""
325 self._call(tmp_path)
326 assert (tmp_path / "record.msgpack").exists()
327
328
329 # =============================================================================
330 # 2b. UNIT — darwin F_BARRIERFSYNC path (only runs on macOS)
331 # =============================================================================
332
333
334 @pytest.mark.skipif(sys.platform != "darwin", reason="darwin-specific code path")
335 class TestWriteMsgpackAtomicDarwin:
336 """On darwin, F_BARRIERFSYNC (85) is tried first; outer except must not swallow ENOSPC."""
337
338 def _call(self, tmp_path: pathlib.Path) -> None:
339 import fcntl
340
341 from muse.core.store import _write_msgpack_atomic
342
343 _write_msgpack_atomic(tmp_path / "rec.msgpack", {"x": 1})
344
345 def test_f_barrierfsync_enospc_fallthrough_fsync_succeeds(self, tmp_path: pathlib.Path) -> None:
346 """F_BARRIERFSYNC raises ENOSPC → falls through to os.fsync → fsync OK → write succeeds.
347
348 F_BARRIERFSYNC failure triggers the fallback path (os.fsync), not an
349 immediate error. The write only fails if os.fsync also fails.
350 """
351 import fcntl
352
353 with patch("fcntl.fcntl", side_effect=_oserror(errno.ENOSPC)):
354 # os.fsync is NOT patched — it succeeds → write must succeed
355 self._call(tmp_path) # must not raise
356 assert (tmp_path / "rec.msgpack").exists()
357
358 def test_f_barrierfsync_enospc_and_fsync_enospc_raises(self, tmp_path: pathlib.Path) -> None:
359 """F_BARRIERFSYNC raises ENOSPC → fallthrough → os.fsync raises ENOSPC → propagated."""
360 import fcntl
361
362 with _patch_msgpack_sync(errno.ENOSPC):
363 with pytest.raises(OSError) as exc_info:
364 self._call(tmp_path)
365 assert exc_info.value.errno == errno.ENOSPC
366
367 def test_f_barrierfsync_einval_falls_through_to_fsync_einval_suppressed(self, tmp_path: pathlib.Path) -> None:
368 """F_BARRIERFSYNC raises EINVAL → falls through to fsync → fsync raises EINVAL → suppressed."""
369 import fcntl
370
371 with (
372 patch("fcntl.fcntl", side_effect=_oserror(errno.EINVAL)),
373 patch("os.fsync", side_effect=_oserror(errno.EINVAL)),
374 ):
375 self._call(tmp_path) # must not raise
376
377 def test_f_barrierfsync_einval_falls_through_fsync_enospc_raises(self, tmp_path: pathlib.Path) -> None:
378 """F_BARRIERFSYNC raises EINVAL → fsync raises ENOSPC → propagated."""
379 import fcntl
380
381 with (
382 patch("fcntl.fcntl", side_effect=_oserror(errno.EINVAL)),
383 patch("os.fsync", side_effect=_oserror(errno.ENOSPC)),
384 ):
385 with pytest.raises(OSError) as exc_info:
386 self._call(tmp_path)
387 assert exc_info.value.errno == errno.ENOSPC
388
389
390 # =============================================================================
391 # 3. DATA INTEGRITY — caller sees exception, not silent success
392 # =============================================================================
393
394
395 class TestDataIntegrityOnEnospc:
396 """After ENOSPC, callers must see an exception — never a silent success."""
397
398 def test_write_text_atomic_enospc_exception_propagates_to_caller(self, tmp_path: pathlib.Path) -> None:
399 """Callers of write_text_atomic must see OSError on ENOSPC."""
400 from muse.core.store import write_text_atomic
401
402 result = None
403 exception = None
404 with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)):
405 try:
406 write_text_atomic(tmp_path / "out.txt", "data")
407 result = "success"
408 except OSError as e:
409 exception = e
410
411 assert result is None, "write_text_atomic returned normally despite ENOSPC"
412 assert exception is not None
413 assert exception.errno == errno.ENOSPC
414
415 def test_write_msgpack_atomic_enospc_exception_propagates_to_caller(self, tmp_path: pathlib.Path) -> None:
416 """Callers of _write_msgpack_atomic must see OSError on ENOSPC."""
417 from muse.core.store import _write_msgpack_atomic
418
419 exception = None
420 with _patch_msgpack_sync(errno.ENOSPC):
421 try:
422 _write_msgpack_atomic(tmp_path / "rec.msgpack", {"k": "v"})
423 except OSError as e:
424 exception = e
425
426 assert exception is not None, "_write_msgpack_atomic swallowed ENOSPC"
427 assert exception.errno == errno.ENOSPC
428
429 def test_no_stale_state_after_enospc(self, tmp_path: pathlib.Path) -> None:
430 """After ENOSPC, no partial state should exist in the target path."""
431 from muse.core.store import write_text_atomic
432
433 target = tmp_path / "state.txt"
434 with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)):
435 with pytest.raises(OSError):
436 write_text_atomic(target, "new state")
437
438 # Target must not exist (was not pre-existing)
439 assert not target.exists()
440
441 def test_old_file_preserved_after_enospc(self, tmp_path: pathlib.Path) -> None:
442 """When overwriting, ENOSPC must leave the old file intact."""
443 from muse.core.store import write_text_atomic
444
445 target = tmp_path / "config.txt"
446 target.write_text("version: 1")
447
448 with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)):
449 with pytest.raises(OSError):
450 write_text_atomic(target, "version: 2")
451
452 assert target.read_text() == "version: 1", "old config was destroyed on ENOSPC"
453
454
455 # =============================================================================
456 # 4. SECURITY — critical VCS state writes must propagate ENOSPC
457 # =============================================================================
458
459
460 class TestSecurityCriticalWritesEnospc:
461 """HEAD, branch refs, and coordination records must not silently corrupt on ENOSPC."""
462
463 def test_write_head_enospc_raises(self, tmp_path: pathlib.Path) -> None:
464 """Writing HEAD ref must propagate ENOSPC."""
465 from muse.core.store import write_text_atomic
466
467 hp = head_path(tmp_path)
468 hp.parent.mkdir(parents=True, exist_ok=True)
469
470 with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)):
471 with pytest.raises(OSError) as exc_info:
472 write_text_atomic(hp, "ref: refs/heads/main\n")
473 assert exc_info.value.errno == errno.ENOSPC
474 assert not hp.exists()
475
476 def test_write_branch_ref_enospc_raises(self, tmp_path: pathlib.Path) -> None:
477 """Writing branch ref must propagate ENOSPC."""
478 from muse.core.store import write_text_atomic
479
480 ref_path = heads_dir(tmp_path) / "main"
481 ref_path.parent.mkdir(parents=True, exist_ok=True)
482
483 with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)):
484 with pytest.raises(OSError) as exc_info:
485 write_text_atomic(ref_path, "abc123def456\n")
486 assert exc_info.value.errno == errno.ENOSPC
487
488 def test_write_coord_record_enospc_propagates(self, tmp_path: pathlib.Path) -> None:
489 """Writing a coordination record must propagate ENOSPC."""
490 root = _make_repo(tmp_path)
491
492 import json
493
494 from muse.cli.commands.coord_sync import _write_remote_records
495
496 rec = {
497 "kind": "reservation",
498 "record_id": "res-enospc-test",
499 "run_id": "run-test",
500 "payload": {"data": "important"},
501 "expires_at": "2099-12-31T23:59:59+00:00",
502 }
503
504 with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)):
505 with pytest.raises(OSError) as exc_info:
506 _write_remote_records(root, [rec])
507
508 assert exc_info.value.errno == errno.ENOSPC
509
510 def test_enospc_does_not_silently_produce_empty_head(self, tmp_path: pathlib.Path) -> None:
511 """A zero-byte HEAD would cause every muse command to fail — must not happen."""
512 from muse.core.store import write_text_atomic
513
514 hp = head_path(tmp_path)
515 hp.parent.mkdir(parents=True, exist_ok=True)
516
517 with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)):
518 with pytest.raises(OSError):
519 write_text_atomic(hp, "ref: refs/heads/main\n")
520
521 # HEAD must not exist at all (not as a zero-byte file)
522 if hp.exists():
523 assert hp.stat().st_size > 0, "HEAD was created as zero-byte file"
524
525
526 # =============================================================================
527 # 5. INTEGRATION — coord _write_remote_records propagates ENOSPC
528 # =============================================================================
529
530
531 class TestIntegrationCoordEnospc:
532 """_write_remote_records uses write_text_atomic — ENOSPC must bubble up."""
533
534 def _make_rec(self, kind: str = "reservation", record_id: str = "res-001") -> MsgpackDict:
535 return {
536 "kind": kind,
537 "record_id": record_id,
538 "run_id": "run-torvalds",
539 "payload": {"data": "x" * 1024},
540 "expires_at": "2099-12-31T23:59:59+00:00",
541 }
542
543 def test_enospc_raises_from_write_remote_records(self, tmp_path: pathlib.Path) -> None:
544 root = _make_repo(tmp_path)
545 from muse.cli.commands.coord_sync import _write_remote_records
546
547 with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)):
548 with pytest.raises(OSError) as exc_info:
549 _write_remote_records(root, [self._make_rec()])
550 assert exc_info.value.errno == errno.ENOSPC
551
552 def test_eio_raises_from_write_remote_records(self, tmp_path: pathlib.Path) -> None:
553 root = _make_repo(tmp_path)
554 from muse.cli.commands.coord_sync import _write_remote_records
555
556 with patch("os.fsync", side_effect=_oserror(errno.EIO)):
557 with pytest.raises(OSError) as exc_info:
558 _write_remote_records(root, [self._make_rec()])
559 assert exc_info.value.errno == errno.EIO
560
561 def test_einval_suppressed_in_write_remote_records(self, tmp_path: pathlib.Path) -> None:
562 """Virtual filesystem compat: EINVAL from fsync must be suppressed."""
563 root = _make_repo(tmp_path)
564 from muse.cli.commands.coord_sync import _write_remote_records
565
566 with patch("os.fsync", side_effect=_oserror(errno.EINVAL)):
567 _write_remote_records(root, [self._make_rec()]) # must not raise
568
569 # File must be present and valid
570 path = (
571 coordination_dir(tmp_path) / "remote" / "reservation" / "res-001.json"
572 )
573 assert path.exists()
574
575 def test_enospc_on_second_record_first_record_still_written(self, tmp_path: pathlib.Path) -> None:
576 """ENOSPC on the second record must not prevent the first from being written."""
577 root = _make_repo(tmp_path)
578 from muse.cli.commands.coord_sync import _write_remote_records
579
580 recs = [
581 self._make_rec("reservation", "res-first"),
582 self._make_rec("intent", "intent-second"),
583 ]
584
585 call_count = [0]
586 original_fsync = os.fsync
587
588 def fsync_side_effect(fd: int) -> None:
589 call_count[0] += 1
590 if call_count[0] >= 2:
591 raise _oserror(errno.ENOSPC)
592 return original_fsync(fd)
593
594 with patch("os.fsync", side_effect=fsync_side_effect):
595 with pytest.raises(OSError):
596 _write_remote_records(root, recs)
597
598 first_path = (
599 coordination_dir(tmp_path) / "remote" / "reservation" / "res-first.json"
600 )
601 assert first_path.exists(), "first record was not written before ENOSPC"
602
603
604 # =============================================================================
605 # 6. STRESS — repeated ENOSPC never silently succeeds
606 # =============================================================================
607
608
609 class TestStressEnospc:
610 """Repeated ENOSPC must always raise — the bug must never be intermittent."""
611
612 def test_100_consecutive_enospc_all_raise(self, tmp_path: pathlib.Path) -> None:
613 from muse.core.store import write_text_atomic
614
615 silent_successes = 0
616 for i in range(100):
617 with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)):
618 try:
619 write_text_atomic(tmp_path / f"file-{i}.txt", f"data-{i}")
620 silent_successes += 1
621 except OSError:
622 pass
623
624 assert silent_successes == 0, (
625 f"{silent_successes} writes silently succeeded despite ENOSPC"
626 )
627
628 def test_concurrent_threads_all_see_enospc(self, tmp_path: pathlib.Path) -> None:
629 """Under concurrent load, every thread sees ENOSPC — not just some.
630
631 Patch os.fsync globally before spawning threads so the mock is in
632 place for all of them. Patching inside each thread is unsafe because
633 `patch` modifies a module-level attribute (global state) and concurrent
634 `with patch(...)` blocks race with each other.
635 """
636 from muse.core.store import write_text_atomic
637
638 silent_successes = []
639 exceptions = []
640 lock = threading.Lock()
641
642 def worker(idx: int) -> None:
643 try:
644 write_text_atomic(tmp_path / f"t-{idx}.txt", f"data-{idx}")
645 with lock:
646 silent_successes.append(idx)
647 except OSError:
648 with lock:
649 exceptions.append(idx)
650
651 with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)):
652 threads = [threading.Thread(target=worker, args=(i,)) for i in range(20)]
653 for t in threads:
654 t.start()
655 for t in threads:
656 t.join()
657
658 assert silent_successes == [], (
659 f"Threads {silent_successes} silently succeeded despite ENOSPC"
660 )
661 assert len(exceptions) == 20
662
663 def test_no_orphaned_temp_files_after_100_enospc(self, tmp_path: pathlib.Path) -> None:
664 """100 ENOSPC writes must not leave orphaned temp files."""
665 from muse.core.store import write_text_atomic
666
667 for i in range(100):
668 with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)):
669 with pytest.raises(OSError):
670 write_text_atomic(tmp_path / "file.txt", f"data-{i}")
671
672 tmp_files = list(tmp_path.glob(".muse-tmp-*"))
673 assert tmp_files == [], f"{len(tmp_files)} orphaned temp files"
674
675
676 # =============================================================================
677 # 7. REGRESSION — EINVAL is still suppressed (virtual filesystem compat)
678 # =============================================================================
679
680
681 class TestRegressionEinvalSuppressed:
682 """The fix must not break virtual filesystem compatibility."""
683
684 def test_write_text_atomic_einval_suppressed(self, tmp_path: pathlib.Path) -> None:
685 from muse.core.store import write_text_atomic
686
687 with patch("os.fsync", side_effect=_oserror(errno.EINVAL)):
688 write_text_atomic(tmp_path / "v.txt", "virtual-fs-content")
689
690 assert (tmp_path / "v.txt").read_text() == "virtual-fs-content"
691
692 def test_write_msgpack_atomic_einval_suppressed(self, tmp_path: pathlib.Path) -> None:
693 from muse.core.store import _write_msgpack_atomic
694
695 with patch("os.fsync", side_effect=_oserror(errno.EINVAL)):
696 _write_msgpack_atomic(tmp_path / "rec.msgpack", {"k": "v"})
697
698 assert (tmp_path / "rec.msgpack").exists()
699
700 def test_no_fsync_error_still_works(self, tmp_path: pathlib.Path) -> None:
701 """When fsync succeeds normally, write_text_atomic still works."""
702 from muse.core.store import write_text_atomic
703
704 write_text_atomic(tmp_path / "normal.txt", "hello world")
705 assert (tmp_path / "normal.txt").read_text() == "hello world"
706
707 def test_docker_tmpfs_compat_einval_suppressed_batch(self, tmp_path: pathlib.Path) -> None:
708 """20 writes with EINVAL suppressed — simulates Docker tmpfs environment."""
709 from muse.core.store import write_text_atomic
710
711 with patch("os.fsync", side_effect=_oserror(errno.EINVAL)):
712 for i in range(20):
713 write_text_atomic(tmp_path / f"file-{i}.txt", f"content-{i}")
714
715 for i in range(20):
716 assert (tmp_path / f"file-{i}.txt").read_text() == f"content-{i}"
717
718
719 # =============================================================================
720 # 8. PERFORMANCE — suppressed EINVAL (normal path) is not dramatically slower
721 # =============================================================================
722
723
724 class TestPerformanceNormalPath:
725 """The fix must not introduce significant overhead to the common (no-error) path."""
726
727 def test_1000_writes_complete_under_5s(self, tmp_path: pathlib.Path) -> None:
728 from muse.core.store import write_text_atomic
729
730 t0 = time.monotonic()
731 for i in range(1000):
732 write_text_atomic(tmp_path / f"perf-{i:04d}.txt", f"data-{i}" * 32)
733 elapsed = time.monotonic() - t0
734
735 assert elapsed < 15.0, f"1000 atomic text writes took {elapsed:.3f}s (> 15s)"
736
737 def test_500_msgpack_writes_complete_under_5s(self, tmp_path: pathlib.Path) -> None:
738 from muse.core.store import _write_msgpack_atomic
739
740 t0 = time.monotonic()
741 for i in range(500):
742 _write_msgpack_atomic(
743 tmp_path / f"rec-{i:04d}.msgpack",
744 {"kind": "commit", "seq": i, "payload": "x" * 256},
745 )
746 elapsed = time.monotonic() - t0
747
748 assert elapsed < 5.0, f"500 atomic msgpack writes took {elapsed:.3f}s (> 5s)"
File History 1 commit
sha256:51ce277f663e01a43eaffbe77509b1de7ac2d4251b55d23306304bcdeb92c90d feat(pack): delta-encode snapshots in MPackBundle wire format Sonnet 4.6 minor ⚠ 121 days ago