gabriel / muse public
test_wire_oc_n_objects.py python
226 lines 7.9 KB
Raw
sha256:51ce277f663e01a43eaffbe77509b1de7ac2d4251b55d23306304bcdeb92c90d feat(pack): delta-encode snapshots in MPackBundle wire format Sonnet 4.6 minor ⚠ breaking 130 days ago
1 """TDD — n_objects in H/E frames must equal distinct objects, not raw OC-entry count.
2
3 Bug: _frame_generator passed len(objects) to both H and E frame n_objects.
4 When an object is split into N OC chunks, it contributes N entries to
5 `objects` but only 1 to n_objects_received on the server side, which
6 counts assembled OC sequences. This produces a count-mismatch 400 on
7 the first real push that contains any chunked (>OC_CHUNK_SIZE) object.
8
9 Observed failure:
10 "count mismatch: E frame claims 1029 object frames, server received 1028"
11 Root cause: 1027 O-frames + 1 OC object × 2 chunks = 1029 entries in
12 objects list, but server assembled 1027 O + 1 OC = 1028 distinct objects.
13
14 Rules:
15
16 P-OC-N0 Empty objects list → H.n_objects=0, E.n_objects=0.
17
18 P-OC-N1 Two plain O-objects → H.n_objects=2, E.n_objects=2.
19
20 P-OC-N2 One OC object with 2 chunk entries in objects list →
21 H.n_objects=1, E.n_objects=1.
22 (Regression test: the broken code emits n_objects=2 here.)
23
24 P-OC-N3 1 O-object + 1 OC-object (3 chunk entries) = 4 list entries →
25 H.n_objects=2, E.n_objects=2.
26
27 P-OC-N4 H.n_objects == E.n_objects for all inputs (consistency invariant).
28 """
29 from __future__ import annotations
30
31 import struct
32
33 import msgpack
34 from muse.core.types import MsgpackDict, blob_id
35
36
37 # ---------------------------------------------------------------------------
38 # Helpers
39 # ---------------------------------------------------------------------------
40
41 def _make_o_object(content: bytes, path: str = "") -> MsgpackDict:
42 """Plain O-frame object entry."""
43 return {
44 "object_id": blob_id(content),
45 "content": content,
46 "encoding": "raw",
47 "path": path,
48 }
49
50
51 def _make_oc_chunks(content: bytes, path: str = "") -> list[dict]:
52 """Two OC-frame entries for a single logical object (simulates Phase-14B chunking)."""
53 oid = blob_id(content)
54 # Split into 2 halves — sizes don't matter, just that t="OC" with same oid.
55 mid = max(1, len(content) // 2)
56 chunk0 = content[:mid]
57 chunk1 = content[mid:]
58 return [
59 {
60 "t": "OC",
61 "object_id": oid,
62 "chunk_index": 0,
63 "total_chunks": 2,
64 "content": chunk0,
65 "encoding": "raw",
66 "path": path,
67 "sz": len(content),
68 },
69 {
70 "t": "OC",
71 "object_id": oid,
72 "chunk_index": 1,
73 "total_chunks": 2,
74 "content": chunk1,
75 },
76 ]
77
78
79 def _parse_mwp_frames(data: bytes) -> list[dict]:
80 """Parse concatenated MWP-framed bytes into a list of decoded payload dicts."""
81 frames: list[dict] = []
82 pos = 0
83 magic = b"muse"
84 while pos < len(data):
85 assert data[pos : pos + 4] == magic, f"bad magic at {pos}: {data[pos:pos+4]!r}"
86 pos += 4
87 pos += 1 # version byte
88 header_len = struct.unpack_from(">I", data, pos)[0]
89 pos += 4
90 pos += header_len # skip envelope header (ft/id/sz)
91 payload_len = struct.unpack_from(">Q", data, pos)[0]
92 pos += 8
93 payload_bytes = data[pos : pos + payload_len]
94 pos += payload_len
95 frames.append(msgpack.unpackb(payload_bytes, raw=False))
96 return frames
97
98
99 def _h_and_e(objects: list[MsgpackDict]) -> tuple[MsgpackDict, MsgpackDict]:
100 """Run _frame_generator and return the decoded H and E frame payloads."""
101 from muse.core.transport import _frame_generator
102
103 raw = b"".join(_frame_generator(objects, [], [], local_head=None))
104 frames = _parse_mwp_frames(raw)
105
106 h_frame = next((f for f in frames if f.get("t") == "H"), None)
107 e_frame = next((f for f in frames if f.get("t") == "E"), None)
108
109 assert h_frame is not None, "no H frame in output"
110 assert e_frame is not None, "no E frame in output"
111 return h_frame, e_frame
112
113
114 # ---------------------------------------------------------------------------
115 # P-OC-N0 — empty objects list
116 # ---------------------------------------------------------------------------
117
118 class TestPOCN0Empty:
119 def test_h_n_objects_is_zero(self) -> None:
120 h, _ = _h_and_e([])
121 assert h["n_objects"] == 0
122
123 def test_e_n_objects_is_zero(self) -> None:
124 _, e = _h_and_e([])
125 assert e["n_objects"] == 0
126
127
128 # ---------------------------------------------------------------------------
129 # P-OC-N1 — two plain O-objects
130 # ---------------------------------------------------------------------------
131
132 class TestPOCN1TwoOObjects:
133 def _objects(self) -> list[MsgpackDict]:
134 return [
135 _make_o_object(b"alpha" * 100),
136 _make_o_object(b"beta" * 100),
137 ]
138
139 def test_h_n_objects_is_two(self) -> None:
140 h, _ = _h_and_e(self._objects())
141 assert h["n_objects"] == 2
142
143 def test_e_n_objects_is_two(self) -> None:
144 _, e = _h_and_e(self._objects())
145 assert e["n_objects"] == 2
146
147
148 # ---------------------------------------------------------------------------
149 # P-OC-N2 — one OC object split into 2 chunks (regression for the bug)
150 # ---------------------------------------------------------------------------
151
152 class TestPOCN2OneOCObjectTwoChunks:
153 """Core regression: 2 OC chunk entries represent 1 distinct object.
154
155 The broken code emits n_objects=2 (len(objects)); the correct code emits 1.
156 """
157
158 def _objects(self) -> list[MsgpackDict]:
159 return _make_oc_chunks(b"x" * 600_000) # 2 entries, same oid
160
161 def test_h_n_objects_is_one(self) -> None:
162 h, _ = _h_and_e(self._objects())
163 assert h["n_objects"] == 1, (
164 f"H frame says n_objects={h['n_objects']}; expected 1. "
165 "Two OC chunk entries for the same oid represent one distinct object."
166 )
167
168 def test_e_n_objects_is_one(self) -> None:
169 _, e = _h_and_e(self._objects())
170 assert e["n_objects"] == 1, (
171 f"E frame says n_objects={e['n_objects']}; expected 1. "
172 "Two OC chunk entries for the same oid represent one distinct object."
173 )
174
175
176 # ---------------------------------------------------------------------------
177 # P-OC-N3 — 1 O + 1 OC(3 chunks) = 4 entries → 2 distinct objects
178 # ---------------------------------------------------------------------------
179
180 class TestPOCN3MixedObjects:
181 def _objects(self) -> list[MsgpackDict]:
182 o_obj = _make_o_object(b"plain" * 50)
183 # Build a 3-chunk OC object manually (simpler than invoking split_object_into_oc_frames)
184 oid = blob_id(b"large" * 200)
185 oc_chunks = [
186 {
187 "t": "OC",
188 "object_id": oid,
189 "chunk_index": i,
190 "total_chunks": 3,
191 "content": b"large" * 67,
192 **({"encoding": "raw", "path": "big.bin", "sz": 1000} if i == 0 else {}),
193 }
194 for i in range(3)
195 ]
196 return [o_obj] + oc_chunks # 4 entries, 2 distinct objects
197
198 def test_h_n_objects_is_two(self) -> None:
199 h, _ = _h_and_e(self._objects())
200 assert h["n_objects"] == 2
201
202 def test_e_n_objects_is_two(self) -> None:
203 _, e = _h_and_e(self._objects())
204 assert e["n_objects"] == 2
205
206
207 # ---------------------------------------------------------------------------
208 # P-OC-N4 — H.n_objects always equals E.n_objects
209 # ---------------------------------------------------------------------------
210
211 class TestPOCN4HEConsistency:
212 def _cases(self) -> list[list]:
213 return [
214 [],
215 [_make_o_object(b"a" * 100)],
216 _make_oc_chunks(b"b" * 600_000),
217 [_make_o_object(b"c" * 100)] + _make_oc_chunks(b"d" * 600_000),
218 ]
219
220 def test_h_equals_e_for_all_cases(self) -> None:
221 for objects in self._cases():
222 h, e = _h_and_e(objects)
223 assert h["n_objects"] == e["n_objects"], (
224 f"H.n_objects={h['n_objects']} != E.n_objects={e['n_objects']} "
225 f"for objects list of length {len(objects)}"
226 )
File History 1 commit
sha256:51ce277f663e01a43eaffbe77509b1de7ac2d4251b55d23306304bcdeb92c90d feat(pack): delta-encode snapshots in MPackBundle wire format Sonnet 4.6 minor ⚠ 130 days ago