gabriel / muse public
test_wire_oc_n_objects.py python
230 lines 7.9 KB
Raw
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9 docs: expand cache plan with all seven testing tiers and do… Sonnet 4.6 131 days ago
1 """TDD — n_objects in H/E frames must equal distinct objects, not raw OC-entry count.
2
3 Bug: _frame_generator passed len(objects) to both H and E frame n_objects.
4 When an object is split into N OC chunks, it contributes N entries to
5 `objects` but only 1 to n_objects_received on the server side, which
6 counts assembled OC sequences. This produces a count-mismatch 400 on
7 the first real push that contains any chunked (>OC_CHUNK_SIZE) object.
8
9 Observed failure:
10 "count mismatch: E frame claims 1029 object frames, server received 1028"
11 Root cause: 1027 O-frames + 1 OC object × 2 chunks = 1029 entries in
12 objects list, but server assembled 1027 O + 1 OC = 1028 distinct objects.
13
14 Rules:
15
16 P-OC-N0 Empty objects list → H.n_objects=0, E.n_objects=0.
17
18 P-OC-N1 Two plain O-objects → H.n_objects=2, E.n_objects=2.
19
20 P-OC-N2 One OC object with 2 chunk entries in objects list →
21 H.n_objects=1, E.n_objects=1.
22 (Regression test: the broken code emits n_objects=2 here.)
23
24 P-OC-N3 1 O-object + 1 OC-object (3 chunk entries) = 4 list entries →
25 H.n_objects=2, E.n_objects=2.
26
27 P-OC-N4 H.n_objects == E.n_objects for all inputs (consistency invariant).
28 """
29 from __future__ import annotations
30
31 import struct
32
33 import msgpack
34 from muse.core._types import MsgpackDict, blob_id
35
36
37 # ---------------------------------------------------------------------------
38 # Helpers
39 # ---------------------------------------------------------------------------
40
41 def _sha256_oid(data: bytes) -> str:
42 return blob_id(data)
43
44
45 def _make_o_object(content: bytes, path: str = "") -> MsgpackDict:
46 """Plain O-frame object entry."""
47 return {
48 "object_id": _sha256_oid(content),
49 "content": content,
50 "encoding": "raw",
51 "path": path,
52 }
53
54
55 def _make_oc_chunks(content: bytes, path: str = "") -> list[dict]:
56 """Two OC-frame entries for a single logical object (simulates Phase-14B chunking)."""
57 oid = _sha256_oid(content)
58 # Split into 2 halves — sizes don't matter, just that t="OC" with same oid.
59 mid = max(1, len(content) // 2)
60 chunk0 = content[:mid]
61 chunk1 = content[mid:]
62 return [
63 {
64 "t": "OC",
65 "object_id": oid,
66 "chunk_index": 0,
67 "total_chunks": 2,
68 "content": chunk0,
69 "encoding": "raw",
70 "path": path,
71 "sz": len(content),
72 },
73 {
74 "t": "OC",
75 "object_id": oid,
76 "chunk_index": 1,
77 "total_chunks": 2,
78 "content": chunk1,
79 },
80 ]
81
82
83 def _parse_mwp_frames(data: bytes) -> list[dict]:
84 """Parse concatenated MWP-framed bytes into a list of decoded payload dicts."""
85 frames: list[dict] = []
86 pos = 0
87 magic = b"muse"
88 while pos < len(data):
89 assert data[pos : pos + 4] == magic, f"bad magic at {pos}: {data[pos:pos+4]!r}"
90 pos += 4
91 pos += 1 # version byte
92 header_len = struct.unpack_from(">I", data, pos)[0]
93 pos += 4
94 pos += header_len # skip envelope header (ft/id/sz)
95 payload_len = struct.unpack_from(">Q", data, pos)[0]
96 pos += 8
97 payload_bytes = data[pos : pos + payload_len]
98 pos += payload_len
99 frames.append(msgpack.unpackb(payload_bytes, raw=False))
100 return frames
101
102
103 def _h_and_e(objects: list[MsgpackDict]) -> tuple[MsgpackDict, MsgpackDict]:
104 """Run _frame_generator and return the decoded H and E frame payloads."""
105 from muse.core.transport import _frame_generator
106
107 raw = b"".join(_frame_generator(objects, [], [], local_head=None))
108 frames = _parse_mwp_frames(raw)
109
110 h_frame = next((f for f in frames if f.get("t") == "H"), None)
111 e_frame = next((f for f in frames if f.get("t") == "E"), None)
112
113 assert h_frame is not None, "no H frame in output"
114 assert e_frame is not None, "no E frame in output"
115 return h_frame, e_frame
116
117
118 # ---------------------------------------------------------------------------
119 # P-OC-N0 — empty objects list
120 # ---------------------------------------------------------------------------
121
122 class TestPOCN0Empty:
123 def test_h_n_objects_is_zero(self) -> None:
124 h, _ = _h_and_e([])
125 assert h["n_objects"] == 0
126
127 def test_e_n_objects_is_zero(self) -> None:
128 _, e = _h_and_e([])
129 assert e["n_objects"] == 0
130
131
132 # ---------------------------------------------------------------------------
133 # P-OC-N1 — two plain O-objects
134 # ---------------------------------------------------------------------------
135
136 class TestPOCN1TwoOObjects:
137 def _objects(self) -> list[MsgpackDict]:
138 return [
139 _make_o_object(b"alpha" * 100),
140 _make_o_object(b"beta" * 100),
141 ]
142
143 def test_h_n_objects_is_two(self) -> None:
144 h, _ = _h_and_e(self._objects())
145 assert h["n_objects"] == 2
146
147 def test_e_n_objects_is_two(self) -> None:
148 _, e = _h_and_e(self._objects())
149 assert e["n_objects"] == 2
150
151
152 # ---------------------------------------------------------------------------
153 # P-OC-N2 — one OC object split into 2 chunks (regression for the bug)
154 # ---------------------------------------------------------------------------
155
156 class TestPOCN2OneOCObjectTwoChunks:
157 """Core regression: 2 OC chunk entries represent 1 distinct object.
158
159 The broken code emits n_objects=2 (len(objects)); the correct code emits 1.
160 """
161
162 def _objects(self) -> list[MsgpackDict]:
163 return _make_oc_chunks(b"x" * 600_000) # 2 entries, same oid
164
165 def test_h_n_objects_is_one(self) -> None:
166 h, _ = _h_and_e(self._objects())
167 assert h["n_objects"] == 1, (
168 f"H frame says n_objects={h['n_objects']}; expected 1. "
169 "Two OC chunk entries for the same oid represent one distinct object."
170 )
171
172 def test_e_n_objects_is_one(self) -> None:
173 _, e = _h_and_e(self._objects())
174 assert e["n_objects"] == 1, (
175 f"E frame says n_objects={e['n_objects']}; expected 1. "
176 "Two OC chunk entries for the same oid represent one distinct object."
177 )
178
179
180 # ---------------------------------------------------------------------------
181 # P-OC-N3 — 1 O + 1 OC(3 chunks) = 4 entries → 2 distinct objects
182 # ---------------------------------------------------------------------------
183
184 class TestPOCN3MixedObjects:
185 def _objects(self) -> list[MsgpackDict]:
186 o_obj = _make_o_object(b"plain" * 50)
187 # Build a 3-chunk OC object manually (simpler than invoking split_object_into_oc_frames)
188 oid = _sha256_oid(b"large" * 200)
189 oc_chunks = [
190 {
191 "t": "OC",
192 "object_id": oid,
193 "chunk_index": i,
194 "total_chunks": 3,
195 "content": b"large" * 67,
196 **({"encoding": "raw", "path": "big.bin", "sz": 1000} if i == 0 else {}),
197 }
198 for i in range(3)
199 ]
200 return [o_obj] + oc_chunks # 4 entries, 2 distinct objects
201
202 def test_h_n_objects_is_two(self) -> None:
203 h, _ = _h_and_e(self._objects())
204 assert h["n_objects"] == 2
205
206 def test_e_n_objects_is_two(self) -> None:
207 _, e = _h_and_e(self._objects())
208 assert e["n_objects"] == 2
209
210
211 # ---------------------------------------------------------------------------
212 # P-OC-N4 — H.n_objects always equals E.n_objects
213 # ---------------------------------------------------------------------------
214
215 class TestPOCN4HEConsistency:
216 def _cases(self) -> list[list]:
217 return [
218 [],
219 [_make_o_object(b"a" * 100)],
220 _make_oc_chunks(b"b" * 600_000),
221 [_make_o_object(b"c" * 100)] + _make_oc_chunks(b"d" * 600_000),
222 ]
223
224 def test_h_equals_e_for_all_cases(self) -> None:
225 for objects in self._cases():
226 h, e = _h_and_e(objects)
227 assert h["n_objects"] == e["n_objects"], (
228 f"H.n_objects={h['n_objects']} != E.n_objects={e['n_objects']} "
229 f"for objects list of length {len(objects)}"
230 )
File History 1 commit
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9 docs: expand cache plan with all seven testing tiers and do… Sonnet 4.6 131 days ago