gabriel / musehub public
musehub_webhook_dispatcher.py python
569 lines 19.0 KB
Raw
sha256:42bc547653458bb15e283222247af0237872d9cb5c2a72ad2ca1c111ed2c0ca4 feat(9A-4 F7): overseer-run provenance enrichment API for S… Human minor ⚠ breaking 74 days ago
1 """MuseHub webhook dispatcher — event-driven HTTP notification delivery.
2
3 This module is the single point responsible for delivering webhook events to
4 registered subscriber URLs. It is called by route handlers after a state-
5 changing operation completes (push, issue create/close, proposal create/merge, etc.).
6
7 Delivery contract:
8 - HTTP POST to the subscriber's ``url`` with a JSON payload.
9 - ``Content-Type: application/json``
10 - ``X-MuseHub-Event: <event_type>`` header identifying the event.
11 - ``X-MuseHub-Delivery: <delivery_id>`` header for idempotency.
12 - ``X-MuseHub-Signature: sha256=<hmac_hex>`` header when ``secret`` is set.
13 - Retry policy: up to 3 attempts with exponential back-off (1 s, 2 s, 4 s).
14 - Each attempt is logged as a separate ``MusehubWebhookDelivery`` row.
15
16 Boundary rules (same as all musehub services):
17 - Must NOT import state stores, SSE queues, or LLM clients.
18 - May import ORM models from musehub.db domain-specific modules.
19 - May import Pydantic models from musehub.models.musehub.
20 """
21
22 import asyncio
23 import hashlib
24 import hmac
25 import json
26 import logging
27 from datetime import datetime, timezone
28
29 import httpx
30 from sqlalchemy import func, select
31 from sqlalchemy.ext.asyncio import AsyncSession
32
33 from musehub.core.genesis import compute_webhook_delivery_id, compute_webhook_id
34 from musehub.db.musehub_webhook_models import MusehubWebhook, MusehubWebhookDelivery
35 from musehub.db.database import AsyncSessionLocal
36 from musehub.types.json_types import StrDict
37 from musehub.models.musehub import (
38 WebhookDeliveryListResponse,
39 WebhookDeliveryResponse,
40 WebhookEventPayload,
41 WebhookListResponse,
42 WebhookRedeliverResponse,
43 WebhookResponse,
44 )
45 from musehub.services.musehub_webhook_crypto import decrypt_secret, encrypt_secret
46
47 logger = logging.getLogger(__name__)
48
49 # Maximum delivery attempts per dispatch call (initial + 2 retries).
50 _MAX_ATTEMPTS = 3
51 # Base back-off in seconds; doubled on each retry.
52 _BACKOFF_BASE = 1.0
53 # HTTP timeout per outbound POST attempt.
54 _REQUEST_TIMEOUT = 10.0
55
56
57 def _utc_now() -> datetime:
58 return datetime.now(tz=timezone.utc)
59
60
61
62 def _sign_payload(secret: str, body: bytes) -> str:
63 """Compute HMAC-SHA256 signature for ``body`` using ``secret``.
64
65 Returns the hex digest prefixed with ``sha256=``, matching GitHub's
66 webhook signature convention so existing verification libraries work.
67 """
68 mac = hmac.new(secret.encode(), body, hashlib.sha256)
69 return f"sha256={mac.hexdigest()}"
70
71
72 def _to_webhook_response(row: MusehubWebhook) -> WebhookResponse:
73 return WebhookResponse(
74 webhook_id=row.webhook_id,
75 repo_id=row.repo_id,
76 url=row.url,
77 events=list(row.events or []),
78 active=row.active,
79 created_at=row.created_at,
80 )
81
82
83 def _to_delivery_response(row: MusehubWebhookDelivery) -> WebhookDeliveryResponse:
84 return WebhookDeliveryResponse(
85 delivery_id=row.delivery_id,
86 webhook_id=row.webhook_id,
87 event_type=row.event_type,
88 payload=row.payload,
89 attempt=row.attempt,
90 success=row.success,
91 response_status=row.response_status,
92 response_body=row.response_body,
93 delivered_at=row.delivered_at,
94 )
95
96
97 # ---------------------------------------------------------------------------
98 # Webhook CRUD
99 # ---------------------------------------------------------------------------
100
101
102 async def create_webhook(
103 session: AsyncSession,
104 *,
105 repo_id: str,
106 url: str,
107 events: list[str],
108 secret: str,
109 ) -> WebhookResponse:
110 """Persist a new webhook subscription and return its wire representation.
111
112 The webhook is created in active state. The caller must commit the session.
113 """
114 _wh_now = datetime.now(timezone.utc)
115 webhook = MusehubWebhook(
116 webhook_id=compute_webhook_id(repo_id, url, _wh_now.isoformat()),
117 repo_id=repo_id,
118 url=url,
119 events=events,
120 secret=encrypt_secret(secret),
121 active=True,
122 created_at=_wh_now,
123 updated_at=_wh_now,
124 )
125 session.add(webhook)
126 await session.flush()
127 await session.refresh(webhook)
128 logger.info("✅ Registered webhook %s for repo %s → %s", webhook.webhook_id, repo_id, url)
129 return _to_webhook_response(webhook)
130
131
132 async def list_webhooks(
133 session: AsyncSession,
134 repo_id: str,
135 cursor: str | None = None,
136 limit: int = 20,
137 ) -> WebhookListResponse:
138 """Return webhook subscriptions for a repo with cursor-based keyset pagination.
139
140 Results are ordered by ``created_at`` ascending.
141
142 ``cursor`` is the ISO 8601 ``created_at`` of the last seen webhook
143 (opaque to callers — pass ``nextCursor`` from a previous response
144 verbatim). Omit to start from the beginning.
145 """
146 conditions = [MusehubWebhook.repo_id == repo_id]
147
148 count_stmt = select(func.count(MusehubWebhook.webhook_id)).where(*conditions)
149 total: int = (await session.execute(count_stmt)).scalar_one()
150
151 data_conditions = list(conditions)
152 if cursor is not None:
153 data_conditions.append(
154 MusehubWebhook.created_at > datetime.fromisoformat(cursor)
155 )
156
157 rows = list(
158 (
159 await session.execute(
160 select(MusehubWebhook)
161 .where(*data_conditions)
162 .order_by(MusehubWebhook.created_at)
163 .limit(limit + 1)
164 )
165 ).scalars()
166 )
167
168 next_cursor: str | None = None
169 if len(rows) == limit + 1:
170 next_cursor = rows[limit - 1].created_at.isoformat()
171 rows = rows[:limit]
172
173 return WebhookListResponse(
174 webhooks=[_to_webhook_response(r) for r in rows],
175 total=total,
176 next_cursor=next_cursor,
177 )
178
179
180 async def get_webhook(
181 session: AsyncSession,
182 repo_id: str,
183 webhook_id: str,
184 ) -> WebhookResponse | None:
185 """Return a single webhook by ID, or None if not found in this repo."""
186 stmt = select(MusehubWebhook).where(
187 MusehubWebhook.repo_id == repo_id,
188 MusehubWebhook.webhook_id == webhook_id,
189 )
190 row = (await session.execute(stmt)).scalar_one_or_none()
191 if row is None:
192 return None
193 return _to_webhook_response(row)
194
195
196 async def delete_webhook(
197 session: AsyncSession,
198 repo_id: str,
199 webhook_id: str,
200 ) -> bool:
201 """Delete a webhook by ID. Returns True if deleted, False if not found.
202
203 The caller must commit the session after a True result.
204 """
205 stmt = select(MusehubWebhook).where(
206 MusehubWebhook.repo_id == repo_id,
207 MusehubWebhook.webhook_id == webhook_id,
208 )
209 row = (await session.execute(stmt)).scalar_one_or_none()
210 if row is None:
211 return False
212 await session.delete(row)
213 await session.flush()
214 logger.info("✅ Deleted webhook %s from repo %s", webhook_id, repo_id)
215 return True
216
217
218 async def list_deliveries(
219 session: AsyncSession,
220 webhook_id: str,
221 cursor: str | None = None,
222 limit: int = 50,
223 ) -> WebhookDeliveryListResponse:
224 """Return delivery history for a webhook with cursor-based keyset pagination.
225
226 Results are ordered by ``delivered_at`` descending (newest first).
227
228 ``cursor`` is the ISO 8601 ``delivered_at`` of the last seen delivery
229 (opaque to callers — pass ``nextCursor`` from a previous response
230 verbatim). Omit to start from the most recent delivery.
231 """
232 conditions = [MusehubWebhookDelivery.webhook_id == webhook_id]
233
234 count_stmt = select(func.count(MusehubWebhookDelivery.delivery_id)).where(*conditions)
235 total: int = (await session.execute(count_stmt)).scalar_one()
236
237 data_conditions = list(conditions)
238 if cursor is not None:
239 data_conditions.append(
240 MusehubWebhookDelivery.delivered_at < datetime.fromisoformat(cursor)
241 )
242
243 rows = list(
244 (
245 await session.execute(
246 select(MusehubWebhookDelivery)
247 .where(*data_conditions)
248 .order_by(MusehubWebhookDelivery.delivered_at.desc())
249 .limit(limit + 1)
250 )
251 ).scalars()
252 )
253
254 next_cursor: str | None = None
255 if len(rows) == limit + 1:
256 next_cursor = rows[limit - 1].delivered_at.isoformat()
257 rows = rows[:limit]
258
259 return WebhookDeliveryListResponse(
260 deliveries=[_to_delivery_response(r) for r in rows],
261 total=total,
262 next_cursor=next_cursor,
263 )
264
265
266 async def get_delivery(
267 session: AsyncSession,
268 webhook_id: str,
269 delivery_id: str,
270 ) -> WebhookDeliveryResponse | None:
271 """Return a single delivery record by ID, or None if not found for this webhook."""
272 stmt = select(MusehubWebhookDelivery).where(
273 MusehubWebhookDelivery.webhook_id == webhook_id,
274 MusehubWebhookDelivery.delivery_id == delivery_id,
275 )
276 row = (await session.execute(stmt)).scalar_one_or_none()
277 if row is None:
278 return None
279 return _to_delivery_response(row)
280
281
282 async def redeliver_delivery(
283 session: AsyncSession,
284 repo_id: str,
285 webhook_id: str,
286 delivery_id: str,
287 ) -> WebhookRedeliverResponse:
288 """Retry a single past delivery attempt using its stored payload.
289
290 Fetches the original delivery row to recover the event type and payload,
291 then executes one new delivery attempt (with full retry policy) against
292 the webhook's current URL. Each retry attempt is persisted as a new
293 ``MusehubWebhookDelivery`` row — the original row is never mutated.
294
295 Raises ``ValueError`` when the delivery or webhook cannot be found, or
296 when the stored payload is empty (delivery predates payload storage).
297 The caller must commit the session after a successful return.
298 """
299 delivery_stmt = select(MusehubWebhookDelivery).where(
300 MusehubWebhookDelivery.webhook_id == webhook_id,
301 MusehubWebhookDelivery.delivery_id == delivery_id,
302 )
303 delivery_row = (await session.execute(delivery_stmt)).scalar_one_or_none()
304 if delivery_row is None:
305 raise ValueError(f"Delivery {delivery_id!r} not found for webhook {webhook_id!r}")
306
307 if not delivery_row.payload:
308 raise ValueError(
309 f"Delivery {delivery_id!r} has no stored payload"
310 "it predates payload storage and cannot be redelivered"
311 )
312
313 webhook_stmt = select(MusehubWebhook).where(
314 MusehubWebhook.webhook_id == webhook_id,
315 MusehubWebhook.repo_id == repo_id,
316 )
317 webhook_row = (await session.execute(webhook_stmt)).scalar_one_or_none()
318 if webhook_row is None:
319 raise ValueError(f"Webhook {webhook_id!r} not found for repo {repo_id!r}")
320
321 payload_bytes = delivery_row.payload.encode()
322 event_type = delivery_row.event_type
323 final_success = False
324 final_status = 0
325 final_body = ""
326
327 async with httpx.AsyncClient() as client:
328 for attempt in range(1, _MAX_ATTEMPTS + 1):
329 delivered_at = _utc_now()
330 new_delivery_id = compute_webhook_delivery_id(webhook_id, event_type, attempt, delivered_at.isoformat())
331 success, status_code, response_body = await _attempt_delivery(
332 client,
333 webhook=webhook_row,
334 event_type=event_type,
335 payload_bytes=payload_bytes,
336 delivery_id=new_delivery_id,
337 attempt=attempt,
338 )
339 new_row = MusehubWebhookDelivery(
340 delivery_id=new_delivery_id,
341 webhook_id=webhook_id,
342 event_type=event_type,
343 payload=delivery_row.payload,
344 attempt=attempt,
345 success=success,
346 response_status=status_code,
347 response_body=response_body,
348 delivered_at=delivered_at,
349 )
350 session.add(new_row)
351 await session.flush()
352
353 final_success = success
354 final_status = status_code
355 final_body = response_body
356
357 if success:
358 logger.info(
359 "✅ Redelivery of %s (webhook %s) succeeded on attempt %d (status %d)",
360 delivery_id,
361 webhook_id,
362 attempt,
363 status_code,
364 )
365 break
366
367 if attempt < _MAX_ATTEMPTS:
368 backoff = _BACKOFF_BASE * (2 ** (attempt - 1))
369 logger.warning(
370 "⚠️ Redelivery of %s attempt %d failed (status %d) — retrying in %.1fs",
371 delivery_id,
372 attempt,
373 status_code,
374 backoff,
375 )
376 await asyncio.sleep(backoff)
377 else:
378 logger.error(
379 "❌ Redelivery of %s failed after %d attempts (last status %d)",
380 delivery_id,
381 _MAX_ATTEMPTS,
382 status_code,
383 )
384
385 return WebhookRedeliverResponse(
386 original_delivery_id=delivery_id,
387 webhook_id=webhook_id,
388 event_type=event_type,
389 success=final_success,
390 response_status=final_status,
391 response_body=final_body,
392 )
393
394
395 # ---------------------------------------------------------------------------
396 # Dispatch
397 # ---------------------------------------------------------------------------
398
399
400 async def _attempt_delivery(
401 client: httpx.AsyncClient,
402 *,
403 webhook: MusehubWebhook,
404 event_type: str,
405 payload_bytes: bytes,
406 delivery_id: str,
407 attempt: int,
408 ) -> tuple[bool, int, str]:
409 """Execute one HTTP POST attempt and return (success, status_code, body_snippet).
410
411 Returns (False, 0, error_message) when the request fails at the transport
412 layer (timeout, DNS failure, connection refused).
413 """
414 headers: StrDict = {
415 "Content-Type": "application/json",
416 "X-MuseHub-Event": event_type,
417 "X-MuseHub-Delivery": delivery_id,
418 "User-Agent": "MuseHub-Webhook/1.0",
419 }
420 if webhook.secret:
421 plaintext_secret = decrypt_secret(webhook.secret)
422 headers["X-MuseHub-Signature"] = _sign_payload(plaintext_secret, payload_bytes)
423
424 # Defence-in-depth SSRF check with DNS resolution — guards against
425 # DNS rebinding between registration time and delivery time.
426 try:
427 from musehub.security.ssrf import validate_outbound_url
428 await validate_outbound_url(webhook.url)
429 except ValueError as ssrf_exc:
430 return False, 0, f"SSRF blocked: {ssrf_exc}"
431
432 try:
433 resp = await client.post(
434 webhook.url,
435 content=payload_bytes,
436 headers=headers,
437 timeout=_REQUEST_TIMEOUT,
438 )
439 success = resp.is_success
440 return success, resp.status_code, resp.text[:512]
441 except httpx.TimeoutException as exc:
442 return False, 0, f"timeout: {exc}"
443 except httpx.RequestError as exc:
444 return False, 0, f"request error: {exc}"
445
446
447 async def dispatch_event(
448 session: AsyncSession,
449 *,
450 repo_id: str,
451 event_type: str,
452 payload: WebhookEventPayload,
453 ) -> None:
454 """Dispatch a webhook event to all active subscribers for ``repo_id``.
455
456 Called by route handlers after a state-changing operation. Does NOT block
457 the caller's HTTP response — this function handles its own retries internally
458 and logs each attempt to ``musehub_webhook_deliveries``.
459
460 The ``payload`` dict is serialised to camelCase JSON before delivery. It
461 should use snake_case keys; the serialiser converts them automatically via
462 the Pydantic CamelModel aliases so that wire format is consistent with the
463 rest of the MuseHub API.
464
465 Delivery is best-effort: failures are logged but never raised.
466 """
467 stmt = select(MusehubWebhook).where(
468 MusehubWebhook.repo_id == repo_id,
469 MusehubWebhook.active.is_(True),
470 )
471 webhooks = (await session.execute(stmt)).scalars().all()
472
473 active = [w for w in webhooks if event_type in (w.events or [])]
474 if not active:
475 return
476
477 payload_bytes = json.dumps(payload, default=str).encode()
478
479 async with httpx.AsyncClient() as client:
480 for webhook in active:
481 success = False
482 status_code = 0
483 response_body = ""
484
485 for attempt in range(1, _MAX_ATTEMPTS + 1):
486 delivered_at = _utc_now()
487 delivery_id = compute_webhook_delivery_id(webhook.webhook_id, event_type, attempt, delivered_at.isoformat())
488 success, status_code, response_body = await _attempt_delivery(
489 client,
490 webhook=webhook,
491 event_type=event_type,
492 payload_bytes=payload_bytes,
493 delivery_id=delivery_id,
494 attempt=attempt,
495 )
496
497 delivery_row = MusehubWebhookDelivery(
498 delivery_id=delivery_id,
499 webhook_id=webhook.webhook_id,
500 event_type=event_type,
501 payload=payload_bytes.decode(),
502 attempt=attempt,
503 success=success,
504 response_status=status_code,
505 response_body=response_body,
506 delivered_at=delivered_at,
507 )
508 session.add(delivery_row)
509 await session.flush()
510
511 if success:
512 logger.info(
513 "✅ Webhook %s delivered '%s' on attempt %d (status %d)",
514 webhook.webhook_id,
515 event_type,
516 attempt,
517 status_code,
518 )
519 break
520
521 if attempt < _MAX_ATTEMPTS:
522 backoff = _BACKOFF_BASE * (2 ** (attempt - 1))
523 logger.warning(
524 "⚠️ Webhook %s attempt %d failed (status %d) — retrying in %.1fs",
525 webhook.webhook_id,
526 attempt,
527 status_code,
528 backoff,
529 )
530 await asyncio.sleep(backoff)
531 else:
532 logger.error(
533 "❌ Webhook %s delivery failed after %d attempts (last status %d)",
534 webhook.webhook_id,
535 _MAX_ATTEMPTS,
536 status_code,
537 )
538
539
540 async def dispatch_event_background(
541 repo_id: str,
542 event_type: str,
543 payload: WebhookEventPayload,
544 ) -> None:
545 """Fire-and-forget webhook dispatch that manages its own DB session.
546
547 Intended for use with FastAPI ``BackgroundTasks`` so that webhook delivery
548 does not block the HTTP response. Errors are logged but never re-raised.
549
550 Usage in a route handler::
551
552 background_tasks.add_task(
553 dispatch_event_background,
554 repo_id=repo_id,
555 event_type="push",
556 payload={"repoId": repo_id, "branch": branch, ...},
557 )
558 """
559 try:
560 async with AsyncSessionLocal() as session:
561 await dispatch_event(session, repo_id=repo_id, event_type=event_type, payload=payload)
562 await session.commit()
563 except Exception as exc:
564 logger.error(
565 "❌ Background webhook dispatch failed for repo %s event '%s': %s",
566 repo_id,
567 event_type,
568 exc,
569 )
File History 1 commit
sha256:42bc547653458bb15e283222247af0237872d9cb5c2a72ad2ca1c111ed2c0ca4 feat(9A-4 F7): overseer-run provenance enrichment API for S… Human minor 74 days ago