"""Alert registry / hooks. The ingestion layer must "alert" rather than silently fail on 404 / 5xx / antibot. This module defines a tiny registry of named sink handlers (log/webhook/...) and a single `fire()` entry point that the rest of the package calls. Adding a new sink (e.g. an email notifier) is a two-line change here -- no edits to the caller. Handler contract: handler(context: AlertContext) -> None context.kind : "not_found" | "transient" | "antibot" | "ok" context.level : "info" | "warning" | "error" context.message : human-readable one-liner context.source : source id ("assembly_daily_file", ...) context.date : requested date (ISO) or None context.url : the URL involved, if any context.extra : free-form dict (status, attempts, ...) """ from __future__ import annotations import logging import sys from dataclasses import dataclass, field from typing import Any, Callable, Dict, Optional log = logging.getLogger("scraper.alerts") @dataclass class AlertContext: kind: str level: str message: str source: Optional[str] = None date: Optional[str] = None url: Optional[str] = None extra: Dict[str, Any] = field(default_factory=dict) # Registry of sinks. A "sink" is any callable AlertContext -> None. LogHandler: Callable[[AlertContext], None] = lambda ctx: log.log( getattr(logging, ctx.level.upper(), logging.INFO), f"[{ctx.kind}] {ctx.message}" + (f" ({ctx.source} {ctx.date})" if ctx.source else ""), ) def log_sink(ctx: AlertContext) -> None: """The default sink: emit a structured line to stderr + the logging stream.""" print(f"[alert:{ctx.level}] {ctx.kind}: {ctx.message}", file=sys.stderr) LogHandler(ctx) def webhook_sink(ctx: AlertContext) -> None: # pragma: no cover - no live webhook import os import json url = os.environ.get("CALI_SCRAPER_WEBHOOK", "") if not url: log_sink(ctx) return import httpx httpx.post(url, json=ctx.__dict__, timeout=10, verify=False) ALERT_REGISTRY: Dict[str, Callable[[AlertContext], None]] = { "log": log_sink, "webhook": webhook_sink, } def register(name: str, handler: Callable[[AlertContext], None]) -> None: ALERT_REGISTRY[name] = handler def fire(config_alerts: Dict[str, Any], ctx: AlertContext) -> None: """Dispatch an alert through the configured sink(s). The alert may be disabled entirely via config alerts.enabled=false, but "ok" and "not_found" are always at minimum logged so the run is auditable. """ if not config_alerts.get("enabled", True) and ctx.level == "info": return sink = config_alerts.get("sink", "log") handler = ALERT_REGISTRY.get(sink, log_sink) try: handler(ctx) except Exception as exc: # an alerting failure must never crash the run log.error("alert sink '%s' failed: %s", sink, exc) log_sink(ctx)