"""HTTP layer with retry + exponential back-off and typed error handling. Design goals (from t_f595542f): * Retry transient failures with exponential back-off, capped at 3 attempts. * Distinguish 404 ("no file today", EXPECTED for non-legislative days) from 5xx (transient, retryable) from 403 (antibot challenge -> alert, don't storm). * Log every request (url, status, bytes, attempts, outcome) to the request log. * Never crash the caller on a "bad" response -- callers get typed exceptions. httpx is used because the project already depends on it (0.28.x). The follow_redirects=True argument is httpx-native. """ from __future__ import annotations import datetime import time from dataclasses import dataclass, field from typing import Any, Callable, Dict, Optional try: import httpx except ImportError as exc: # pragma: no cover raise SystemExit( "httpx is required. Install with: python3 -m pip install --user httpx" ) from exc # ---- typed error hierarchy ------------------------------------------- class FetchError(Exception): """Base class for all fetch-layer failures.""" class NotFoundError(FetchError): """404. For the Senate PDF this is EXPECTED (no publication that day).""" def __init__(self, url: str, status: int = 404): super().__init__("404 Not Found: " + url) self.url = url self.status = status class AntibotError(FetchError): """403 / antibot challenge. Surface as an alert; do NOT retry-storm it.""" def __init__(self, url: str, status: int = 403, body: bytes = b""): super().__init__("antibot %d: %s" % (status, url)) self.url = url self.status = status self.body = body class TransientError(FetchError): """5xx / network fault that survived all retry attempts.""" def __init__(self, url, status=0, attempts=0, cause=None): super().__init__( "transient after %d attempt(s) for %s: %s" % (attempts, url, status or cause) ) self.url = url self.status = status self.attempts = attempts self.cause = cause @dataclass class FetchResult: """Successful fetch result.""" url: str status: int content: bytes attempts: int = 1 cache_hit: bool = False headers: Dict[str, str] = field(default_factory=dict) outcome: str = "ok" # ok | not_found | antibot | transient def _now_iso(): return datetime.datetime.now(datetime.timezone.utc).isoformat( timespec="seconds" ) # ---- client ---------------------------------------------------------- class HttpClient: """Configurable GET client. `log_request` is a callback fired per attempt.""" def __init__( self, user_agent: str, timeout_seconds: float = 20, max_retries: int = 3, backoff_base_seconds: float = 1.0, backoff_cap_seconds: float = 30.0, retry_status=None, antibot_status=None, log_request: Optional[Callable[[Dict[str, Any]], None]] = None, sleep_fn: Optional[Callable[[float], None]] = None, ): self.user_agent = user_agent self.timeout = timeout_seconds self.max_retries = max(1, int(max_retries)) self.backoff_base = backoff_base_seconds self.backoff_cap = backoff_cap_seconds self.retry_status = set(retry_status or [500, 502, 503, 504]) self.antibot_status = set(antibot_status or [403]) self.log_request = log_request or (lambda evt: None) self.sleep = sleep_fn or time.sleep self._client: Optional[httpx.Client] = None # lifecycle @property def client(self) -> httpx.Client: if self._client is None: self._client = httpx.Client( headers={"User-Agent": self.user_agent}, timeout=self.timeout, ) return self._client def close(self) -> None: if self._client is not None: self._client.close() self._client = None def __enter__(self) -> "HttpClient": return self def __exit__(self, *exc) -> None: self.close() # ---- public API ---- def get(self, url: str, source: str) -> FetchResult: """GET with retry + back-off. Returns a FetchResult on success, or raises a typed FetchError subclass: NotFoundError -> 404 (never retried) AntibotError -> 403 / antibot (alert; no retry-storm) TransientError-> 5xx / network fault after all attempts Emits one request-log event per attempt. """ self.last_attempts = 0 last_status = 0 last_exc: Optional[Exception] = None last_content = b"" for attempt in range(1, self.max_retries + 1): self.last_attempts = attempt outcome, status, content, headers, exc = self._do(url) last_status = status last_exc = exc last_content = content or b"" # transient (5xx in retry list, or a network error): retry or give up if outcome == "transient": if attempt < self.max_retries: self._log( source, url, status, len(last_content), attempt, "retrying" if exc else "5xx_retrying", ) delay = min( self.backoff_base * (2 ** (attempt - 1)), self.backoff_cap, ) self.sleep(delay) continue self._log( source, url, status, len(last_content), attempt, "transient_gave_up", ) raise TransientError(url, status, attempt, exc) # success on 2xx/3xx (redirects are fully resolved by httpx) if outcome == "ok": self._log(source, url, status, len(content), attempt, "ok") return FetchResult( url=url, status=status, content=content, attempts=attempt, headers=headers, outcome="ok", ) # 404 -> definitive "no file"; never retried if outcome == "not_found": self._log(source, url, status, len(content), attempt, "not_found") raise NotFoundError(url, status) # antibot challenge: alert, do not retry-storm if outcome == "antibot": self._log(source, url, status, len(content), attempt, "antibot") raise AntibotError(url, status, content) raise TransientError(url, last_status, self.max_retries, last_exc) def _do(self, url: str): """Perform one fetch. Returns (outcome, status, content, headers, exc).""" try: resp = self.client.get(url, follow_redirects=True) except httpx.HTTPError as exc: return "transient", 0, b"", {}, exc status = resp.status_code content = resp.content headers = { k: v for k, v in resp.headers.items() if k.lower() in ("content-type", "etag", "last-modified") } if status < 400: return "ok", status, content, headers, None if status == 404: return "not_found", status, content, headers, None if status in self.antibot_status: return "antibot", status, content, headers, None if status in self.retry_status: return "transient", status, content, headers, None # unexpected 4xx (not 404/antibot): surface it via the transient path so the # alerting hook can classify the exact status; don't retry-storm. return "transient", status, content, headers, None # ---- helpers ---- def _log(self, source, url, status, nbytes, attempt, outcome) -> None: self.log_request( { "ts": _now_iso(), "source": source, "url": url, "status": status, "bytes": nbytes, "attempt": attempt, "outcome": outcome, } )