238 lines
8.1 KiB
Python
238 lines
8.1 KiB
Python
"""HTTP layer with retry + exponential back-off and typed error handling.
|
|
|
|
Design goals (from t_f595542f):
|
|
* Retry transient failures with exponential back-off, capped at 3 attempts.
|
|
* Distinguish 404 ("no file today", EXPECTED for non-legislative days) from
|
|
5xx (transient, retryable) from 403 (antibot challenge -> alert, don't storm).
|
|
* Log every request (url, status, bytes, attempts, outcome) to the request log.
|
|
* Never crash the caller on a "bad" response -- callers get typed exceptions.
|
|
|
|
httpx is used because the project already depends on it (0.28.x). The
|
|
follow_redirects=True argument is httpx-native.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import datetime
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
from typing import Any, Callable, Dict, Optional
|
|
|
|
try:
|
|
import httpx
|
|
except ImportError as exc: # pragma: no cover
|
|
raise SystemExit(
|
|
"httpx is required. Install with: python3 -m pip install --user httpx"
|
|
) from exc
|
|
|
|
|
|
# ---- typed error hierarchy -------------------------------------------
|
|
class FetchError(Exception):
|
|
"""Base class for all fetch-layer failures."""
|
|
|
|
|
|
class NotFoundError(FetchError):
|
|
"""404. For the Senate PDF this is EXPECTED (no publication that day)."""
|
|
|
|
def __init__(self, url: str, status: int = 404):
|
|
super().__init__("404 Not Found: " + url)
|
|
self.url = url
|
|
self.status = status
|
|
|
|
|
|
class AntibotError(FetchError):
|
|
"""403 / antibot challenge. Surface as an alert; do NOT retry-storm it."""
|
|
|
|
def __init__(self, url: str, status: int = 403, body: bytes = b""):
|
|
super().__init__("antibot %d: %s" % (status, url))
|
|
self.url = url
|
|
self.status = status
|
|
self.body = body
|
|
|
|
|
|
class TransientError(FetchError):
|
|
"""5xx / network fault that survived all retry attempts."""
|
|
|
|
def __init__(self, url, status=0, attempts=0, cause=None):
|
|
super().__init__(
|
|
"transient after %d attempt(s) for %s: %s"
|
|
% (attempts, url, status or cause)
|
|
)
|
|
self.url = url
|
|
self.status = status
|
|
self.attempts = attempts
|
|
self.cause = cause
|
|
|
|
|
|
@dataclass
|
|
class FetchResult:
|
|
"""Successful fetch result."""
|
|
|
|
url: str
|
|
status: int
|
|
content: bytes
|
|
attempts: int = 1
|
|
cache_hit: bool = False
|
|
headers: Dict[str, str] = field(default_factory=dict)
|
|
outcome: str = "ok" # ok | not_found | antibot | transient
|
|
|
|
|
|
def _now_iso():
|
|
return datetime.datetime.now(datetime.timezone.utc).isoformat(
|
|
timespec="seconds"
|
|
)
|
|
|
|
|
|
# ---- client ----------------------------------------------------------
|
|
class HttpClient:
|
|
"""Configurable GET client. `log_request` is a callback fired per attempt."""
|
|
|
|
def __init__(
|
|
self,
|
|
user_agent: str,
|
|
timeout_seconds: float = 20,
|
|
max_retries: int = 3,
|
|
backoff_base_seconds: float = 1.0,
|
|
backoff_cap_seconds: float = 30.0,
|
|
retry_status=None,
|
|
antibot_status=None,
|
|
log_request: Optional[Callable[[Dict[str, Any]], None]] = None,
|
|
sleep_fn: Optional[Callable[[float], None]] = None,
|
|
):
|
|
self.user_agent = user_agent
|
|
self.timeout = timeout_seconds
|
|
self.max_retries = max(1, int(max_retries))
|
|
self.backoff_base = backoff_base_seconds
|
|
self.backoff_cap = backoff_cap_seconds
|
|
self.retry_status = set(retry_status or [500, 502, 503, 504])
|
|
self.antibot_status = set(antibot_status or [403])
|
|
self.log_request = log_request or (lambda evt: None)
|
|
self.sleep = sleep_fn or time.sleep
|
|
self._client: Optional[httpx.Client] = None
|
|
|
|
# lifecycle
|
|
@property
|
|
def client(self) -> httpx.Client:
|
|
if self._client is None:
|
|
self._client = httpx.Client(
|
|
headers={"User-Agent": self.user_agent},
|
|
timeout=self.timeout,
|
|
)
|
|
return self._client
|
|
|
|
def close(self) -> None:
|
|
if self._client is not None:
|
|
self._client.close()
|
|
self._client = None
|
|
|
|
def __enter__(self) -> "HttpClient":
|
|
return self
|
|
|
|
def __exit__(self, *exc) -> None:
|
|
self.close()
|
|
|
|
# ---- public API ----
|
|
def get(self, url: str, source: str) -> FetchResult:
|
|
"""GET with retry + back-off.
|
|
|
|
Returns a FetchResult on success, or raises a typed FetchError subclass:
|
|
NotFoundError -> 404 (never retried)
|
|
AntibotError -> 403 / antibot (alert; no retry-storm)
|
|
TransientError-> 5xx / network fault after all attempts
|
|
|
|
Emits one request-log event per attempt.
|
|
"""
|
|
self.last_attempts = 0
|
|
last_status = 0
|
|
last_exc: Optional[Exception] = None
|
|
last_content = b""
|
|
for attempt in range(1, self.max_retries + 1):
|
|
self.last_attempts = attempt
|
|
outcome, status, content, headers, exc = self._do(url)
|
|
last_status = status
|
|
last_exc = exc
|
|
last_content = content or b""
|
|
|
|
# transient (5xx in retry list, or a network error): retry or give up
|
|
if outcome == "transient":
|
|
if attempt < self.max_retries:
|
|
self._log(
|
|
source, url, status, len(last_content), attempt,
|
|
"retrying" if exc else "5xx_retrying",
|
|
)
|
|
delay = min(
|
|
self.backoff_base * (2 ** (attempt - 1)),
|
|
self.backoff_cap,
|
|
)
|
|
self.sleep(delay)
|
|
continue
|
|
self._log(
|
|
source, url, status, len(last_content), attempt,
|
|
"transient_gave_up",
|
|
)
|
|
raise TransientError(url, status, attempt, exc)
|
|
|
|
# success on 2xx/3xx (redirects are fully resolved by httpx)
|
|
if outcome == "ok":
|
|
self._log(source, url, status, len(content), attempt, "ok")
|
|
return FetchResult(
|
|
url=url,
|
|
status=status,
|
|
content=content,
|
|
attempts=attempt,
|
|
headers=headers,
|
|
outcome="ok",
|
|
)
|
|
|
|
# 404 -> definitive "no file"; never retried
|
|
if outcome == "not_found":
|
|
self._log(source, url, status, len(content), attempt, "not_found")
|
|
raise NotFoundError(url, status)
|
|
|
|
# antibot challenge: alert, do not retry-storm
|
|
if outcome == "antibot":
|
|
self._log(source, url, status, len(content), attempt, "antibot")
|
|
raise AntibotError(url, status, content)
|
|
|
|
raise TransientError(url, last_status, self.max_retries, last_exc)
|
|
|
|
def _do(self, url: str):
|
|
"""Perform one fetch. Returns (outcome, status, content, headers, exc)."""
|
|
try:
|
|
resp = self.client.get(url, follow_redirects=True)
|
|
except httpx.HTTPError as exc:
|
|
return "transient", 0, b"", {}, exc
|
|
|
|
status = resp.status_code
|
|
content = resp.content
|
|
headers = {
|
|
k: v
|
|
for k, v in resp.headers.items()
|
|
if k.lower() in ("content-type", "etag", "last-modified")
|
|
}
|
|
if status < 400:
|
|
return "ok", status, content, headers, None
|
|
if status == 404:
|
|
return "not_found", status, content, headers, None
|
|
if status in self.antibot_status:
|
|
return "antibot", status, content, headers, None
|
|
if status in self.retry_status:
|
|
return "transient", status, content, headers, None
|
|
# unexpected 4xx (not 404/antibot): surface it via the transient path so the
|
|
# alerting hook can classify the exact status; don't retry-storm.
|
|
return "transient", status, content, headers, None
|
|
|
|
# ---- helpers ----
|
|
def _log(self, source, url, status, nbytes, attempt, outcome) -> None:
|
|
self.log_request(
|
|
{
|
|
"ts": _now_iso(),
|
|
"source": source,
|
|
"url": url,
|
|
"status": status,
|
|
"bytes": nbytes,
|
|
"attempt": attempt,
|
|
"outcome": outcome,
|
|
}
|
|
)
|