Initial commit
This commit is contained in:
kfj001 committed
commit
5f2db08b43
100 files changed
+16985
No files matched your search
@@ -0,0 +1,237 @@
|
||||
"""HTTP layer with retry + exponential back-off and typed error handling.
|
||||
|
||||
Design goals (from t_f595542f):
|
||||
* Retry transient failures with exponential back-off, capped at 3 attempts.
|
||||
* Distinguish 404 ("no file today", EXPECTED for non-legislative days) from
|
||||
5xx (transient, retryable) from 403 (antibot challenge -> alert, don't storm).
|
||||
* Log every request (url, status, bytes, attempts, outcome) to the request log.
|
||||
* Never crash the caller on a "bad" response -- callers get typed exceptions.
|
||||
|
||||
httpx is used because the project already depends on it (0.28.x). The
|
||||
follow_redirects=True argument is httpx-native.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
|
||||
try:
|
||||
import httpx
|
||||
except ImportError as exc: # pragma: no cover
|
||||
raise SystemExit(
|
||||
"httpx is required. Install with: python3 -m pip install --user httpx"
|
||||
) from exc
|
||||
|
||||
|
||||
# ---- typed error hierarchy -------------------------------------------
|
||||
class FetchError(Exception):
|
||||
"""Base class for all fetch-layer failures."""
|
||||
|
||||
|
||||
class NotFoundError(FetchError):
|
||||
"""404. For the Senate PDF this is EXPECTED (no publication that day)."""
|
||||
|
||||
def __init__(self, url: str, status: int = 404):
|
||||
super().__init__("404 Not Found: " + url)
|
||||
self.url = url
|
||||
self.status = status
|
||||
|
||||
|
||||
class AntibotError(FetchError):
|
||||
"""403 / antibot challenge. Surface as an alert; do NOT retry-storm it."""
|
||||
|
||||
def __init__(self, url: str, status: int = 403, body: bytes = b""):
|
||||
super().__init__("antibot %d: %s" % (status, url))
|
||||
self.url = url
|
||||
self.status = status
|
||||
self.body = body
|
||||
|
||||
|
||||
class TransientError(FetchError):
|
||||
"""5xx / network fault that survived all retry attempts."""
|
||||
|
||||
def __init__(self, url, status=0, attempts=0, cause=None):
|
||||
super().__init__(
|
||||
"transient after %d attempt(s) for %s: %s"
|
||||
% (attempts, url, status or cause)
|
||||
)
|
||||
self.url = url
|
||||
self.status = status
|
||||
self.attempts = attempts
|
||||
self.cause = cause
|
||||
|
||||
|
||||
@dataclass
|
||||
class FetchResult:
|
||||
"""Successful fetch result."""
|
||||
|
||||
url: str
|
||||
status: int
|
||||
content: bytes
|
||||
attempts: int = 1
|
||||
cache_hit: bool = False
|
||||
headers: Dict[str, str] = field(default_factory=dict)
|
||||
outcome: str = "ok" # ok | not_found | antibot | transient
|
||||
|
||||
|
||||
def _now_iso():
|
||||
return datetime.datetime.now(datetime.timezone.utc).isoformat(
|
||||
timespec="seconds"
|
||||
)
|
||||
|
||||
|
||||
# ---- client ----------------------------------------------------------
|
||||
class HttpClient:
|
||||
"""Configurable GET client. `log_request` is a callback fired per attempt."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
user_agent: str,
|
||||
timeout_seconds: float = 20,
|
||||
max_retries: int = 3,
|
||||
backoff_base_seconds: float = 1.0,
|
||||
backoff_cap_seconds: float = 30.0,
|
||||
retry_status=None,
|
||||
antibot_status=None,
|
||||
log_request: Optional[Callable[[Dict[str, Any]], None]] = None,
|
||||
sleep_fn: Optional[Callable[[float], None]] = None,
|
||||
):
|
||||
self.user_agent = user_agent
|
||||
self.timeout = timeout_seconds
|
||||
self.max_retries = max(1, int(max_retries))
|
||||
self.backoff_base = backoff_base_seconds
|
||||
self.backoff_cap = backoff_cap_seconds
|
||||
self.retry_status = set(retry_status or [500, 502, 503, 504])
|
||||
self.antibot_status = set(antibot_status or [403])
|
||||
self.log_request = log_request or (lambda evt: None)
|
||||
self.sleep = sleep_fn or time.sleep
|
||||
self._client: Optional[httpx.Client] = None
|
||||
|
||||
# lifecycle
|
||||
@property
|
||||
def client(self) -> httpx.Client:
|
||||
if self._client is None:
|
||||
self._client = httpx.Client(
|
||||
headers={"User-Agent": self.user_agent},
|
||||
timeout=self.timeout,
|
||||
)
|
||||
return self._client
|
||||
|
||||
def close(self) -> None:
|
||||
if self._client is not None:
|
||||
self._client.close()
|
||||
self._client = None
|
||||
|
||||
def __enter__(self) -> "HttpClient":
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc) -> None:
|
||||
self.close()
|
||||
|
||||
# ---- public API ----
|
||||
def get(self, url: str, source: str) -> FetchResult:
|
||||
"""GET with retry + back-off.
|
||||
|
||||
Returns a FetchResult on success, or raises a typed FetchError subclass:
|
||||
NotFoundError -> 404 (never retried)
|
||||
AntibotError -> 403 / antibot (alert; no retry-storm)
|
||||
TransientError-> 5xx / network fault after all attempts
|
||||
|
||||
Emits one request-log event per attempt.
|
||||
"""
|
||||
self.last_attempts = 0
|
||||
last_status = 0
|
||||
last_exc: Optional[Exception] = None
|
||||
last_content = b""
|
||||
for attempt in range(1, self.max_retries + 1):
|
||||
self.last_attempts = attempt
|
||||
outcome, status, content, headers, exc = self._do(url)
|
||||
last_status = status
|
||||
last_exc = exc
|
||||
last_content = content or b""
|
||||
|
||||
# transient (5xx in retry list, or a network error): retry or give up
|
||||
if outcome == "transient":
|
||||
if attempt < self.max_retries:
|
||||
self._log(
|
||||
source, url, status, len(last_content), attempt,
|
||||
"retrying" if exc else "5xx_retrying",
|
||||
)
|
||||
delay = min(
|
||||
self.backoff_base * (2 ** (attempt - 1)),
|
||||
self.backoff_cap,
|
||||
)
|
||||
self.sleep(delay)
|
||||
continue
|
||||
self._log(
|
||||
source, url, status, len(last_content), attempt,
|
||||
"transient_gave_up",
|
||||
)
|
||||
raise TransientError(url, status, attempt, exc)
|
||||
|
||||
# success on 2xx/3xx (redirects are fully resolved by httpx)
|
||||
if outcome == "ok":
|
||||
self._log(source, url, status, len(content), attempt, "ok")
|
||||
return FetchResult(
|
||||
url=url,
|
||||
status=status,
|
||||
content=content,
|
||||
attempts=attempt,
|
||||
headers=headers,
|
||||
outcome="ok",
|
||||
)
|
||||
|
||||
# 404 -> definitive "no file"; never retried
|
||||
if outcome == "not_found":
|
||||
self._log(source, url, status, len(content), attempt, "not_found")
|
||||
raise NotFoundError(url, status)
|
||||
|
||||
# antibot challenge: alert, do not retry-storm
|
||||
if outcome == "antibot":
|
||||
self._log(source, url, status, len(content), attempt, "antibot")
|
||||
raise AntibotError(url, status, content)
|
||||
|
||||
raise TransientError(url, last_status, self.max_retries, last_exc)
|
||||
|
||||
def _do(self, url: str):
|
||||
"""Perform one fetch. Returns (outcome, status, content, headers, exc)."""
|
||||
try:
|
||||
resp = self.client.get(url, follow_redirects=True)
|
||||
except httpx.HTTPError as exc:
|
||||
return "transient", 0, b"", {}, exc
|
||||
|
||||
status = resp.status_code
|
||||
content = resp.content
|
||||
headers = {
|
||||
k: v
|
||||
for k, v in resp.headers.items()
|
||||
if k.lower() in ("content-type", "etag", "last-modified")
|
||||
}
|
||||
if status < 400:
|
||||
return "ok", status, content, headers, None
|
||||
if status == 404:
|
||||
return "not_found", status, content, headers, None
|
||||
if status in self.antibot_status:
|
||||
return "antibot", status, content, headers, None
|
||||
if status in self.retry_status:
|
||||
return "transient", status, content, headers, None
|
||||
# unexpected 4xx (not 404/antibot): surface it via the transient path so the
|
||||
# alerting hook can classify the exact status; don't retry-storm.
|
||||
return "transient", status, content, headers, None
|
||||
|
||||
# ---- helpers ----
|
||||
def _log(self, source, url, status, nbytes, attempt, outcome) -> None:
|
||||
self.log_request(
|
||||
{
|
||||
"ts": _now_iso(),
|
||||
"source": source,
|
||||
"url": url,
|
||||
"status": status,
|
||||
"bytes": nbytes,
|
||||
"attempt": attempt,
|
||||
"outcome": outcome,
|
||||
}
|
||||
)
|
||||
Reference in new issue
Block a user