"""Local raw-byte cache. Layout under cache.raw_dir: {source}/{YYYY-MM-DD}.{ext} the raw bytes (html or pdf) {source}/{YYYY-MM-DD}.meta.json sidecar: url, status, sha256, fetched_at A cache "hit" is served straight from disk (cache_hit=True) and is the ONLY way a re-ingest avoids a network round-trip. Writes are atomic (tmp + os.replace) so a crash mid-download never leaves a truncated file. """ from __future__ import annotations import hashlib import json import os from pathlib import Path from typing import Optional def sha256_bytes(data: bytes) -> str: return hashlib.sha256(data).hexdigest() class RawCache: """On-disk keyed cache of raw response bodies.""" def __init__(self, raw_dir): self.raw_dir = Path(raw_dir) def _key_path(self, source: str, date: str, ext: str) -> Path: return self.raw_dir / source / f"{date}.{ext}" def meta_path(self, source: str, date: str) -> Path: return self.raw_dir / source / f"{date}.meta.json" def get(self, source: str, date: str, ext: str) -> Optional[bytes]: """Return cached bytes if present AND matching its recorded sha256 (guards a truncated/corrupt write). Else None (treated as a miss). """ p = self._key_path(source, date, ext) meta = self._read_meta(source, date) if not p.exists() or meta is None: return None try: data = p.read_bytes() except OSError: return None if meta.get("sha256") and sha256_bytes(data) != meta["sha256"]: return None # integrity mismatch -> miss return data def put(self, source: str, date: str, ext: str, data: bytes, url: str, status: int, fetched_at: str) -> Path: path = self._key_path(source, date, ext) path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_name(path.name + ".tmp") tmp.write_bytes(data) os.replace(tmp, path) # atomic on POSIX meta = { "source": source, "date": date, "ext": ext, "url": url, "status": status, "sha256": sha256_bytes(data), "bytes": len(data), "fetched_at": fetched_at, } self._write_meta(source, date, meta) return path def exists(self, source: str, date: str, ext: str) -> bool: return self.get(source, date, ext) is not None # -- meta helpers -- def _read_meta(self, source: str, date: str) -> Optional[dict]: mp = self.meta_path(source, date) if not mp.exists(): return None try: return json.loads(mp.read_text()) except (OSError, ValueError): return None def _write_meta(self, source: str, date: str, meta: dict) -> None: mp = self.meta_path(source, date) mp.parent.mkdir(parents=True, exist_ok=True) tmp = mp.with_suffix(".json.tmp") tmp.write_text(json.dumps(meta, indent=2, sort_keys=True) + "\n") os.replace(tmp, mp)