Initial commit

scraper/cache.py

@@ -0,0 +1,93 @@
+"""Local raw-byte cache.
+
+Layout under cache.raw_dir:
+     {source}/{YYYY-MM-DD}.{ext}      the raw bytes (html or pdf)
+     {source}/{YYYY-MM-DD}.meta.json  sidecar: url, status, sha256, fetched_at
+
+A cache "hit" is served straight from disk (cache_hit=True) and is the ONLY way a
+re-ingest avoids a network round-trip. Writes are atomic (tmp + os.replace) so a
+crash mid-download never leaves a truncated file.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from pathlib import Path
+from typing import Optional
+
+
+def sha256_bytes(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+class RawCache:
+    """On-disk keyed cache of raw response bodies."""
+
+    def __init__(self, raw_dir):
+        self.raw_dir = Path(raw_dir)
+
+    def _key_path(self, source: str, date: str, ext: str) -> Path:
+        return self.raw_dir / source / f"{date}.{ext}"
+
+    def meta_path(self, source: str, date: str) -> Path:
+        return self.raw_dir / source / f"{date}.meta.json"
+
+    def get(self, source: str, date: str, ext: str) -> Optional[bytes]:
+        """Return cached bytes if present AND matching its recorded sha256
+        (guards a truncated/corrupt write). Else None (treated as a miss).
+        """
+        p = self._key_path(source, date, ext)
+        meta = self._read_meta(source, date)
+        if not p.exists() or meta is None:
+            return None
+        try:
+            data = p.read_bytes()
+        except OSError:
+            return None
+        if meta.get("sha256") and sha256_bytes(data) != meta["sha256"]:
+            return None  # integrity mismatch -> miss
+        return data
+
+    def put(self, source: str, date: str, ext: str, data: bytes,
+            url: str, status: int, fetched_at: str) -> Path:
+        path = self._key_path(source, date, ext)
+        path.parent.mkdir(parents=True, exist_ok=True)
+
+        tmp = path.with_name(path.name + ".tmp")
+        tmp.write_bytes(data)
+        os.replace(tmp, path)  # atomic on POSIX
+
+        meta = {
+            "source": source,
+            "date": date,
+            "ext": ext,
+            "url": url,
+            "status": status,
+            "sha256": sha256_bytes(data),
+            "bytes": len(data),
+            "fetched_at": fetched_at,
+        }
+        self._write_meta(source, date, meta)
+        return path
+
+    def exists(self, source: str, date: str, ext: str) -> bool:
+        return self.get(source, date, ext) is not None
+
+    # -- meta helpers --
+    def _read_meta(self, source: str, date: str) -> Optional[dict]:
+        mp = self.meta_path(source, date)
+        if not mp.exists():
+            return None
+        try:
+            return json.loads(mp.read_text())
+        except (OSError, ValueError):
+            return None
+
+    def _write_meta(self, source: str, date: str, meta: dict) -> None:
+        mp = self.meta_path(source, date)
+        mp.parent.mkdir(parents=True, exist_ok=True)
+        tmp = mp.with_suffix(".json.tmp")
+        tmp.write_text(json.dumps(meta, indent=2, sort_keys=True) + "\n")
+        os.replace(tmp, mp)