94 lines
3.0 KiB
Python
94 lines
3.0 KiB
Python
"""Local raw-byte cache.
|
|
|
|
Layout under cache.raw_dir:
|
|
{source}/{YYYY-MM-DD}.{ext} the raw bytes (html or pdf)
|
|
{source}/{YYYY-MM-DD}.meta.json sidecar: url, status, sha256, fetched_at
|
|
|
|
A cache "hit" is served straight from disk (cache_hit=True) and is the ONLY way a
|
|
re-ingest avoids a network round-trip. Writes are atomic (tmp + os.replace) so a
|
|
crash mid-download never leaves a truncated file.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
|
|
def sha256_bytes(data: bytes) -> str:
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
class RawCache:
|
|
"""On-disk keyed cache of raw response bodies."""
|
|
|
|
def __init__(self, raw_dir):
|
|
self.raw_dir = Path(raw_dir)
|
|
|
|
def _key_path(self, source: str, date: str, ext: str) -> Path:
|
|
return self.raw_dir / source / f"{date}.{ext}"
|
|
|
|
def meta_path(self, source: str, date: str) -> Path:
|
|
return self.raw_dir / source / f"{date}.meta.json"
|
|
|
|
def get(self, source: str, date: str, ext: str) -> Optional[bytes]:
|
|
"""Return cached bytes if present AND matching its recorded sha256
|
|
(guards a truncated/corrupt write). Else None (treated as a miss).
|
|
"""
|
|
p = self._key_path(source, date, ext)
|
|
meta = self._read_meta(source, date)
|
|
if not p.exists() or meta is None:
|
|
return None
|
|
try:
|
|
data = p.read_bytes()
|
|
except OSError:
|
|
return None
|
|
if meta.get("sha256") and sha256_bytes(data) != meta["sha256"]:
|
|
return None # integrity mismatch -> miss
|
|
return data
|
|
|
|
def put(self, source: str, date: str, ext: str, data: bytes,
|
|
url: str, status: int, fetched_at: str) -> Path:
|
|
path = self._key_path(source, date, ext)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
tmp = path.with_name(path.name + ".tmp")
|
|
tmp.write_bytes(data)
|
|
os.replace(tmp, path) # atomic on POSIX
|
|
|
|
meta = {
|
|
"source": source,
|
|
"date": date,
|
|
"ext": ext,
|
|
"url": url,
|
|
"status": status,
|
|
"sha256": sha256_bytes(data),
|
|
"bytes": len(data),
|
|
"fetched_at": fetched_at,
|
|
}
|
|
self._write_meta(source, date, meta)
|
|
return path
|
|
|
|
def exists(self, source: str, date: str, ext: str) -> bool:
|
|
return self.get(source, date, ext) is not None
|
|
|
|
# -- meta helpers --
|
|
def _read_meta(self, source: str, date: str) -> Optional[dict]:
|
|
mp = self.meta_path(source, date)
|
|
if not mp.exists():
|
|
return None
|
|
try:
|
|
return json.loads(mp.read_text())
|
|
except (OSError, ValueError):
|
|
return None
|
|
|
|
def _write_meta(self, source: str, date: str, meta: dict) -> None:
|
|
mp = self.meta_path(source, date)
|
|
mp.parent.mkdir(parents=True, exist_ok=True)
|
|
tmp = mp.with_suffix(".json.tmp")
|
|
tmp.write_text(json.dumps(meta, indent=2, sort_keys=True) + "\n")
|
|
os.replace(tmp, mp)
|