Initial commit

scripts/test_ingest_offline.py

@@ -0,0 +1,157 @@
+"""Deterministic acceptance test for scraper.ingest -- no real network.
+
+Uses a fake HttpClient and an in-process RawCache (cache/raw) so every path is
+proven without depending on the live servers:
+   1. Happy path: assembly HTML download + cache miss -> ok
+   2. Cache hit on re-run -> no network, cache_hit=True
+   3. 404 EXPECTED (Senate non-legislative day) -> 'not_found', not a crash
+   4. 403 antibot -> 'antibot' status + alert, no crash
+   5. 5xx transient -> retries (3 attempts) -> 'transient' + alert
+   6. Cache-hit on the 404-affected source is NOT persisted (404s are not cached)
+"""
+
+from __future__ import annotations
+
+import time
+import tempfile
+import os
+import sys
+
+# Make the repo importable when run from anywhere.
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, ROOT)
+
+from scraper.ingest import Ingestor  # noqa: E402
+from scraper.http_client import (  # noqa: E402
+    AntibotError,
+    FetchResult,
+    NotFoundError,
+    TransientError,
+)
+from scraper.request_log import RequestLog  # noqa: E402
+
+
+PASS = []
+FAIL = []
+
+
+def check(name, cond, detail=""):
+    (PASS if cond else FAIL).append(name)
+    print(("  PASS  " if cond else "  FAIL  ") + name +
+          (f"   -- {detail}" if detail and not cond else ""))
+
+
+# --- Fake transport: mimics httpx.Client.get() so the REAL retry loop runs ---
+class FakeResponse:
+    def __init__(self, status, content, content_type="text/html"):
+        self.status_code = status
+        self.content = content
+        self.headers = {"content-type": content_type}
+
+
+class FakeTransport:
+         # Injected as Ingestor.http._client. The real HttpClient.get() runs its
+         # own retry/backoff loop on top of this, so transient/404/403 paths all
+         # exercise the real code. route(url) -> (status, content).
+    def __init__(self, route):
+        self.route = route
+        self.calls = []
+
+    def get(self, url, follow_redirects=False):
+        self.calls.append(url)
+        status, content = self.route(url)
+        return FakeResponse(status, content)
+
+    def close(self):
+        pass
+
+
+def inject_transport(ing, route, sleep_spy):
+         # Use the REAL HttpClient (real retry loop) but fake the underlying
+         # transport so no real network happens. No-op the backoff sleep so the
+         # test is instant but still records the *number* of sleeps.
+    ing.http._client = FakeTransport(route)
+    ing.http.sleep = lambda d: sleep_spy.append(d)
+    return ing.http._client
+
+
+def main():
+    from pathlib import Path
+    tmp = tempfile.mkdtemp(prefix="scraper_test_")
+
+    def fresh(cache_sub, log_name):
+          # Build an Ingestor wired to a real cache/log in a temp dir.
+        ing = Ingestor(config_path=os.path.join(ROOT, "config.yaml"))
+        ing.cache.raw_dir = Path(tmp) / cache_sub
+        ing.log.path = Path(tmp) / log_name
+        return ing
+
+    print("== Test 1: assembly happy path (cache miss -> ok) ==")
+    ing1 = fresh("raw1", "log1.jsonl")
+    tp1 = inject_transport(ing1, lambda url: (200, b"<html>ASM BODY</html>"), [])
+    res = ing1.ingest("2026-09-02", source="assembly_daily_file")
+    c = res.chambers[0]
+    check("t1 status ok", c.status == "ok", c.status)
+    check("t1 wrote bytes", c.bytes == len(b"<html>ASM BODY</html>"))
+    check("t1 not cached", c.cache_hit is False)
+    check("t1 one network call", len(tp1.calls) == 1, str(tp1.calls))
+    check("t1 all_ok", res.all_ok is True)
+
+    print("== Test 2: cache hit on re-run (no network) ==")
+    ing2 = fresh("raw1", "log2.jsonl")     # same cache dir as test 1
+    tp2 = inject_transport(ing2, lambda url: (_ for _ in ()).throw(
+         AssertionError("network SHOULD NOT be called on a cache hit")), [])
+    res2 = ing2.ingest("2026-09-02", source="assembly_daily_file")
+    c2 = res2.chambers[0]
+    check("t2 status cached", c2.status == "cached", c2.status)
+    check("t2 cache_hit True", c2.cache_hit is True)
+    check("t2 no network call", len(tp2.calls) == 0, str(tp2.calls))
+
+    print("== Test 3: expected 404 (Senate non-legislative day) ==")
+    ing3 = fresh("raw3", "log3.jsonl")
+    inject_transport(ing3, lambda url: (404, b""), [])
+    res3 = ing3.ingest("2026-09-04", source="senate_daily_file")
+    c3 = res3.chambers[0]
+    check("t3 not a crash", True)
+    check("t3 status not_found", c3.status == "not_found", c3.status)
+    check("t3 expected note", "expected" in c3.note, c3.note)
+
+    print("== Test 4: antibot 403 -> 'antibot' status, no crash ==")
+    ing4 = fresh("raw4", "log4.jsonl")
+    inject_transport(ing4, lambda url: (403, b"<html>antibot</html>"), [])
+    res4 = ing4.ingest("2026-09-02", source="senate_daily_file")
+    c4 = res4.chambers[0]
+    check("t4 status antibot", c4.status == "antibot", c4.status)
+    check("t4 not all_ok", res4.all_ok is False)
+
+    print("== Test 5: 5xx transient retries then gives up ==")
+    sleeps5 = []
+    ing5 = fresh("raw5", "log5.jsonl")
+    tp5 = inject_transport(ing5, lambda url: (503, b"busy"), sleeps5)
+    res5 = ing5.ingest("2026-09-02", source="assembly_daily_file")
+    c5 = res5.chambers[0]
+    check("t5 status transient", c5.status == "transient", c5.status)
+    check("t5 retried all 3 attempts", len(tp5.calls) == 3, str(len(tp5.calls)))
+    check("t5 gave up after max_retries", c5.attempts == 3, str(c5.attempts))
+    check("t5 backed off 2x (3 tries -> 2 sleeps)", len(sleeps5) == 2,
+           str(sleeps5))
+    check("t5 exponential backoff (1s,2s)", sleeps5 == [1.0, 2.0], str(sleeps5))
+
+    print("== Test 6: log records every attempt ==")
+    log5 = RequestLog.read(Path(tmp) / "log5.jsonl")
+    check("t6 attempts logged", len(log5) >= 3, f"{len(log5)} lines")
+    outcomes = [e.get("outcome") for e in log5]
+    check("t6 retry outcomes recorded", "5xx_retrying" in outcomes,
+           str(outcomes))
+
+    print()
+    print(f"PASSED {len(PASS)} / {len(PASS)+len(FAIL)}")
+    if FAIL:
+        print("FAILED:", FAIL)
+        sys.exit(1)
+    print("ALL DETERMINISTIC TESTS PASSED")
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()