287 lines
13 KiB
Python
287 lines
13 KiB
Python
"""Integration tests: ingest -> parse end-to-end, asserting the JSONL contract.
|
|
|
|
Scope for task t_8e5a3821 item (2): a test that runs ingest -> parse on a
|
|
fixture and asserts the JSONL schema, plus the ingest-layer failure modes
|
|
(expected-404 "not_found", antibot, transient/retry, cache-hit, request-log
|
|
recording, alert-resilience) that the parser never sees.
|
|
|
|
No live network: the HTTP transport is faked (conftest.make_ingestor) so the
|
|
REAL retry/backoff loop and the REAL cache are exercised against a fixture body.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
|
|
import pytest
|
|
|
|
from scraper import parsers
|
|
from scraper.cache import RawCache
|
|
from scraper.io import write_jsonl
|
|
from scraper.request_log import RequestLog
|
|
from scraper.alerts import ALERT_REGISTRY, register
|
|
from scraper.schema import BillAction
|
|
from conftest import make_ingestor, read_jsonl, CONFIG_PATH
|
|
from tests import fixtures as F
|
|
|
|
|
|
# The canonical JSONL field set a downstream consumer may see (spec section 7).
|
|
KNOWN_FIELDS = set(BillAction.field_names())
|
|
BILL_ID_RE = re.compile(r"^\d{8}\d?(AB|SB|ACR|SCR|HR|SJR|ABR)\d+$")
|
|
|
|
|
|
def assert_schema(rows):
|
|
"""Every JSONL row must contain only known, well-typed fields."""
|
|
assert rows, "no records were written"
|
|
for row in rows:
|
|
extra = set(row.keys()) - KNOWN_FIELDS
|
|
assert not extra, "unknown fields in JSONL row: %s" % extra
|
|
assert isinstance(row.get("record_type"), str)
|
|
assert row["record_type"] in ("bill", "hearing")
|
|
if row["record_type"] == "bill":
|
|
if row.get("bill_id"):
|
|
assert BILL_ID_RE.match(row["bill_id"]), row["bill_id"]
|
|
assert row["bill_id"][:8].isdigit(), row["bill_id"]
|
|
assert row.get("bill_number") is not None or row.get("bill_id")
|
|
assert row.get("id")
|
|
assert len(row["id"]) == 40, "bad id length: %s" % row
|
|
|
|
|
|
class TestIngestToParseEndToEnd:
|
|
def test_happy_path_ingest_then_parse_then_jsonl(self, tmp_path,
|
|
monkeypatch):
|
|
monkeypatch.chdir(tmp_path)
|
|
asm = F.assembly_daily_file_html()
|
|
raw_dir = str(tmp_path / "cache" / "raw")
|
|
log_path = str(tmp_path / "cache" / "req.jsonl")
|
|
ing, transport = make_ingestor(
|
|
CONFIG_PATH, raw_dir, log_path,
|
|
route=lambda url: (200, asm.encode("utf-8")),
|
|
)
|
|
res = ing.ingest("2026-08-31", source="assembly_daily_file")
|
|
assert res.all_ok
|
|
cr = res.chambers[0]
|
|
assert cr.status == "ok"
|
|
assert cr.bytes == len(asm.encode("utf-8"))
|
|
assert cr.path is not None and cr.path.exists()
|
|
|
|
# The cached raw file must itself be parseable by the real parser.
|
|
recs = parsers.parse(str(cr.path))
|
|
assert recs, "parser produced no records from the cached HTML"
|
|
# Re-parse the same cache file: deterministic, same record count.
|
|
recs2 = parsers.parse(str(cr.path))
|
|
assert len(recs2) == len(recs)
|
|
|
|
# Serialise to JSONL and assert the schema contract on the real output.
|
|
out = tmp_path / "records.jsonl"
|
|
written = write_jsonl(recs, str(out))
|
|
assert written == len(recs)
|
|
rows = read_jsonl(str(out))
|
|
assert len(rows) == len(recs)
|
|
assert_schema(rows)
|
|
# The fixture's known records must survive the whole pipeline.
|
|
bns = {r["bill_number"] for r in rows if r["record_type"] == "bill"}
|
|
assert "AB 28" in bns and "SB 795" in bns
|
|
|
|
def test_request_log_records_the_ingest(self, tmp_path, monkeypatch):
|
|
monkeypatch.chdir(tmp_path)
|
|
asm = F.assembly_daily_file_html()
|
|
log_path = str(tmp_path / "cache" / "req.jsonl")
|
|
ing, _ = make_ingestor(
|
|
CONFIG_PATH, str(tmp_path / "cache" / "raw"), log_path,
|
|
route=lambda url: (200, asm.encode()),
|
|
)
|
|
ing.ingest("2026-08-31", source="assembly_daily_file")
|
|
events = RequestLog.read(log_path)
|
|
assert events, "request log is empty"
|
|
for e in events:
|
|
for key in ("ts", "source", "url", "status", "bytes",
|
|
"attempt", "outcome", "log_ts"):
|
|
assert key in e, "missing key %s in request-log event" % key
|
|
assert e["status"] == 200
|
|
assert e["source"] == "assembly_daily_file"
|
|
|
|
|
|
class TestIngestFailureModes:
|
|
def test_expected_404_is_not_an_error(self, tmp_path, monkeypatch):
|
|
monkeypatch.chdir(tmp_path)
|
|
ing, _ = make_ingestor(
|
|
CONFIG_PATH, str(tmp_path / "cache" / "raw"),
|
|
str(tmp_path / "req.jsonl"),
|
|
route=lambda url: (404, b""),
|
|
)
|
|
res = ing.ingest("2026-09-04", source="senate_daily_file")
|
|
cr = res.chambers[0]
|
|
assert cr.status == "not_found", cr.status
|
|
assert "expected" in cr.note.lower()
|
|
assert cr.status not in ("antibot", "transient")
|
|
|
|
def test_unexpected_404_on_assembly_is_a_warning_not_silent(
|
|
self, tmp_path, monkeypatch):
|
|
# The Assembly source has no missing_is_expected_404, so its 404 is a
|
|
# real problem: not_found with an "unexpect..." note, not "expected".
|
|
monkeypatch.chdir(tmp_path)
|
|
ing, _ = make_ingestor(
|
|
CONFIG_PATH, str(tmp_path / "cache" / "raw"),
|
|
str(tmp_path / "req.jsonl"),
|
|
route=lambda url: (404, b""),
|
|
)
|
|
res = ing.ingest("2026-09-04", source="assembly_daily_file")
|
|
cr = res.chambers[0]
|
|
assert cr.status == "not_found", cr.status
|
|
# Match on "unexpect": the substring "expected" also appears inside
|
|
# the word "unEXPEcted", so a naive substring check is ambiguous.
|
|
note = cr.note.lower()
|
|
assert "unexpect" in note, cr.note
|
|
|
|
def test_antibot_403_is_flagged(self, tmp_path, monkeypatch):
|
|
monkeypatch.chdir(tmp_path)
|
|
ing, _ = make_ingestor(
|
|
CONFIG_PATH, str(tmp_path / "cache" / "raw"),
|
|
str(tmp_path / "req.jsonl"),
|
|
route=lambda url: (403, b"<html>antibot</html>"),
|
|
)
|
|
res = ing.ingest("2026-09-02", source="senate_daily_file")
|
|
cr = res.chambers[0]
|
|
assert cr.status == "antibot", cr.status
|
|
assert res.all_ok is False
|
|
|
|
def test_transient_retries_three_times(self, tmp_path, monkeypatch):
|
|
monkeypatch.chdir(tmp_path)
|
|
ing, transport = make_ingestor(
|
|
CONFIG_PATH, str(tmp_path / "cache" / "raw"),
|
|
str(tmp_path / "req.jsonl"),
|
|
route=lambda url: (503, b"busy"),
|
|
)
|
|
res = ing.ingest("2026-09-02", source="assembly_daily_file")
|
|
cr = res.chambers[0]
|
|
assert cr.status == "transient", cr.status
|
|
assert cr.attempts == 3, cr.attempts
|
|
# Three attempts -> two back-off sleeps (1s, 2s), recorded not slept.
|
|
assert len(transport.calls) == 3, transport.calls
|
|
assert ing._sleeps == [1.0, 2.0], ing._sleeps
|
|
|
|
|
|
# QA FINDING (P2): scraper/http_client.py has a catch-all "return 'transient'"
|
|
# for any non-404/non-antibot status, and the retry loop treats *any* transient
|
|
# as retryable. Net effect: a permanent 4xx such as 400 is retry-stormed 3x
|
|
# exactly like a transient 5xx, even though the code's own comment says it
|
|
# "should not retry-storm". The xfail guard below pins the CURRENT (buggy)
|
|
# behaviour as a regression trip-wire: when the engineer fixes the retry guard
|
|
# to give up on non-retryable 4xx, the xfail PASSES and -- being strict -- flips
|
|
# to a failure signalling "finding resolved, update the guard". See QA_CHECKLIST.
|
|
class TestRetryStormRegressionGuard:
|
|
@pytest.mark.xfail(
|
|
strict=True,
|
|
reason="P2 finding: non-retryable 4xx (400) is retry-stormed 3x instead "
|
|
"of giving up on attempt 1 (http_client.py catch-all transient). "
|
|
"See QA_CHECKLIST.md. Remove this guard once the engine is fixed.")
|
|
def test_non_retryable_4xx_must_give_up_on_attempt_1(
|
|
self, tmp_path, monkeypatch):
|
|
# Desired behaviour (currently unmet): a 400 is neither a 404, an
|
|
# antibot, nor a transient-network fault, so it must NOT retry-storm --
|
|
# one attempt, zero back-off sleeps.
|
|
monkeypatch.chdir(tmp_path)
|
|
ing, transport = make_ingestor(
|
|
CONFIG_PATH, str(tmp_path / "cache" / "raw"),
|
|
str(tmp_path / "req.jsonl"),
|
|
route=lambda url: (400, b"bad request"),
|
|
)
|
|
res = ing.ingest("2026-09-02", source="assembly_daily_file")
|
|
cr = res.chambers[0]
|
|
assert cr.status == "transient", cr.status
|
|
assert cr.attempts == 1, "non-retryable 4xx must give up on attempt 1"
|
|
assert len(ing._sleeps) == 0, "no backoff for a non-retryable 4xx"
|
|
|
|
def test_retry_storm_on_4xx_is_currently_3_attempts(self, tmp_path,
|
|
monkeypatch):
|
|
# Current, PASSING behaviour (the buggy side): a 400 really is retried
|
|
# 3x with back-off [1.0, 2.0]. Pinned so the gate stays green while the
|
|
# strict-xfail guard above records the defect. Remove when fixed.
|
|
monkeypatch.chdir(tmp_path)
|
|
ing, transport = make_ingestor(
|
|
CONFIG_PATH, str(tmp_path / "cache" / "raw"),
|
|
str(tmp_path / "req.jsonl"),
|
|
route=lambda url: (400, b"bad request"),
|
|
)
|
|
res = ing.ingest("2026-09-02", source="assembly_daily_file")
|
|
cr = res.chambers[0]
|
|
assert cr.status == "transient", cr.status
|
|
assert cr.attempts == 3, cr.attempts
|
|
assert ing._sleeps == [1.0, 2.0], ing._sleeps
|
|
|
|
|
|
class TestCacheIntegrity:
|
|
def test_cache_hit_serves_from_disk_without_network(self, tmp_path,
|
|
monkeypatch):
|
|
monkeypatch.chdir(tmp_path)
|
|
asm = F.assembly_daily_file_html()
|
|
raw_dir = str(tmp_path / "cache" / "raw")
|
|
log_path = str(tmp_path / "cache" / "req.jsonl")
|
|
# 1) First ingest: network miss -> downloads + caches.
|
|
ing1, tp1 = make_ingestor(
|
|
CONFIG_PATH, raw_dir, log_path,
|
|
route=lambda url: (200, asm.encode()),
|
|
)
|
|
res1 = ing1.ingest("2026-08-31", source="assembly_daily_file")
|
|
assert res1.chambers[0].status == "ok"
|
|
assert len(tp1.calls) == 1
|
|
|
|
# 2) Second ingest, same cache, force=False -> cache hit, no network.
|
|
ing2, tp2 = make_ingestor(
|
|
CONFIG_PATH, raw_dir, log_path, route=None,
|
|
)
|
|
ing2.force = False
|
|
res2 = ing2.ingest("2026-08-31", source="assembly_daily_file")
|
|
cr2 = res2.chambers[0]
|
|
assert cr2.status == "cached", cr2.status
|
|
assert cr2.cache_hit is True
|
|
assert tp2 is None # no fake transport wired -> no network on hit
|
|
|
|
def test_alert_sink_failure_does_not_crash_run(self, tmp_path,
|
|
monkeypatch):
|
|
# An alerting sink that raises must not crash the ingest run; the
|
|
# failure is logged and falls back to the log sink.
|
|
monkeypatch.chdir(tmp_path)
|
|
asm = F.assembly_daily_file_html()
|
|
|
|
def _boom(ctx):
|
|
raise RuntimeError("alert sink on fire")
|
|
|
|
register("_test_boom_sink", _boom)
|
|
ing, _ = make_ingestor(
|
|
CONFIG_PATH, str(tmp_path / "cache" / "raw"),
|
|
str(tmp_path / "req.jsonl"),
|
|
route=lambda url: (200, asm.encode()),
|
|
)
|
|
# Point the alert sink at the bomb (config is a live dict).
|
|
ing.cfg._alerts["sink"] = "_test_boom_sink"
|
|
res = ing.ingest("2026-08-31", source="assembly_daily_file")
|
|
assert res.chambers[0].status == "ok", "alert failure must not break ingest"
|
|
# Clean up the registry so the bomb sink does not leak into other tests.
|
|
ALERT_REGISTRY.pop("_test_boom_sink", None)
|
|
|
|
def test_cache_roundtrip_preserves_bytes(self, tmp_path):
|
|
cache = RawCache(str(tmp_path / "raw"))
|
|
data = F.assembly_daily_file_html().encode("utf-8")
|
|
path = cache.put(
|
|
"assembly_daily_file", "2026-09-01", "html", data,
|
|
url="http://example/a", status=200,
|
|
fetched_at="2026-09-01T00:00:00+00:00",
|
|
)
|
|
assert path is not None
|
|
got = cache.get("assembly_daily_file", "2026-09-01", "html")
|
|
assert got == data
|
|
assert cache.exists("assembly_daily_file", "2026-09-01", "html")
|
|
|
|
def test_cache_corruption_is_treated_as_a_miss(self, tmp_path):
|
|
# A corrupt file (sha mismatch vs its meta) must be a cache miss, not a
|
|
# poisoned hit.
|
|
cache = RawCache(str(tmp_path / "raw"))
|
|
bad = tmp_path / "raw" / "assembly_daily_file" / "2026-09-01.html"
|
|
bad.parent.mkdir(parents=True, exist_ok=True)
|
|
bad.write_text("garbage-not-the-cached-bytes")
|
|
meta = cache.meta_path("assembly_daily_file", "2026-09-01")
|
|
meta.write_text('{"sha256": "deadbeef", "bytes": 8}\n')
|
|
assert cache.get("assembly_daily_file", "2026-09-01", "html") is None
|
|
assert cache.exists("assembly_daily_file", "2026-09-01", "html") is False
|