"""Integration tests: ingest -> parse end-to-end, asserting the JSONL contract.

Scope for task t_8e5a3821 item (2): a test that runs ingest -> parse on a
fixture and asserts the JSONL schema, plus the ingest-layer failure modes
(expected-404 "not_found", antibot, transient/retry, cache-hit, request-log
recording, alert-resilience) that the parser never sees.

No live network: the HTTP transport is faked (conftest.make_ingestor) so the
REAL retry/backoff loop and the REAL cache are exercised against a fixture body.
"""
from __future__ import annotations

import re

import pytest

from scraper import parsers
from scraper.cache import RawCache
from scraper.io import write_jsonl
from scraper.request_log import RequestLog
from scraper.alerts import ALERT_REGISTRY, register
from scraper.schema import BillAction
from conftest import make_ingestor, read_jsonl, CONFIG_PATH
from tests import fixtures as F


# The canonical JSONL field set a downstream consumer may see (spec section 7).
KNOWN_FIELDS = set(BillAction.field_names())
BILL_ID_RE = re.compile(r"^\d{8}\d?(AB|SB|ACR|SCR|HR|SJR|ABR)\d+$")


def assert_schema(rows):
    """Every JSONL row must contain only known, well-typed fields."""
    assert rows, "no records were written"
    for row in rows:
        extra = set(row.keys()) - KNOWN_FIELDS
        assert not extra, "unknown fields in JSONL row: %s" % extra
        assert isinstance(row.get("record_type"), str)
        assert row["record_type"] in ("bill", "hearing")
        if row["record_type"] == "bill":
            if row.get("bill_id"):
                assert BILL_ID_RE.match(row["bill_id"]), row["bill_id"]
                assert row["bill_id"][:8].isdigit(), row["bill_id"]
            assert row.get("bill_number") is not None or row.get("bill_id")
            assert row.get("id")
            assert len(row["id"]) == 40, "bad id length: %s" % row


class TestIngestToParseEndToEnd:
    def test_happy_path_ingest_then_parse_then_jsonl(self, tmp_path,
                                                      monkeypatch):
        monkeypatch.chdir(tmp_path)
        asm = F.assembly_daily_file_html()
        raw_dir = str(tmp_path / "cache" / "raw")
        log_path = str(tmp_path / "cache" / "req.jsonl")
        ing, transport = make_ingestor(
            CONFIG_PATH, raw_dir, log_path,
            route=lambda url: (200, asm.encode("utf-8")),
        )
        res = ing.ingest("2026-08-31", source="assembly_daily_file")
        assert res.all_ok
        cr = res.chambers[0]
        assert cr.status == "ok"
        assert cr.bytes == len(asm.encode("utf-8"))
        assert cr.path is not None and cr.path.exists()

        # The cached raw file must itself be parseable by the real parser.
        recs = parsers.parse(str(cr.path))
        assert recs, "parser produced no records from the cached HTML"
        # Re-parse the same cache file: deterministic, same record count.
        recs2 = parsers.parse(str(cr.path))
        assert len(recs2) == len(recs)

        # Serialise to JSONL and assert the schema contract on the real output.
        out = tmp_path / "records.jsonl"
        written = write_jsonl(recs, str(out))
        assert written == len(recs)
        rows = read_jsonl(str(out))
        assert len(rows) == len(recs)
        assert_schema(rows)
        # The fixture's known records must survive the whole pipeline.
        bns = {r["bill_number"] for r in rows if r["record_type"] == "bill"}
        assert "AB 28" in bns and "SB 795" in bns

    def test_request_log_records_the_ingest(self, tmp_path, monkeypatch):
        monkeypatch.chdir(tmp_path)
        asm = F.assembly_daily_file_html()
        log_path = str(tmp_path / "cache" / "req.jsonl")
        ing, _ = make_ingestor(
            CONFIG_PATH, str(tmp_path / "cache" / "raw"), log_path,
            route=lambda url: (200, asm.encode()),
        )
        ing.ingest("2026-08-31", source="assembly_daily_file")
        events = RequestLog.read(log_path)
        assert events, "request log is empty"
        for e in events:
            for key in ("ts", "source", "url", "status", "bytes",
                        "attempt", "outcome", "log_ts"):
                assert key in e, "missing key %s in request-log event" % key
            assert e["status"] == 200
            assert e["source"] == "assembly_daily_file"


class TestIngestFailureModes:
    def test_expected_404_is_not_an_error(self, tmp_path, monkeypatch):
        monkeypatch.chdir(tmp_path)
        ing, _ = make_ingestor(
            CONFIG_PATH, str(tmp_path / "cache" / "raw"),
            str(tmp_path / "req.jsonl"),
            route=lambda url: (404, b""),
        )
        res = ing.ingest("2026-09-04", source="senate_daily_file")
        cr = res.chambers[0]
        assert cr.status == "not_found", cr.status
        assert "expected" in cr.note.lower()
        assert cr.status not in ("antibot", "transient")

    def test_unexpected_404_on_assembly_is_a_warning_not_silent(
            self, tmp_path, monkeypatch):
        # The Assembly source has no missing_is_expected_404, so its 404 is a
        # real problem: not_found with an "unexpect..." note, not "expected".
        monkeypatch.chdir(tmp_path)
        ing, _ = make_ingestor(
            CONFIG_PATH, str(tmp_path / "cache" / "raw"),
            str(tmp_path / "req.jsonl"),
            route=lambda url: (404, b""),
        )
        res = ing.ingest("2026-09-04", source="assembly_daily_file")
        cr = res.chambers[0]
        assert cr.status == "not_found", cr.status
        # Match on "unexpect": the substring "expected" also appears inside
        # the word "unEXPEcted", so a naive substring check is ambiguous.
        note = cr.note.lower()
        assert "unexpect" in note, cr.note

    def test_antibot_403_is_flagged(self, tmp_path, monkeypatch):
        monkeypatch.chdir(tmp_path)
        ing, _ = make_ingestor(
            CONFIG_PATH, str(tmp_path / "cache" / "raw"),
            str(tmp_path / "req.jsonl"),
            route=lambda url: (403, b"<html>antibot</html>"),
        )
        res = ing.ingest("2026-09-02", source="senate_daily_file")
        cr = res.chambers[0]
        assert cr.status == "antibot", cr.status
        assert res.all_ok is False

    def test_transient_retries_three_times(self, tmp_path, monkeypatch):
        monkeypatch.chdir(tmp_path)
        ing, transport = make_ingestor(
            CONFIG_PATH, str(tmp_path / "cache" / "raw"),
            str(tmp_path / "req.jsonl"),
            route=lambda url: (503, b"busy"),
        )
        res = ing.ingest("2026-09-02", source="assembly_daily_file")
        cr = res.chambers[0]
        assert cr.status == "transient", cr.status
        assert cr.attempts == 3, cr.attempts
        # Three attempts -> two back-off sleeps (1s, 2s), recorded not slept.
        assert len(transport.calls) == 3, transport.calls
        assert ing._sleeps == [1.0, 2.0], ing._sleeps


# QA FINDING (P2): scraper/http_client.py has a catch-all "return 'transient'"
# for any non-404/non-antibot status, and the retry loop treats *any* transient
# as retryable. Net effect: a permanent 4xx such as 400 is retry-stormed 3x
# exactly like a transient 5xx, even though the code's own comment says it
# "should not retry-storm". The xfail guard below pins the CURRENT (buggy)
# behaviour as a regression trip-wire: when the engineer fixes the retry guard
# to give up on non-retryable 4xx, the xfail PASSES and -- being strict -- flips
# to a failure signalling "finding resolved, update the guard". See QA_CHECKLIST.
class TestRetryStormRegressionGuard:
    @pytest.mark.xfail(
        strict=True,
        reason="P2 finding: non-retryable 4xx (400) is retry-stormed 3x instead "
        "of giving up on attempt 1 (http_client.py catch-all transient). "
        "See QA_CHECKLIST.md. Remove this guard once the engine is fixed.")
    def test_non_retryable_4xx_must_give_up_on_attempt_1(
            self, tmp_path, monkeypatch):
        # Desired behaviour (currently unmet): a 400 is neither a 404, an
        # antibot, nor a transient-network fault, so it must NOT retry-storm --
        # one attempt, zero back-off sleeps.
        monkeypatch.chdir(tmp_path)
        ing, transport = make_ingestor(
            CONFIG_PATH, str(tmp_path / "cache" / "raw"),
            str(tmp_path / "req.jsonl"),
            route=lambda url: (400, b"bad request"),
        )
        res = ing.ingest("2026-09-02", source="assembly_daily_file")
        cr = res.chambers[0]
        assert cr.status == "transient", cr.status
        assert cr.attempts == 1, "non-retryable 4xx must give up on attempt 1"
        assert len(ing._sleeps) == 0, "no backoff for a non-retryable 4xx"

    def test_retry_storm_on_4xx_is_currently_3_attempts(self, tmp_path,
                                                        monkeypatch):
        # Current, PASSING behaviour (the buggy side): a 400 really is retried
        # 3x with back-off [1.0, 2.0]. Pinned so the gate stays green while the
        # strict-xfail guard above records the defect. Remove when fixed.
        monkeypatch.chdir(tmp_path)
        ing, transport = make_ingestor(
            CONFIG_PATH, str(tmp_path / "cache" / "raw"),
            str(tmp_path / "req.jsonl"),
            route=lambda url: (400, b"bad request"),
        )
        res = ing.ingest("2026-09-02", source="assembly_daily_file")
        cr = res.chambers[0]
        assert cr.status == "transient", cr.status
        assert cr.attempts == 3, cr.attempts
        assert ing._sleeps == [1.0, 2.0], ing._sleeps


class TestCacheIntegrity:
    def test_cache_hit_serves_from_disk_without_network(self, tmp_path,
                                                        monkeypatch):
        monkeypatch.chdir(tmp_path)
        asm = F.assembly_daily_file_html()
        raw_dir = str(tmp_path / "cache" / "raw")
        log_path = str(tmp_path / "cache" / "req.jsonl")
        # 1) First ingest: network miss -> downloads + caches.
        ing1, tp1 = make_ingestor(
            CONFIG_PATH, raw_dir, log_path,
            route=lambda url: (200, asm.encode()),
        )
        res1 = ing1.ingest("2026-08-31", source="assembly_daily_file")
        assert res1.chambers[0].status == "ok"
        assert len(tp1.calls) == 1

        # 2) Second ingest, same cache, force=False -> cache hit, no network.
        ing2, tp2 = make_ingestor(
            CONFIG_PATH, raw_dir, log_path, route=None,
        )
        ing2.force = False
        res2 = ing2.ingest("2026-08-31", source="assembly_daily_file")
        cr2 = res2.chambers[0]
        assert cr2.status == "cached", cr2.status
        assert cr2.cache_hit is True
        assert tp2 is None  # no fake transport wired -> no network on hit

    def test_alert_sink_failure_does_not_crash_run(self, tmp_path,
                                                    monkeypatch):
        # An alerting sink that raises must not crash the ingest run; the
        # failure is logged and falls back to the log sink.
        monkeypatch.chdir(tmp_path)
        asm = F.assembly_daily_file_html()

        def _boom(ctx):
            raise RuntimeError("alert sink on fire")

        register("_test_boom_sink", _boom)
        ing, _ = make_ingestor(
            CONFIG_PATH, str(tmp_path / "cache" / "raw"),
            str(tmp_path / "req.jsonl"),
            route=lambda url: (200, asm.encode()),
        )
        # Point the alert sink at the bomb (config is a live dict).
        ing.cfg._alerts["sink"] = "_test_boom_sink"
        res = ing.ingest("2026-08-31", source="assembly_daily_file")
        assert res.chambers[0].status == "ok", "alert failure must not break ingest"
        # Clean up the registry so the bomb sink does not leak into other tests.
        ALERT_REGISTRY.pop("_test_boom_sink", None)

    def test_cache_roundtrip_preserves_bytes(self, tmp_path):
        cache = RawCache(str(tmp_path / "raw"))
        data = F.assembly_daily_file_html().encode("utf-8")
        path = cache.put(
            "assembly_daily_file", "2026-09-01", "html", data,
            url="http://example/a", status=200,
            fetched_at="2026-09-01T00:00:00+00:00",
        )
        assert path is not None
        got = cache.get("assembly_daily_file", "2026-09-01", "html")
        assert got == data
        assert cache.exists("assembly_daily_file", "2026-09-01", "html")

    def test_cache_corruption_is_treated_as_a_miss(self, tmp_path):
        # A corrupt file (sha mismatch vs its meta) must be a cache miss, not a
        # poisoned hit.
        cache = RawCache(str(tmp_path / "raw"))
        bad = tmp_path / "raw" / "assembly_daily_file" / "2026-09-01.html"
        bad.parent.mkdir(parents=True, exist_ok=True)
        bad.write_text("garbage-not-the-cached-bytes")
        meta = cache.meta_path("assembly_daily_file", "2026-09-01")
        meta.write_text('{"sha256": "deadbeef", "bytes": 8}\n')
        assert cache.get("assembly_daily_file", "2026-09-01", "html") is None
        assert cache.exists("assembly_daily_file", "2026-09-01", "html") is False