cali_scraper/tests/test_schema.py

"""Unit tests for scraper.schema (BillAction) and its JSONL contract.

The normalised record schema is spec section 7. This file pins the field set,
the content-hash id, the null-stripping serialisation, and the round-trip
through JSON Lines -- the output contract the downstream RDB / UI lanes depend
on, so it must be stable and asserted explicitly.
"""
from __future__ import annotations

import hashlib
import json

import pytest

from conftest import read_jsonl
from scraper.schema import (
     ASSEMBLY_DAILY_FILE,
     BillAction,
     HOUSE,
     SENATE_DAILY_FILE_PDF,
)


# A realistic bill record, mirroring the real AB 28 output.
def sample_bill() -> BillAction:
    return BillAction(
         date="2026-08-31",
         house="assembly",
         source=ASSEMBLY_DAILY_FILE,
         record_type="bill",
         bill_number="AB 28",
         bill_id="202520260AB28",
         session="2025-2026",
         chamber_origin="ASSEMBLY",
         author="Schiavo et al.",
         short_title="An act relating to solid waste.",
         description="An act relating to solid waste.",
         location="Unfinished Business",
         item_number=32,
         supplement=2,
     )


class TestFieldSet:
    def test_field_names_match_schema(self):
        names = set(BillAction.field_names())
        # The core fields every downstream consumer expects (spec section 7).
        for expected in [
              "id", "date", "house", "source", "record_type",
              "bill_number", "bill_id", "session", "chamber_origin",
              "author", "short_title", "description", "committee",
              "hearing_time", "location", "item_number", "supplement",
              "ics_url", "agenda_id", "raw_selector",
         ]:
            assert expected in names, f"missing field {expected!r}"

    def test_default_house_is_assembly(self):
        assert HOUSE == "assembly"
        assert BillAction().house == "assembly"

    def test_module_source_constants(self):
        assert {ASSEMBLY_DAILY_FILE, SENATE_DAILY_FILE_PDF} <= {
             ASSEMBLY_DAILY_FILE, SENATE_DAILY_FILE_PDF, "LEGINFO_BILL_RESULTS",
         }


class TestWithId:
    def test_id_is_deterministic_content_hash(self):
        a = sample_bill()
        b = sample_bill()
        a.with_id()
        b.with_id()
        assert a.id == b.id
        assert a.id == hashlib.sha1(
             "|".join([a.house, a.bill_number, a.date, a.committee or ""])
              .encode("utf-8")).hexdigest()

    def test_missing_components_render_empty_in_hash(self):
        rec = sample_bill()
        rec.committee = None
        rec.date = None
        rec.with_id()
        expected = hashlib.sha1(
             "|".join([rec.house or "", rec.bill_number or "",
                         rec.date or "", rec.committee or ""]
                      ).encode("utf-8")).hexdigest()
        assert rec.id == expected

    def test_explicit_content_used_when_supplied(self):
        rec = BillAction()
        rec.with_id("explicit-key")
        assert rec.id == hashlib.sha1("explicit-key".encode()).hexdigest()

    def test_with_id_returns_self_for_chaining(self):
        rec = sample_bill()
        assert rec.with_id() is rec


class TestToDictToJson:
    def test_to_dict_strips_nulls_by_default(self):
        rec = sample_bill().with_id()
        data = rec.to_dict()
        assert None not in data.values()
        assert data["bill_number"] == "AB 28"
        assert data["item_number"] == 32

    def test_to_dict_include_nulls(self):
        rec = sample_bill().with_id()
        data = rec.to_dict(include_nulls=True)
        # A field that is None in this record must appear when requested.
        assert "committee" in data
        assert data["committee"] is None

    def test_to_json_is_single_line(self):
        rec = sample_bill()
        line = rec.to_json()
        assert "\n" not in line
        # Re-parse and confirm it is a well-formed object.
        parsed = json.loads(line)
        assert parsed["bill_number"] == "AB 28"

    def test_to_json_sorts_keys(self):
        rec = sample_bill()
        line = rec.to_json()
        parsed = json.loads(line)
        assert list(parsed.keys()) == sorted(parsed.keys())

    def test_to_json_fills_id_when_absent(self):
        rec = sample_bill()     # id is None
        parsed = json.loads(rec.to_json())
        assert "id" in parsed
        assert parsed["id"]

    def test_to_json_omits_nulls(self):
        rec = BillAction(record_type="bill", bill_number="AB 1")
        parsed = json.loads(rec.to_json())
        # null-only fields must not appear
        assert "committee" not in parsed
        assert "hearing_time" not in parsed


class TestFromDict:
    def test_from_dict_ignores_unknown_keys(self):
        data = {"bill_number": "AB 1", "surprise": "extra", "date": "2026-01-01"}
        rec = BillAction.from_dict(data)
        assert rec.bill_number == "AB 1"
        assert rec.date == "2026-01-01"
        assert not hasattr(rec, "surprise")

    def test_from_dict_empty(self):
        rec = BillAction.from_dict({})
        assert rec.record_type == "bill"     # default
        assert rec.house == "assembly"       # default


class TestJsonlRoundTrip:
    def test_write_read_roundtrip(self, tmp_path):
        import scraper.io as io
        recs = [sample_bill(), BillAction(record_type="hearing",
                                         committee="Budget",
                                         hearing_time="9:30am")]
        out = tmp_path / "out.jsonl"
        written = io.write_jsonl(recs, str(out))
        assert written == 2
        rows = read_jsonl(str(out))
        assert len(rows) == 2
        assert rows[0]["bill_number"] == "AB 28"
        assert rows[1]["record_type"] == "hearing"

    def test_jsonl_lines_are_valid_json(self, tmp_path):
        import scraper.io as io
        recs = [sample_bill()] * 5
        out = tmp_path / "out.jsonl"
        io.write_jsonl(recs, str(out))
        with open(str(out), encoding="utf-8") as fh:
            for line in fh:
                assert line.strip()
                json.loads(line)     # raises if any line is malformed