Initial commit

tests/test_schema.py

@@ -0,0 +1,178 @@
+"""Unit tests for scraper.schema (BillAction) and its JSONL contract.
+
+The normalised record schema is spec section 7. This file pins the field set,
+the content-hash id, the null-stripping serialisation, and the round-trip
+through JSON Lines -- the output contract the downstream RDB / UI lanes depend
+on, so it must be stable and asserted explicitly.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+
+import pytest
+
+from conftest import read_jsonl
+from scraper.schema import (
+     ASSEMBLY_DAILY_FILE,
+     BillAction,
+     HOUSE,
+     SENATE_DAILY_FILE_PDF,
+)
+
+
+# A realistic bill record, mirroring the real AB 28 output.
+def sample_bill() -> BillAction:
+    return BillAction(
+         date="2026-08-31",
+         house="assembly",
+         source=ASSEMBLY_DAILY_FILE,
+         record_type="bill",
+         bill_number="AB 28",
+         bill_id="202520260AB28",
+         session="2025-2026",
+         chamber_origin="ASSEMBLY",
+         author="Schiavo et al.",
+         short_title="An act relating to solid waste.",
+         description="An act relating to solid waste.",
+         location="Unfinished Business",
+         item_number=32,
+         supplement=2,
+     )
+
+
+class TestFieldSet:
+    def test_field_names_match_schema(self):
+        names = set(BillAction.field_names())
+        # The core fields every downstream consumer expects (spec section 7).
+        for expected in [
+              "id", "date", "house", "source", "record_type",
+              "bill_number", "bill_id", "session", "chamber_origin",
+              "author", "short_title", "description", "committee",
+              "hearing_time", "location", "item_number", "supplement",
+              "ics_url", "agenda_id", "raw_selector",
+         ]:
+            assert expected in names, f"missing field {expected!r}"
+
+    def test_default_house_is_assembly(self):
+        assert HOUSE == "assembly"
+        assert BillAction().house == "assembly"
+
+    def test_module_source_constants(self):
+        assert {ASSEMBLY_DAILY_FILE, SENATE_DAILY_FILE_PDF} <= {
+             ASSEMBLY_DAILY_FILE, SENATE_DAILY_FILE_PDF, "LEGINFO_BILL_RESULTS",
+         }
+
+
+class TestWithId:
+    def test_id_is_deterministic_content_hash(self):
+        a = sample_bill()
+        b = sample_bill()
+        a.with_id()
+        b.with_id()
+        assert a.id == b.id
+        assert a.id == hashlib.sha1(
+             "|".join([a.house, a.bill_number, a.date, a.committee or ""])
+              .encode("utf-8")).hexdigest()
+
+    def test_missing_components_render_empty_in_hash(self):
+        rec = sample_bill()
+        rec.committee = None
+        rec.date = None
+        rec.with_id()
+        expected = hashlib.sha1(
+             "|".join([rec.house or "", rec.bill_number or "",
+                         rec.date or "", rec.committee or ""]
+                      ).encode("utf-8")).hexdigest()
+        assert rec.id == expected
+
+    def test_explicit_content_used_when_supplied(self):
+        rec = BillAction()
+        rec.with_id("explicit-key")
+        assert rec.id == hashlib.sha1("explicit-key".encode()).hexdigest()
+
+    def test_with_id_returns_self_for_chaining(self):
+        rec = sample_bill()
+        assert rec.with_id() is rec
+
+
+class TestToDictToJson:
+    def test_to_dict_strips_nulls_by_default(self):
+        rec = sample_bill().with_id()
+        data = rec.to_dict()
+        assert None not in data.values()
+        assert data["bill_number"] == "AB 28"
+        assert data["item_number"] == 32
+
+    def test_to_dict_include_nulls(self):
+        rec = sample_bill().with_id()
+        data = rec.to_dict(include_nulls=True)
+        # A field that is None in this record must appear when requested.
+        assert "committee" in data
+        assert data["committee"] is None
+
+    def test_to_json_is_single_line(self):
+        rec = sample_bill()
+        line = rec.to_json()
+        assert "\n" not in line
+        # Re-parse and confirm it is a well-formed object.
+        parsed = json.loads(line)
+        assert parsed["bill_number"] == "AB 28"
+
+    def test_to_json_sorts_keys(self):
+        rec = sample_bill()
+        line = rec.to_json()
+        parsed = json.loads(line)
+        assert list(parsed.keys()) == sorted(parsed.keys())
+
+    def test_to_json_fills_id_when_absent(self):
+        rec = sample_bill()     # id is None
+        parsed = json.loads(rec.to_json())
+        assert "id" in parsed
+        assert parsed["id"]
+
+    def test_to_json_omits_nulls(self):
+        rec = BillAction(record_type="bill", bill_number="AB 1")
+        parsed = json.loads(rec.to_json())
+        # null-only fields must not appear
+        assert "committee" not in parsed
+        assert "hearing_time" not in parsed
+
+
+class TestFromDict:
+    def test_from_dict_ignores_unknown_keys(self):
+        data = {"bill_number": "AB 1", "surprise": "extra", "date": "2026-01-01"}
+        rec = BillAction.from_dict(data)
+        assert rec.bill_number == "AB 1"
+        assert rec.date == "2026-01-01"
+        assert not hasattr(rec, "surprise")
+
+    def test_from_dict_empty(self):
+        rec = BillAction.from_dict({})
+        assert rec.record_type == "bill"     # default
+        assert rec.house == "assembly"       # default
+
+
+class TestJsonlRoundTrip:
+    def test_write_read_roundtrip(self, tmp_path):
+        import scraper.io as io
+        recs = [sample_bill(), BillAction(record_type="hearing",
+                                         committee="Budget",
+                                         hearing_time="9:30am")]
+        out = tmp_path / "out.jsonl"
+        written = io.write_jsonl(recs, str(out))
+        assert written == 2
+        rows = read_jsonl(str(out))
+        assert len(rows) == 2
+        assert rows[0]["bill_number"] == "AB 28"
+        assert rows[1]["record_type"] == "hearing"
+
+    def test_jsonl_lines_are_valid_json(self, tmp_path):
+        import scraper.io as io
+        recs = [sample_bill()] * 5
+        out = tmp_path / "out.jsonl"
+        io.write_jsonl(recs, str(out))
+        with open(str(out), encoding="utf-8") as fh:
+            for line in fh:
+                assert line.strip()
+                json.loads(line)     # raises if any line is malformed