"""Tests for the relational correlation lane (spec 2.5, task t_2736f40d). Covers the acceptance criteria: (a) migrate is idempotent on a fresh db and on an existing db; (b) a bill with a non-null ``committee`` ends up in ``bill_committees``; (c) a committee member (from a roster) appears in the committee<->member join; (d) the flat ``v_daily_file_records`` view preserves the flat contract. Plus the "create new info / never fabricate" rule: a committee with no roster source is created with an EMPTY member set (a WARN is logged, no members invented). """ from __future__ import annotations import json from scraper import relational from scraper import relate_cli from scraper.schema import BillAction from scraper.io import write_sqlite def build_flat_db(path): recs = [ BillAction(date="2026-08-31", house="assembly", record_type="hearing", committee="Emergency Management", hearing_time="9:30am", location="Room 444", source="ASSEMBLY_DAILY_FILE"), BillAction(date="2026-08-31", house="assembly", record_type="bill", bill_number="AB 28", bill_id="202520260AB28", session="2025-2026", chamber_origin="assembly", author="Macedo et al.", short_title="solid waste", committee="Emergency Management", source="ASSEMBLY_DAILY_FILE"), BillAction(date="2026-08-31", house="assembly", record_type="bill", bill_number="AB 568", bill_id="202520260AB568", session="2025-2026", chamber_origin="assembly", author="Macedo", short_title="water districts", source="ASSEMBLY_DAILY_FILE"), BillAction(date="2026-08-31", house="assembly", record_type="bill", bill_number="SB 133", bill_id="202520260SB133", session="2025-2026", chamber_origin="senate", author="Committee on Budget and Fiscal Review", short_title="budget fix", source="ASSEMBLY_DAILY_FILE"), ] write_sqlite(recs, path, table=relational.FLAT_TABLE) return path def write_roster_file(path): data = [ {"committee": "Emergency Management", "session": "2025-2026", "chamber": "assembly", "members": [ {"name": "Smith", "role": "Chair"}, {"name": "Jones", "role": "Member"}, ]}, ] with open(path, "w", encoding="utf-8") as fh: json.dump(data, fh) return path def _fetch_rows(db, sql): c = relational.connect(db) try: cur = c.execute(sql) return [tuple(r) for r in cur.fetchall()] finally: c.close() def _count_table(db, table): c = relational.connect(db) try: cur = c.execute("SELECT COUNT(*) FROM " + table) return cur.fetchone()[0] finally: c.close() def _columns(db, table): c = relational.connect(db) try: cur = c.execute("PRAGMA table_info(" + table + ")") return [row[1] for row in cur.fetchall()] finally: c.close() def _is_view(db, name): c = relational.connect(db) try: cur = c.execute( "SELECT 1 FROM sqlite_master WHERE type='view' AND name=?", (name,)) return cur.fetchone() is not None finally: c.close() # --------------------------------------------------------------------------- # (a) idempotency: fresh db, then re-run # --------------------------------------------------------------------------- def test_migrate_fresh_db_is_idempotent(tmp_path): db = str(tmp_path / "rel.sqlite") s1 = relational.migrate(db) assert s1["migrated"] is True for obj in ("bills", "committees", "members", "committee_members", "bill_committees", "schema_migrations", "v_daily_file_records"): assert obj in s1["objects"], obj s2 = relational.migrate(db) assert set(s1["objects"]) == set(s2["objects"]) assert s2["flat_rows"] == 0 assert s2["version"] == s1["version"] def test_migrate_on_existing_flat_db_preserves_rows(tmp_path): db = str(tmp_path / "rel_existing.sqlite") build_flat_db(db) assert _count_table(db, relational.FLAT_TABLE) == 4 s = relational.migrate(db) assert s["flat_rows"] == 4 assert _count_table(db, relational.FLAT_TABLE) == 4 for obj in ("bills", "committees", "bill_committees"): assert obj in s["objects"] def test_correlate_is_idempotent(tmp_path): db = str(tmp_path / "rel_c.sqlite") build_flat_db(db) relational.migrate(db) c1 = relational.correlate(db, quiet=True) c2 = relational.correlate(db, quiet=True) assert c1["bills"] == c2["bills"] assert c1["committees"] == c2["committees"] assert c1["bill_committees"] == c2["bill_committees"] assert c1["members"] == c2["members"] assert c1["committee_members"] == c2["committee_members"] # --------------------------------------------------------------------------- # (b) bill<->committee # --------------------------------------------------------------------------- def test_bill_with_committee_lands_in_bill_committees(tmp_path): db = str(tmp_path / "rel_bc.sqlite") build_flat_db(db) relational.correlate(db, quiet=True) rows = _fetch_rows(db, "SELECT bill_id, committee " "FROM bill_committees WHERE bill_id='202520260AB28'") assert rows, "expected AB 28 to be linked to a committee" assert rows[0][1] == "emergency-management" assert _count_table(db, "committees") >= 1 def test_rebuild_clears_then_backfills(tmp_path): db = str(tmp_path / "rel_rc.sqlite") build_flat_db(db) relational.correlate(db, quiet=True) first = _count_table(db, "bill_committees") c = relational.connect(db) c.execute("INSERT INTO bill_committees (bill_id, committee, session, " "chamber) VALUES ('X999','bogus','','unknown')") c.commit() c.close() assert _count_table(db, "bill_committees") == first + 1 relational.correlate(db, rebuild=True, quiet=True) assert _count_table(db, "bill_committees") == first # --------------------------------------------------------------------------- # (c) committee<->member + (full 3-way join) # --------------------------------------------------------------------------- def test_committee_member_appears_in_join(tmp_path): db = str(tmp_path / "rel_cm.sqlite") build_flat_db(db) roster = write_roster_file(str(tmp_path / "rosters.json")) relational.correlate(db, rosters=roster, quiet=True) rows = _fetch_rows(db, "SELECT member, role FROM committee_members " "WHERE committee='emergency-management'") names = {r[0] for r in rows} assert names == {"smith", "jones"}, names joined = _fetch_rows(db, "SELECT bc.bill_id, c.name_raw, m.full_name, cm.role " "FROM bill_committees bc " "JOIN committees c " "ON c.name_key=bc.committee AND c.chamber=bc.chamber " "AND c.session=bc.session " "JOIN committee_members cm " "ON cm.committee=c.name_key AND cm.chamber=c.chamber " "AND cm.session=c.session " "JOIN members m ON m.name_norm=cm.member") got = {(r[0], r[2]) for r in joined} assert ("202520260AB28", "Smith") in got assert ("202520260AB28", "Jones") in got # --------------------------------------------------------------------------- # never-fabricate rule # --------------------------------------------------------------------------- def test_committee_without_roster_has_no_members(tmp_path, logcap): db = str(tmp_path / "rel_nf.sqlite") build_flat_db(db) cap = logcap("scraper.relate") roster = write_roster_file(str(tmp_path / "rosters.json")) counts = relational.correlate(db, rosters=roster, quiet=True) cap.remove() members_for = _fetch_rows(db, "SELECT COUNT(*) FROM committee_members cm " "JOIN committees c ON c.name_key=cm.committee " "WHERE c.name_raw='Budget and Fiscal Review'") assert members_for[0][0] == 0, "members must NOT be fabricated" assert counts["committees_missing_roster"] >= 1 joined_cap = " ".join(cap.warnings).lower() assert ("no roster" in joined_cap) or ("fabricat" in joined_cap), \ cap.warnings # --------------------------------------------------------------------------- # (d) flat-view contract preserved # --------------------------------------------------------------------------- def test_flat_view_matches_table(tmp_path): db = str(tmp_path / "rel_view.sqlite") build_flat_db(db) relational.migrate(db) t_cols = _columns(db, relational.FLAT_TABLE) v_cols = _columns(db, "v_daily_file_records") assert t_cols == v_cols, (t_cols, v_cols) t_rows = _fetch_rows(db, "SELECT * FROM " + relational.FLAT_TABLE + " ORDER BY 1") v_rows = _fetch_rows(db, "SELECT * FROM v_daily_file_records ORDER BY 1") assert t_rows == v_rows, "view row-set must equal the flat table" assert _is_view(db, "v_daily_file_records") # --------------------------------------------------------------------------- # CLI / wiring: additive registration must not clobber siblings # --------------------------------------------------------------------------- def test_cli_registers_migrate_and_correlate_without_clobbering(): from scraper import cli parser = cli.build_parser() a = parser.parse_args(["migrate", "--db", "x.sqlite"]) assert a.command == "migrate" b = parser.parse_args(["correlate", "--db", "x.sqlite", "--roster", "r.json"]) assert b.command == "correlate" assert parser.parse_args(["parse", "--input", "f.html"]).command == "parse" assert parser.parse_args(["ingest", "--date", "2026-09-01"]).command == "ingest" assert parser.parse_args(["list"]).command == "list" def test_migrate_then_correlate_cli_end_to_end(tmp_path): import argparse db = str(tmp_path / "rel_cli.sqlite") build_flat_db(db) roster = write_roster_file(str(tmp_path / "rosters.json")) ns = argparse.Namespace(db=db, check=False) assert relate_cli.handle_migrate(ns) == 0 ns2 = argparse.Namespace(db=db, roster=roster, rebuild=False) assert relate_cli.handle_correlate(ns2) == 0 assert _count_table(db, "committee_members") == 2