270 lines
10 KiB
Python
270 lines
10 KiB
Python
"""Tests for the relational correlation lane (spec 2.5, task t_2736f40d).
|
|
|
|
Covers the acceptance criteria:
|
|
(a) migrate is idempotent on a fresh db and on an existing db;
|
|
(b) a bill with a non-null ``committee`` ends up in ``bill_committees``;
|
|
(c) a committee member (from a roster) appears in the committee<->member join;
|
|
(d) the flat ``v_daily_file_records`` view preserves the flat contract.
|
|
Plus the "create new info / never fabricate" rule: a committee with no roster
|
|
source is created with an EMPTY member set (a WARN is logged, no members
|
|
invented).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
from scraper import relational
|
|
from scraper import relate_cli
|
|
from scraper.schema import BillAction
|
|
from scraper.io import write_sqlite
|
|
|
|
|
|
def build_flat_db(path):
|
|
recs = [
|
|
BillAction(date="2026-08-31", house="assembly",
|
|
record_type="hearing", committee="Emergency Management",
|
|
hearing_time="9:30am", location="Room 444",
|
|
source="ASSEMBLY_DAILY_FILE"),
|
|
BillAction(date="2026-08-31", house="assembly", record_type="bill",
|
|
bill_number="AB 28", bill_id="202520260AB28",
|
|
session="2025-2026", chamber_origin="assembly",
|
|
author="Macedo et al.", short_title="solid waste",
|
|
committee="Emergency Management",
|
|
source="ASSEMBLY_DAILY_FILE"),
|
|
BillAction(date="2026-08-31", house="assembly", record_type="bill",
|
|
bill_number="AB 568", bill_id="202520260AB568",
|
|
session="2025-2026", chamber_origin="assembly",
|
|
author="Macedo", short_title="water districts",
|
|
source="ASSEMBLY_DAILY_FILE"),
|
|
BillAction(date="2026-08-31", house="assembly", record_type="bill",
|
|
bill_number="SB 133", bill_id="202520260SB133",
|
|
session="2025-2026", chamber_origin="senate",
|
|
author="Committee on Budget and Fiscal Review",
|
|
short_title="budget fix", source="ASSEMBLY_DAILY_FILE"),
|
|
]
|
|
write_sqlite(recs, path, table=relational.FLAT_TABLE)
|
|
return path
|
|
|
|
|
|
def write_roster_file(path):
|
|
data = [
|
|
{"committee": "Emergency Management",
|
|
"session": "2025-2026",
|
|
"chamber": "assembly",
|
|
"members": [
|
|
{"name": "Smith", "role": "Chair"},
|
|
{"name": "Jones", "role": "Member"},
|
|
]},
|
|
]
|
|
with open(path, "w", encoding="utf-8") as fh:
|
|
json.dump(data, fh)
|
|
return path
|
|
|
|
|
|
def _fetch_rows(db, sql):
|
|
c = relational.connect(db)
|
|
try:
|
|
cur = c.execute(sql)
|
|
return [tuple(r) for r in cur.fetchall()]
|
|
finally:
|
|
c.close()
|
|
|
|
|
|
def _count_table(db, table):
|
|
c = relational.connect(db)
|
|
try:
|
|
cur = c.execute("SELECT COUNT(*) FROM " + table)
|
|
return cur.fetchone()[0]
|
|
finally:
|
|
c.close()
|
|
|
|
|
|
def _columns(db, table):
|
|
c = relational.connect(db)
|
|
try:
|
|
cur = c.execute("PRAGMA table_info(" + table + ")")
|
|
return [row[1] for row in cur.fetchall()]
|
|
finally:
|
|
c.close()
|
|
|
|
|
|
def _is_view(db, name):
|
|
c = relational.connect(db)
|
|
try:
|
|
cur = c.execute(
|
|
"SELECT 1 FROM sqlite_master WHERE type='view' AND name=?",
|
|
(name,))
|
|
return cur.fetchone() is not None
|
|
finally:
|
|
c.close()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# (a) idempotency: fresh db, then re-run
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_migrate_fresh_db_is_idempotent(tmp_path):
|
|
db = str(tmp_path / "rel.sqlite")
|
|
s1 = relational.migrate(db)
|
|
assert s1["migrated"] is True
|
|
for obj in ("bills", "committees", "members", "committee_members",
|
|
"bill_committees", "schema_migrations", "v_daily_file_records"):
|
|
assert obj in s1["objects"], obj
|
|
s2 = relational.migrate(db)
|
|
assert set(s1["objects"]) == set(s2["objects"])
|
|
assert s2["flat_rows"] == 0
|
|
assert s2["version"] == s1["version"]
|
|
|
|
|
|
def test_migrate_on_existing_flat_db_preserves_rows(tmp_path):
|
|
db = str(tmp_path / "rel_existing.sqlite")
|
|
build_flat_db(db)
|
|
assert _count_table(db, relational.FLAT_TABLE) == 4
|
|
s = relational.migrate(db)
|
|
assert s["flat_rows"] == 4
|
|
assert _count_table(db, relational.FLAT_TABLE) == 4
|
|
for obj in ("bills", "committees", "bill_committees"):
|
|
assert obj in s["objects"]
|
|
|
|
|
|
def test_correlate_is_idempotent(tmp_path):
|
|
db = str(tmp_path / "rel_c.sqlite")
|
|
build_flat_db(db)
|
|
relational.migrate(db)
|
|
c1 = relational.correlate(db, quiet=True)
|
|
c2 = relational.correlate(db, quiet=True)
|
|
assert c1["bills"] == c2["bills"]
|
|
assert c1["committees"] == c2["committees"]
|
|
assert c1["bill_committees"] == c2["bill_committees"]
|
|
assert c1["members"] == c2["members"]
|
|
assert c1["committee_members"] == c2["committee_members"]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# (b) bill<->committee
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_bill_with_committee_lands_in_bill_committees(tmp_path):
|
|
db = str(tmp_path / "rel_bc.sqlite")
|
|
build_flat_db(db)
|
|
relational.correlate(db, quiet=True)
|
|
rows = _fetch_rows(db,
|
|
"SELECT bill_id, committee "
|
|
"FROM bill_committees WHERE bill_id='202520260AB28'")
|
|
assert rows, "expected AB 28 to be linked to a committee"
|
|
assert rows[0][1] == "emergency-management"
|
|
assert _count_table(db, "committees") >= 1
|
|
|
|
|
|
def test_rebuild_clears_then_backfills(tmp_path):
|
|
db = str(tmp_path / "rel_rc.sqlite")
|
|
build_flat_db(db)
|
|
relational.correlate(db, quiet=True)
|
|
first = _count_table(db, "bill_committees")
|
|
c = relational.connect(db)
|
|
c.execute("INSERT INTO bill_committees (bill_id, committee, session, "
|
|
"chamber) VALUES ('X999','bogus','','unknown')")
|
|
c.commit()
|
|
c.close()
|
|
assert _count_table(db, "bill_committees") == first + 1
|
|
relational.correlate(db, rebuild=True, quiet=True)
|
|
assert _count_table(db, "bill_committees") == first
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# (c) committee<->member + (full 3-way join)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_committee_member_appears_in_join(tmp_path):
|
|
db = str(tmp_path / "rel_cm.sqlite")
|
|
build_flat_db(db)
|
|
roster = write_roster_file(str(tmp_path / "rosters.json"))
|
|
relational.correlate(db, rosters=roster, quiet=True)
|
|
rows = _fetch_rows(db,
|
|
"SELECT member, role FROM committee_members "
|
|
"WHERE committee='emergency-management'")
|
|
names = {r[0] for r in rows}
|
|
assert names == {"smith", "jones"}, names
|
|
joined = _fetch_rows(db,
|
|
"SELECT bc.bill_id, c.name_raw, m.full_name, cm.role "
|
|
"FROM bill_committees bc "
|
|
"JOIN committees c "
|
|
"ON c.name_key=bc.committee AND c.chamber=bc.chamber "
|
|
"AND c.session=bc.session "
|
|
"JOIN committee_members cm "
|
|
"ON cm.committee=c.name_key AND cm.chamber=c.chamber "
|
|
"AND cm.session=c.session "
|
|
"JOIN members m ON m.name_norm=cm.member")
|
|
got = {(r[0], r[2]) for r in joined}
|
|
assert ("202520260AB28", "Smith") in got
|
|
assert ("202520260AB28", "Jones") in got
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# never-fabricate rule
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_committee_without_roster_has_no_members(tmp_path, logcap):
|
|
db = str(tmp_path / "rel_nf.sqlite")
|
|
build_flat_db(db)
|
|
cap = logcap("scraper.relate")
|
|
roster = write_roster_file(str(tmp_path / "rosters.json"))
|
|
counts = relational.correlate(db, rosters=roster, quiet=True)
|
|
cap.remove()
|
|
members_for = _fetch_rows(db,
|
|
"SELECT COUNT(*) FROM committee_members cm "
|
|
"JOIN committees c ON c.name_key=cm.committee "
|
|
"WHERE c.name_raw='Budget and Fiscal Review'")
|
|
assert members_for[0][0] == 0, "members must NOT be fabricated"
|
|
assert counts["committees_missing_roster"] >= 1
|
|
joined_cap = " ".join(cap.warnings).lower()
|
|
assert ("no roster" in joined_cap) or ("fabricat" in joined_cap), \
|
|
cap.warnings
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# (d) flat-view contract preserved
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_flat_view_matches_table(tmp_path):
|
|
db = str(tmp_path / "rel_view.sqlite")
|
|
build_flat_db(db)
|
|
relational.migrate(db)
|
|
t_cols = _columns(db, relational.FLAT_TABLE)
|
|
v_cols = _columns(db, "v_daily_file_records")
|
|
assert t_cols == v_cols, (t_cols, v_cols)
|
|
t_rows = _fetch_rows(db,
|
|
"SELECT * FROM " + relational.FLAT_TABLE + " ORDER BY 1")
|
|
v_rows = _fetch_rows(db, "SELECT * FROM v_daily_file_records ORDER BY 1")
|
|
assert t_rows == v_rows, "view row-set must equal the flat table"
|
|
assert _is_view(db, "v_daily_file_records")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CLI / wiring: additive registration must not clobber siblings
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_cli_registers_migrate_and_correlate_without_clobbering():
|
|
from scraper import cli
|
|
parser = cli.build_parser()
|
|
a = parser.parse_args(["migrate", "--db", "x.sqlite"])
|
|
assert a.command == "migrate"
|
|
b = parser.parse_args(["correlate", "--db", "x.sqlite", "--roster", "r.json"])
|
|
assert b.command == "correlate"
|
|
assert parser.parse_args(["parse", "--input", "f.html"]).command == "parse"
|
|
assert parser.parse_args(["ingest", "--date", "2026-09-01"]).command == "ingest"
|
|
assert parser.parse_args(["list"]).command == "list"
|
|
|
|
|
|
def test_migrate_then_correlate_cli_end_to_end(tmp_path):
|
|
import argparse
|
|
db = str(tmp_path / "rel_cli.sqlite")
|
|
build_flat_db(db)
|
|
roster = write_roster_file(str(tmp_path / "rosters.json"))
|
|
ns = argparse.Namespace(db=db, check=False)
|
|
assert relate_cli.handle_migrate(ns) == 0
|
|
ns2 = argparse.Namespace(db=db, roster=roster, rebuild=False)
|
|
assert relate_cli.handle_correlate(ns2) == 0
|
|
assert _count_table(db, "committee_members") == 2
|