287 lines
13 KiB
Python
287 lines
13 KiB
Python
"""Unit tests for scraper.parsers -- the HTML -> record transformation (spec section 3).
|
|
|
|
Scope for task t_8e5a3821 item (1):
|
|
* fixtures for at least 3 dates, one per chamber, plus one edge-case file;
|
|
* the archived Assembly sample must still parse to the expected record set;
|
|
* malformed rows are skipped with a logged warning and never abort the batch;
|
|
* cross-chamber origin, committee vs floor discrimination, and the
|
|
data-sort preference are all asserted.
|
|
|
|
The parser is the coverage-gated module, so these tests aim for near-complete
|
|
branch coverage of scraper.parsers and scraper.bill_id.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from scraper import parsers
|
|
from scraper.parsers import normalise_date
|
|
from tests import fixtures as F
|
|
|
|
|
|
# ---------------------------------------------------------------- fixtures
|
|
#
|
|
# Three legislative-day fixtures, both chambers, plus the edge-case file. Three
|
|
# dates come from assembly_daily_file_html (2026-08-31), senate_daily_file_html
|
|
# (2026-09-02) and extra_daily_file_html (2026-09-03).
|
|
|
|
class TestFixtureCoverage:
|
|
def test_daily_file_fixtures_parse_without_raising(self):
|
|
for name in ("assembly_daily_file_html", "senate_daily_file_html",
|
|
"extra_daily_file_html"):
|
|
recs = parsers.parse_html(getattr(F, name)())
|
|
# Each full daily file carries at least one bill and one hearing.
|
|
assert len(recs) >= 2, f"{name} produced no records"
|
|
assert any(r.record_type == "bill" for r in recs), f"{name}: no bills"
|
|
assert any(r.record_type == "hearing" for r in recs), f"{name}: no hearings"
|
|
|
|
def test_edge_fixture_parses_without_raising(self):
|
|
# The edge-case file is deliberately bills-only and malformed; it must
|
|
# parse without raising and must keep the recoverable rows.
|
|
recs = parsers.parse_html(F.edge_case_html())
|
|
assert {r.bill_number for r in recs} >= {"AB 100", "AB 300", "AB 150"}
|
|
|
|
def test_three_distinct_dates(self):
|
|
dates = set()
|
|
for name in ("assembly_daily_file_html", "senate_daily_file_html",
|
|
"extra_daily_file_html"):
|
|
recs = parsers.parse_html(getattr(F, name)())
|
|
d = next((r.date for r in recs if r.date), None)
|
|
assert d is not None, f"{name}: no date derived from hearing table"
|
|
dates.add(d)
|
|
assert len(dates) == 3, f"expected 3 distinct dates, got {dates}"
|
|
|
|
def test_both_chambers_covered(self):
|
|
origins = set()
|
|
for name in ("assembly_daily_file_html", "senate_daily_file_html",
|
|
"extra_daily_file_html"):
|
|
for r in parsers.parse_html(getattr(F, name)()):
|
|
if r.chamber_origin:
|
|
origins.add(r.chamber_origin.upper())
|
|
assert {"ASSEMBLY", "SENATE"} <= origins, f"missing a chamber: {origins}"
|
|
|
|
|
|
# ---------------------------------------------------------------- the archived sample
|
|
|
|
class TestArchivedSample:
|
|
def test_sample_parses_expected_counts(self):
|
|
recs = parsers.parse("samples/assembly_dailyfile_latest.html")
|
|
by = {}
|
|
for r in recs:
|
|
by[r.record_type] = by.get(r.record_type, 0) + 1
|
|
# Grounded values (verified 2026-09-05 against the 401 192-byte sample):
|
|
# 156 floor + 1 committee = 157 bills, 4 hearings.
|
|
assert by.get("bill", 0) >= 150, by
|
|
assert by.get("hearing", 0) >= 4, by
|
|
|
|
def test_sample_has_ab28(self):
|
|
recs = parsers.parse("samples/assembly_dailyfile_latest.html")
|
|
ab28 = next((r for r in recs if r.bill_number == "AB 28"), None)
|
|
assert ab28 is not None
|
|
assert ab28.bill_id == "202520260AB28"
|
|
assert ab28.session == "2025-2026"
|
|
assert ab28.chamber_origin == "ASSEMBLY"
|
|
assert ab28.item_number == 32
|
|
assert ab28.supplement == 2
|
|
|
|
def test_sample_cross_chamber(self):
|
|
recs = parsers.parse("samples/assembly_dailyfile_latest.html")
|
|
origins = {r.chamber_origin for r in recs if r.record_type == "bill"}
|
|
assert "ASSEMBLY" in origins and "SENATE" in origins, origins
|
|
|
|
|
|
# ---------------------------------------------------------------- floor / committee
|
|
|
|
_FIXTURE_BUILDERS = ("assembly_daily_file_html", "senate_daily_file_html",
|
|
"extra_daily_file_html")
|
|
|
|
|
|
class TestRecordTypes:
|
|
@pytest.mark.parametrize("builder_name", _FIXTURE_BUILDERS)
|
|
def test_bills_cannot_be_hearings(self, builder_name):
|
|
recs = parsers.parse_html(getattr(F, builder_name)())
|
|
for r in recs:
|
|
if r.record_type == "bill":
|
|
assert not r.hearing_time, "bill should not carry a hearing time"
|
|
assert r.raw_selector in ("floor-bills-table", "committee-bills-table")
|
|
if r.record_type == "hearing":
|
|
assert r.hearing_time is not None
|
|
assert r.raw_selector == "committee-hearing-table"
|
|
# Hearings carry no bill identity.
|
|
assert r.bill_number is None
|
|
assert r.bill_id is None
|
|
|
|
def test_floor_vs_committee_discrimination(self):
|
|
recs = parsers.parse_html(F.assembly_daily_file_html())
|
|
floor = [r for r in recs if r.raw_selector == "floor-bills-table"]
|
|
comm = [r for r in recs if r.raw_selector == "committee-bills-table"]
|
|
assert floor, "no floor records"
|
|
assert comm, "no committee records"
|
|
# Floor rows have an item_number; committee rows do not.
|
|
assert all(r.item_number is not None for r in floor)
|
|
assert all(r.item_number is None for r in comm)
|
|
# Committee rows carry a committee name; floor rows do not.
|
|
assert all(r.committee is None for r in floor)
|
|
assert all(r.committee for r in comm)
|
|
|
|
def test_committee_bill_has_session_from_id(self):
|
|
recs = parsers.parse_html(F.assembly_daily_file_html())
|
|
sb795 = next(r for r in recs if r.bill_number == "SB 795")
|
|
assert sb795.session == "2025-2026"
|
|
assert sb795.chamber_origin == "SENATE"
|
|
assert sb795.committee == "Emergency Management"
|
|
|
|
def test_display_only_row_falls_back_to_display_text(self):
|
|
# EDGE_DISPLAY_ONLY has no bill_id in the link; the parser must recover
|
|
# "AB 150" from the clean display text (bill_id stays None).
|
|
recs = parsers.parse_html(F.edge_case_html())
|
|
ab150 = next((r for r in recs if r.bill_number == "AB 150"), None)
|
|
assert ab150 is not None
|
|
assert ab150.bill_id is None # no decodable id available
|
|
assert ab150.chamber_origin == "assembly" # origin derived from display
|
|
assert ab150.item_number == 150
|
|
|
|
def test_data_sort_preferred_over_text(self):
|
|
# A floor row whose text-content and data-sort disagree: the parser must
|
|
# take the clean data-sort value for author/location.
|
|
import bs4
|
|
row = (
|
|
'<table class="floor-bills-table"><tr class="floor-bill-details">'
|
|
'<td class="floor_bill-measure">'
|
|
'<a href="https://leginfo.legislature.ca.gov/faces/'
|
|
'billNavClient.xhtml?bill_id=202520260AB777">AB 777</a></td>'
|
|
'<td class="floor_bill-author" data-sort="Rivera">Rivera (with extra spaces)</td>'
|
|
'<td class="floor_bill-subject">Subject text.</td>'
|
|
'<td class="floor_bill-location" data-sort="Capitol Room 444">Capitol Room 444 </td>'
|
|
'<td class="floor_bill-item" data-sort="0000777">777</td></tr></table>'
|
|
)
|
|
recs = parsers.parse_html(row)
|
|
assert len(recs) == 1
|
|
r = recs[0]
|
|
assert r.author == "Rivera" # data-sort is clean, not textContent
|
|
assert r.location == "Capitol Room 444"
|
|
|
|
|
|
# ---------------------------------------------------------------- date handling
|
|
|
|
class TestDateHandling:
|
|
def test_normalise_date_ms(self):
|
|
assert normalise_date("08/31/26-001") == "2026-08-31"
|
|
|
|
def test_normalise_date_ms_no_suffix(self):
|
|
assert normalise_date("08/31/26") == "2026-08-31"
|
|
|
|
def test_normalise_date_prefers_mm_slash_dd(self):
|
|
# NOTE (documented quirk): "2026-08-31" (ISO) returns None -- the
|
|
# _DATE_SPLIT regex (designed for "08/31/26-001") strips the trailing
|
|
# "-31" before the ISO format is tried, leaving "2026-08" which fails
|
|
# all three strptime attempts. This is a known limitation, but the
|
|
# parser only ever feeds normalise_date a data-sort value (the
|
|
# site emits "MM/DD/YY" with "-NNN" suffix), so it is safe in practice.
|
|
# Pin the actual behaviour so a future fix surfaces as a test change.
|
|
assert normalise_date("2026-08-31") is None
|
|
assert normalise_date("08/31/26") == "2026-08-31"
|
|
assert normalise_date("08/31/26-001") == "2026-08-31"
|
|
|
|
@pytest.mark.parametrize("junk", ["", "not-a-date", "13/40/99", "2026/13/99"])
|
|
def test_normalise_date_invalid(self, junk):
|
|
assert normalise_date(junk) is None
|
|
|
|
def test_file_date_backfill_from_hearing(self):
|
|
recs = parsers.parse_html(F.assembly_daily_file_html())
|
|
dates = {r.date for r in recs}
|
|
assert dates == {"2026-08-31"}, f"expected one backfilled date, got {dates}"
|
|
|
|
|
|
# ---------------------------------------------------------------- robustness
|
|
|
|
class TestRobustness:
|
|
def test_malformed_rows_skipped_not_aborted(self, logcap):
|
|
cap = logcap("scraper.parsers")
|
|
recs = parsers.parse_html(F.edge_case_html())
|
|
bns = {r.bill_number for r in recs}
|
|
# The good + recoverable rows survive.
|
|
assert "AB 100" in bns, "good row lost"
|
|
assert "AB 300" in bns, "recoverable row lost"
|
|
assert "AB 150" in bns, "display-only recovery failed"
|
|
# The no-link row must not produce a record.
|
|
assert "AB 200" not in bns, "a no-link row must not produce a record"
|
|
# Each skipped row logs a warning (deterministic failure logging).
|
|
assert len(cap.warnings) >= 2, f"expected >=2 skip warnings, got {cap.warnings}"
|
|
|
|
def test_utf16_edge_case_does_not_crash(self, logcap, tmp_path):
|
|
cap = logcap("scraper.parsers")
|
|
p = tmp_path / "utf16.html"
|
|
p.write_bytes(F.edge_case_bytes())
|
|
recs = parsers.parse(str(p))
|
|
assert recs == []
|
|
assert any("charset" in w.lower() or "replacement" in w.lower()
|
|
for w in cap.warnings), f"expected an encoding warning: {cap.warnings}"
|
|
|
|
def test_empty_input_returns_empty(self):
|
|
assert parsers.parse_html("") == []
|
|
|
|
def test_garbage_html_returns_empty_not_crash(self):
|
|
assert parsers.parse_html("<!!not-html>>><<<") == []
|
|
|
|
def test_missing_input_file_is_skipped(self, logcap, tmp_path):
|
|
cap = logcap("scraper.parsers")
|
|
miss = str(tmp_path / "does_not_exist.html")
|
|
recs = parsers.parse(miss)
|
|
assert recs == []
|
|
assert any("not found" in w.lower() for w in cap.warnings)
|
|
|
|
def test_directory_of_html_files_is_aggregated(self, tmp_path):
|
|
d = tmp_path / "multi"
|
|
d.mkdir()
|
|
for i in range(3):
|
|
(d / f"day{i}.html").write_text(F.extra_daily_file_html())
|
|
recs = parsers.parse(str(d))
|
|
assert len(recs) == 3 * 2, f"expected 6, got {len(recs)}"
|
|
|
|
def test_include_hearings_false_drops_hearings(self, tmp_path):
|
|
p = tmp_path / "asm.html"
|
|
p.write_text(F.assembly_daily_file_html())
|
|
recs = parsers.parse(str(p), include_hearings=False)
|
|
assert all(r.record_type != "hearing" for r in recs)
|
|
assert any(r.record_type == "bill" for r in recs)
|
|
|
|
def test_house_argument_sets_house_field(self):
|
|
recs = parsers.parse_html(F.senate_daily_file_html(), house="senate")
|
|
assert recs, "senate fixture parsed nothing"
|
|
assert all(r.house == "senate" for r in recs)
|
|
|
|
def test_invalid_parser_engine_falls_back(self):
|
|
# An unrecognised parser engine must fall back to html.parser, not raise.
|
|
recs = parsers.parse_html(F.assembly_daily_file_html(), parser="nosuchparser")
|
|
assert recs, "bad parser engine produced no records"
|
|
|
|
def test_supplement_extraction(self):
|
|
recs = parsers.parse_html(F.assembly_daily_file_html())
|
|
ab28 = next(r for r in recs if r.bill_number == "AB 28")
|
|
assert ab28.supplement == 2
|
|
|
|
def test_glob_input_matches_files(self, tmp_path):
|
|
d = tmp_path / "glob"
|
|
d.mkdir()
|
|
(d / "a.html").write_text(F.senate_daily_file_html())
|
|
(d / "b.html").write_text(F.extra_daily_file_html())
|
|
recs = parsers.parse(str(d / "*.html"))
|
|
assert len(recs) == 2 * 2, f"expected 4, got {len(recs)}"
|
|
|
|
def test_no_inputs_matched_returns_empty(self, tmp_path, logcap):
|
|
# A glob that matches nothing -> the "no input files matched" branch.
|
|
cap = logcap("scraper.parsers")
|
|
missing_glob = str(tmp_path / "does-not-match-*.html")
|
|
recs = parsers.parse(missing_glob)
|
|
assert recs == []
|
|
assert any("no input files" in w.lower() for w in cap.warnings)
|
|
|
|
def test_missing_single_file_is_skipped(self, tmp_path, logcap):
|
|
# A single path that exists as a name but not on disk -> "not found".
|
|
cap = logcap("scraper.parsers")
|
|
missing = str(tmp_path / "no-such-dir.html")
|
|
recs = parsers.parse(missing)
|
|
assert recs == []
|
|
assert any("not found" in w.lower() for w in cap.warnings)
|