256 lines
11 KiB
Python
256 lines
11 KiB
Python
"""Fixture builders for the QA test-suite (task t_8e5a3821).
|
|
|
|
The task asks for parser unit tests over "archived sample HTML fixtures (at
|
|
least 3 dates, one per chamber, plus one edge-case file)". California publishes
|
|
its daily file with NO historical date parameter -- the Assembly page always
|
|
renders the current live file and the Senate page is a PDF only (see
|
|
DAILY_FILE_PARSE_SPEC.md section 1 / section 8 #2). So a "different date" in these
|
|
tests means a *different legislative day's content*; the dates are carried as data
|
|
values in the hearing table (the parser derives the file date from it, spec 3.2).
|
|
|
|
The builders below reproduce the canonical table markup verbatim from the archived
|
|
sample (class + data-sort names copied from samples/assembly_dailyfile_latest.html
|
|
and spec section 3) so the parser's date/hearing/selector machinery runs on genuine
|
|
markup, combined with freshly-constructed floor / committee rows. Each builder emits
|
|
one distinct legislative date and covers BOTH chambers' origin, giving the three
|
|
"different dates, one per chamber" fixtures plus the edge-case file the task wants.
|
|
|
|
The builders are pure except that they decode a bill_id through the real
|
|
``scraper.bill_id.parse_bill_id`` codec -- itself a unit-tested subject -- so the
|
|
generated HTML is always well-formed and the id is guaranteed decodable. No network,
|
|
no file I/O: the fixtures are fully reproducible.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from typing import Optional
|
|
|
|
from scraper.bill_id import parse_bill_id
|
|
|
|
|
|
# ---------------------------------------------------------------- markup bits
|
|
|
|
def _url(raw: str) -> str:
|
|
return f"https://leginfo.legislature.ca.gov/faces/billNavClient.xhtml?bill_id={raw}"
|
|
|
|
|
|
# A hearing row: class + data-sort names copied verbatim from the archived sample
|
|
# so _parse_hearings / normalise_date run on genuine markup.
|
|
_HEARING_ROW = (
|
|
'<table class="committee-hearing-table">'
|
|
'<tr class="committee-hearing-details">'
|
|
'<td class="committee_hearing-date" data-sort="{date}">{date_short}</td>'
|
|
'<td class="committee_hearing-time">{time}</td>'
|
|
'<td class="committee_hearing-name" data-sort="{committee}">{committee}</td>'
|
|
'<td class="committee_hearing-location" data-sort="{location}">{location}</td>'
|
|
'<td class="committee_hearing-menu">'
|
|
'<a href="/api/ics/generate?name={ics}&start=1&end=2">ics</a>'
|
|
'<a href="/api/dailyfile/agenda/{agenda}">agenda</a>'
|
|
'</td>'
|
|
'</tr></table>'
|
|
)
|
|
|
|
# A floor bill row: canonical floor-bills-table shape (spec section 3.2). The
|
|
# measure cell carries bill_id in the anchor href (decoded by scraper.bill_id)
|
|
# plus a "Supplemental N" note after a <br> when supplied.
|
|
_FLOOR_ROW = (
|
|
'<table class="floor-bills-table">'
|
|
'<tr class="floor-bill-details">'
|
|
'<td class="floor_bill-measure" data-sort="{bill}">'
|
|
'<a href="{url}" target="_blank">{disp}</a>{supp}</td>'
|
|
'<td class="floor_bill-author" data-sort="{author}">{author}</td>'
|
|
'<td class="floor_bill-subject">{title}</td>'
|
|
'<td class="floor_bill-location" data-sort="{location}">{location}</td>'
|
|
'<td class="floor_bill-item" data-sort="{item_padded}">{item}</td>'
|
|
'<td class="floor_bill-view-details"><dialog class="was-dialog">ignore-me</dialog></td>'
|
|
'</tr></table>'
|
|
)
|
|
|
|
# A committee bill row: canonical committee-bills-table shape (spec section 3.3).
|
|
_COMMITTEE_ROW = (
|
|
'<table class="committee-bills-table">'
|
|
'<tr class="committee-bills-details">'
|
|
'<td class="committee_bill-number" data-sort="{bill}">'
|
|
'<a href="{url}" target="_blank">{disp}</a></td>'
|
|
'<td class="committee_bill-author" data-sort="{author}">{author}</td>'
|
|
'<td class="committee_bill-subject">{title}</td>'
|
|
'<td class="committee_bill-committee" data-sort="{committee}">{committee}</td>'
|
|
'</tr></table>'
|
|
)
|
|
|
|
# The page wrapper is a minimal, well-formed HTML5 doc.
|
|
_WRAPPER = (
|
|
'<!doctype html><html lang="en"><head>'
|
|
'<meta charset="utf-8"><title>Daily File {date}</title>'
|
|
'</head><body>{body}</body></html>'
|
|
)
|
|
|
|
|
|
# --------------------- edge-case fragments (mirror test_fixtures.py) ---------------------
|
|
|
|
EDGE_BAD_NO_LINK = (
|
|
'<table class="floor-bills-table"><tr class="floor-bill-details">'
|
|
'<td class="floor_bill-measure" data-sort="AB 00200">AB 200 (no hyperlink)</td>'
|
|
'<td class="floor_bill-author" data-sort="Jones">Jones</td>'
|
|
'<td class="floor_bill-subject">Missing bill link.</td>'
|
|
'<td class="floor_bill-item" data-sort="0000002">2</td></tr></table>'
|
|
)
|
|
EDGE_BAD_ITEM = (
|
|
'<table class="floor-bills-table"><tr class="floor-bill-details">'
|
|
'<td class="floor_bill-measure">'
|
|
'<a href="https://leginfo.legislature.ca.gov/faces/billNavClient.xhtml'
|
|
'?bill_id=202520260AB300">AB 300</a></td>'
|
|
'<td class="floor_bill-author" data-sort="Lee">Lee</td>'
|
|
'<td class="floor_bill-subject">Non-numeric item cell.</td>'
|
|
'<td class="floor_bill-item">not-a-number</td></tr></table>'
|
|
)
|
|
EDGE_HEARING_NO_COMMITTEE = (
|
|
'<table class="committee-hearing-table"><tr class="committee-hearing-details">'
|
|
'<td class="committee_hearing-time">9:00am</td>'
|
|
'<td class="committee_hearing-menu">no agenda link</td></tr></table>'
|
|
)
|
|
# A floor row whose link/text is only a clean "AB NNNNN" display, recoverable via
|
|
# the parser's display-text fallback.
|
|
EDGE_DISPLAY_ONLY = (
|
|
'<table class="floor-bills-table"><tr class="floor-bill-details">'
|
|
'<td class="floor_bill-measure">AB 150</td>'
|
|
'<td class="floor_bill-author" data-sort="Adams">Adams</td>'
|
|
'<td class="floor_bill-subject">Display-only recovery.</td>'
|
|
'<td class="floor_bill-item" data-sort="0000150">150</td></tr></table>'
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------- builders
|
|
|
|
_SUPP_SUFFIX = '<br>Supplemental {n}'
|
|
|
|
|
|
def _short(date_iso: str) -> str:
|
|
year, month, day = date_iso.split("-")
|
|
return f"{month}/{day}/{year[2:]}" # '2026-08-31' -> '08/31/26'
|
|
|
|
|
|
def hearing(date_iso, time, committee, location, agenda, ics):
|
|
s = _short(date_iso)
|
|
return _HEARING_ROW.format(
|
|
date=f"{s}-001", # the real data-sort carries a '-001' display suffix
|
|
date_short=s,
|
|
time=time,
|
|
committee=committee,
|
|
location=location,
|
|
ics=ics,
|
|
agenda=agenda,
|
|
)
|
|
|
|
|
|
def floor_bill(raw_id, author, title, item, location="State Capitol",
|
|
supplement: Optional[int] = None):
|
|
dec = parse_bill_id(raw_id)
|
|
if dec is None:
|
|
raise ValueError(f"floor_bill builder got undecodable id {raw_id!r}")
|
|
supp = _SUPP_SUFFIX.format(n=supplement) if supplement else ""
|
|
return _FLOOR_ROW.format(
|
|
bill=f"{dec['chamber']} {dec['number']:05d}",
|
|
url=_url(raw_id),
|
|
disp=f"{dec['chamber']} {dec['number']}",
|
|
supp=supp,
|
|
author=author,
|
|
title=title,
|
|
location=location,
|
|
item=str(item),
|
|
item_padded=f"{item:07d}",
|
|
)
|
|
|
|
|
|
def committee_bill(raw_id, author, title, committee):
|
|
dec = parse_bill_id(raw_id)
|
|
if dec is None:
|
|
raise ValueError(f"committee_bill builder got undecodable id {raw_id!r}")
|
|
return _COMMITTEE_ROW.format(
|
|
bill=f"{dec['chamber']} {dec['number']:05d}",
|
|
url=_url(raw_id),
|
|
disp=f"{dec['chamber']} {dec['number']}",
|
|
author=author,
|
|
title=title,
|
|
committee=committee,
|
|
)
|
|
|
|
|
|
# -------- the three "different dates, one per chamber" daily-file fixtures --------
|
|
|
|
def assembly_daily_file_html() -> str:
|
|
date_iso = "2026-08-31" # verbatim from the archived hearing sample
|
|
body = (
|
|
hearing(date_iso, "Upon Call of the Chair", "Budget",
|
|
"1021 O Street, Room 1100", "20361", "name=Budget")
|
|
+ floor_bill("202520260AB28", "Schiavo et al.",
|
|
"An act relating to solid waste, and making an appropriation "
|
|
"therefor.", 32,
|
|
"Unfinished Business - Concurrence In Senate Amendments",
|
|
supplement=2)
|
|
# A Senate-origin bill on the Assembly file (cross-chamber referral,
|
|
# spec section 8 #7) -> chamber_origin must resolve to SENATE.
|
|
+ floor_bill("202520260SB16", "Blakespear et al.",
|
|
"An act relating to mental health.", 62,
|
|
"Senate Third Reading File")
|
|
+ committee_bill("202520260SB795", "Archuleta",
|
|
"Air pollution: gasoline vapor control systems: "
|
|
"ethanol blends.", "Emergency Management")
|
|
)
|
|
return _WRAPPER.format(date=date_iso, body=body)
|
|
|
|
|
|
def senate_daily_file_html() -> str:
|
|
date_iso = "2026-09-02" # the Senate committee daily-file date (spec section 1.2)
|
|
body = (
|
|
hearing(date_iso, "10:00am", "Government Operations",
|
|
"700 N Street, Room 6035", "30481", "name=GovernmentOperations")
|
|
+ floor_bill("202520260SB801", "Garcia et al.",
|
|
"An act relating to water conservation.", 12,
|
|
"State Capitol, Room 417")
|
|
)
|
|
return _WRAPPER.format(date=date_iso, body=body)
|
|
|
|
|
|
def extra_daily_file_html() -> str:
|
|
date_iso = "2026-09-03" # an additional legislative day -> 3rd distinct date
|
|
body = (
|
|
hearing(date_iso, "Upon Call of the Chair", "Rules",
|
|
"1021 O Street, Room 2000", "30512", "name=Rules")
|
|
+ floor_bill("202520260AB42", "Nguyen",
|
|
"An act relating to public schools.", 7,
|
|
"Unfinished Business")
|
|
)
|
|
return _WRAPPER.format(date=date_iso, body=body)
|
|
|
|
|
|
# -------- edge cases --------
|
|
|
|
def edge_case_html() -> str:
|
|
"""Every documented malformed-row class plus one recoverable display-only row
|
|
and one good row, so a good batch must skip the bad rows (warning each),
|
|
recover the display-only row, and keep the good one.
|
|
|
|
Mirrors scraper.test_fixtures.malformed_document() so the parser's
|
|
robustness contract has one authoritative edge fixture here too.
|
|
"""
|
|
good = floor_bill("202520260AB100", "Smith", "An act to do X.", 1, "State Capitol")
|
|
body = good + EDGE_BAD_NO_LINK + EDGE_BAD_ITEM + EDGE_HEARING_NO_COMMITTEE \
|
|
+ EDGE_DISPLAY_ONLY
|
|
return _WRAPPER.format(date="2026-08-31", body=body)
|
|
|
|
|
|
def edge_case_bytes(html: str = "AB 100 <html>not-utf8</html>") -> bytes:
|
|
"""A UTF-16 (BOM-prefixed) file. The parser's charset sniff reads the BOM,
|
|
decodes it, and the text carries replacement chars -> the parser must log a
|
|
warning and return [] rather than crash the batch."""
|
|
return html.encode("utf-16")
|
|
|
|
|
|
# Dated + chamber coverage for the test-suite's own bookkeeping.
|
|
FIXTURE_DATES = {
|
|
"assembly_daily_file_html": ("Assembly", "2026-08-31"),
|
|
"senate_daily_file_html": ("Senate", "2026-09-02"),
|
|
"extra_daily_file_html": ("Assembly", "2026-09-03"),
|
|
"edge_case_html": ("Assembly", "2026-08-31"),
|
|
}
|