WesternTechnologies/cali_scraper
2
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
kfj001 committed 2026-09-06 10:18:26 -07:00
commit 5f2db08b43
100 files changed
+16985
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/fixtures.py
+255
View File
@@ -0,0 +1,255 @@
"""Fixture builders for the QA test-suite (task t_8e5a3821).
The task asks for parser unit tests over "archived sample HTML fixtures (at
least 3 dates, one per chamber, plus one edge-case file)". California publishes
its daily file with NO historical date parameter -- the Assembly page always
renders the current live file and the Senate page is a PDF only (see
DAILY_FILE_PARSE_SPEC.md section 1 / section 8 #2). So a "different date" in these
tests means a *different legislative day's content*; the dates are carried as data
values in the hearing table (the parser derives the file date from it, spec 3.2).
The builders below reproduce the canonical table markup verbatim from the archived
sample (class + data-sort names copied from samples/assembly_dailyfile_latest.html
and spec section 3) so the parser's date/hearing/selector machinery runs on genuine
markup, combined with freshly-constructed floor / committee rows. Each builder emits
one distinct legislative date and covers BOTH chambers' origin, giving the three
"different dates, one per chamber" fixtures plus the edge-case file the task wants.
The builders are pure except that they decode a bill_id through the real
``scraper.bill_id.parse_bill_id`` codec -- itself a unit-tested subject -- so the
generated HTML is always well-formed and the id is guaranteed decodable. No network,
no file I/O: the fixtures are fully reproducible.
"""
from __future__ import annotations
from typing import Optional
from scraper.bill_id import parse_bill_id
# ---------------------------------------------------------------- markup bits
def _url(raw: str) -> str:
return f"https://leginfo.legislature.ca.gov/faces/billNavClient.xhtml?bill_id={raw}"
# A hearing row: class + data-sort names copied verbatim from the archived sample
# so _parse_hearings / normalise_date run on genuine markup.
_HEARING_ROW = (
'<table class="committee-hearing-table">'
'<tr class="committee-hearing-details">'
'<td class="committee_hearing-date" data-sort="{date}">{date_short}</td>'
'<td class="committee_hearing-time">{time}</td>'
'<td class="committee_hearing-name" data-sort="{committee}">{committee}</td>'
'<td class="committee_hearing-location" data-sort="{location}">{location}</td>'
'<td class="committee_hearing-menu">'
'<a href="/api/ics/generate?name={ics}&amp;start=1&amp;end=2">ics</a>'
'<a href="/api/dailyfile/agenda/{agenda}">agenda</a>'
'</td>'
'</tr></table>'
)
# A floor bill row: canonical floor-bills-table shape (spec section 3.2). The
# measure cell carries bill_id in the anchor href (decoded by scraper.bill_id)
# plus a "Supplemental N" note after a <br> when supplied.
_FLOOR_ROW = (
'<table class="floor-bills-table">'
'<tr class="floor-bill-details">'
'<td class="floor_bill-measure" data-sort="{bill}">'
'<a href="{url}" target="_blank">{disp}</a>{supp}</td>'
'<td class="floor_bill-author" data-sort="{author}">{author}</td>'
'<td class="floor_bill-subject">{title}</td>'
'<td class="floor_bill-location" data-sort="{location}">{location}</td>'
'<td class="floor_bill-item" data-sort="{item_padded}">{item}</td>'
'<td class="floor_bill-view-details"><dialog class="was-dialog">ignore-me</dialog></td>'
'</tr></table>'
)
# A committee bill row: canonical committee-bills-table shape (spec section 3.3).
_COMMITTEE_ROW = (
'<table class="committee-bills-table">'
'<tr class="committee-bills-details">'
'<td class="committee_bill-number" data-sort="{bill}">'
'<a href="{url}" target="_blank">{disp}</a></td>'
'<td class="committee_bill-author" data-sort="{author}">{author}</td>'
'<td class="committee_bill-subject">{title}</td>'
'<td class="committee_bill-committee" data-sort="{committee}">{committee}</td>'
'</tr></table>'
)
# The page wrapper is a minimal, well-formed HTML5 doc.
_WRAPPER = (
'<!doctype html><html lang="en"><head>'
'<meta charset="utf-8"><title>Daily File {date}</title>'
'</head><body>{body}</body></html>'
)
# --------------------- edge-case fragments (mirror test_fixtures.py) ---------------------
EDGE_BAD_NO_LINK = (
'<table class="floor-bills-table"><tr class="floor-bill-details">'
'<td class="floor_bill-measure" data-sort="AB 00200">AB 200 (no hyperlink)</td>'
'<td class="floor_bill-author" data-sort="Jones">Jones</td>'
'<td class="floor_bill-subject">Missing bill link.</td>'
'<td class="floor_bill-item" data-sort="0000002">2</td></tr></table>'
)
EDGE_BAD_ITEM = (
'<table class="floor-bills-table"><tr class="floor-bill-details">'
'<td class="floor_bill-measure">'
'<a href="https://leginfo.legislature.ca.gov/faces/billNavClient.xhtml'
'?bill_id=202520260AB300">AB 300</a></td>'
'<td class="floor_bill-author" data-sort="Lee">Lee</td>'
'<td class="floor_bill-subject">Non-numeric item cell.</td>'
'<td class="floor_bill-item">not-a-number</td></tr></table>'
)
EDGE_HEARING_NO_COMMITTEE = (
'<table class="committee-hearing-table"><tr class="committee-hearing-details">'
'<td class="committee_hearing-time">9:00am</td>'
'<td class="committee_hearing-menu">no agenda link</td></tr></table>'
)
# A floor row whose link/text is only a clean "AB NNNNN" display, recoverable via
# the parser's display-text fallback.
EDGE_DISPLAY_ONLY = (
'<table class="floor-bills-table"><tr class="floor-bill-details">'
'<td class="floor_bill-measure">AB 150</td>'
'<td class="floor_bill-author" data-sort="Adams">Adams</td>'
'<td class="floor_bill-subject">Display-only recovery.</td>'
'<td class="floor_bill-item" data-sort="0000150">150</td></tr></table>'
)
# ---------------------------------------------------------------- builders
_SUPP_SUFFIX = '<br>Supplemental {n}'
def _short(date_iso: str) -> str:
year, month, day = date_iso.split("-")
return f"{month}/{day}/{year[2:]}" # '2026-08-31' -> '08/31/26'
def hearing(date_iso, time, committee, location, agenda, ics):
s = _short(date_iso)
return _HEARING_ROW.format(
date=f"{s}-001", # the real data-sort carries a '-001' display suffix
date_short=s,
time=time,
committee=committee,
location=location,
ics=ics,
agenda=agenda,
)
def floor_bill(raw_id, author, title, item, location="State Capitol",
supplement: Optional[int] = None):
dec = parse_bill_id(raw_id)
if dec is None:
raise ValueError(f"floor_bill builder got undecodable id {raw_id!r}")
supp = _SUPP_SUFFIX.format(n=supplement) if supplement else ""
return _FLOOR_ROW.format(
bill=f"{dec['chamber']} {dec['number']:05d}",
url=_url(raw_id),
disp=f"{dec['chamber']} {dec['number']}",
supp=supp,
author=author,
title=title,
location=location,
item=str(item),
item_padded=f"{item:07d}",
)
def committee_bill(raw_id, author, title, committee):
dec = parse_bill_id(raw_id)
if dec is None:
raise ValueError(f"committee_bill builder got undecodable id {raw_id!r}")
return _COMMITTEE_ROW.format(
bill=f"{dec['chamber']} {dec['number']:05d}",
url=_url(raw_id),
disp=f"{dec['chamber']} {dec['number']}",
author=author,
title=title,
committee=committee,
)
# -------- the three "different dates, one per chamber" daily-file fixtures --------
def assembly_daily_file_html() -> str:
date_iso = "2026-08-31" # verbatim from the archived hearing sample
body = (
hearing(date_iso, "Upon Call of the Chair", "Budget",
"1021 O Street, Room 1100", "20361", "name=Budget")
+ floor_bill("202520260AB28", "Schiavo et al.",
"An act relating to solid waste, and making an appropriation "
"therefor.", 32,
"Unfinished Business - Concurrence In Senate Amendments",
supplement=2)
# A Senate-origin bill on the Assembly file (cross-chamber referral,
# spec section 8 #7) -> chamber_origin must resolve to SENATE.
+ floor_bill("202520260SB16", "Blakespear et al.",
"An act relating to mental health.", 62,
"Senate Third Reading File")
+ committee_bill("202520260SB795", "Archuleta",
"Air pollution: gasoline vapor control systems: "
"ethanol blends.", "Emergency Management")
)
return _WRAPPER.format(date=date_iso, body=body)
def senate_daily_file_html() -> str:
date_iso = "2026-09-02" # the Senate committee daily-file date (spec section 1.2)
body = (
hearing(date_iso, "10:00am", "Government Operations",
"700 N Street, Room 6035", "30481", "name=GovernmentOperations")
+ floor_bill("202520260SB801", "Garcia et al.",
"An act relating to water conservation.", 12,
"State Capitol, Room 417")
)
return _WRAPPER.format(date=date_iso, body=body)
def extra_daily_file_html() -> str:
date_iso = "2026-09-03" # an additional legislative day -> 3rd distinct date
body = (
hearing(date_iso, "Upon Call of the Chair", "Rules",
"1021 O Street, Room 2000", "30512", "name=Rules")
+ floor_bill("202520260AB42", "Nguyen",
"An act relating to public schools.", 7,
"Unfinished Business")
)
return _WRAPPER.format(date=date_iso, body=body)
# -------- edge cases --------
def edge_case_html() -> str:
"""Every documented malformed-row class plus one recoverable display-only row
and one good row, so a good batch must skip the bad rows (warning each),
recover the display-only row, and keep the good one.
Mirrors scraper.test_fixtures.malformed_document() so the parser's
robustness contract has one authoritative edge fixture here too.
"""
good = floor_bill("202520260AB100", "Smith", "An act to do X.", 1, "State Capitol")
body = good + EDGE_BAD_NO_LINK + EDGE_BAD_ITEM + EDGE_HEARING_NO_COMMITTEE \
+ EDGE_DISPLAY_ONLY
return _WRAPPER.format(date="2026-08-31", body=body)
def edge_case_bytes(html: str = "AB 100 <html>not-utf8</html>") -> bytes:
"""A UTF-16 (BOM-prefixed) file. The parser's charset sniff reads the BOM,
decodes it, and the text carries replacement chars -> the parser must log a
warning and return [] rather than crash the batch."""
return html.encode("utf-16")
# Dated + chamber coverage for the test-suite's own bookkeeping.
FIXTURE_DATES = {
"assembly_daily_file_html": ("Assembly", "2026-08-31"),
"senate_daily_file_html": ("Senate", "2026-09-02"),
"extra_daily_file_html": ("Assembly", "2026-09-03"),
"edge_case_html": ("Assembly", "2026-08-31"),
}