"""HTML parsers: transform cached raw Daily File HTML into BillAction records.
Selector contracts follow DAILY_FILE_PARSE_SPEC.md section 3:
- Committee hearings : table.committee-hearing-table / tr.committee-hearing-details
- Bills on the floor : table.floor-bills-table / tr.floor-bill-details
- Bills in committee : table.committee-bills-table / tr[class~=committee-bills-details]
Robustness (task acceptance criteria):
* malformed rows (missing bill link, no number, decode failures) are skipped
with a logged warning and never abort the batch;
* encoding issues are handled by a charset decode with fallback + logging;
* multi-page / multi-file input is handled by iterating every *.html the
input resolves to.
"""
from __future__ import annotations
import glob
import logging
import os
import re
from datetime import datetime
from typing import List, Optional
from .bill_id import parse_bill_id, parse_display, parse_supplement, origin_house
from .schema import BillAction, ASSEMBLY_DAILY_FILE
log = logging.getLogger("scraper.parsers")
# ---- section selectors ---------------------------------------------------
HEARING_SELECTOR = "table.committee-hearing-table tr.committee-hearing-details"
FLOOR_SELECTOR = "table.floor-bills-table tr.floor-bill-details"
COMMITTEE_SELECTOR = "table.committee-bills-table tr[class~=committee-bills-details]"
# Looser fallbacks in case a table class is missing but rows are still present.
HEARING_ROW = "tr[class*=committee-hearing-details]"
FLOOR_ROW = "tr[class*=floor-bill-details]"
COMMITTEE_ROW = "tr[class*=committee-bills-details]"
_SUPP = re.compile(r"Supplemental\s+(\d+)", re.IGNORECASE)
_DATE_SPLIT = re.compile(r"-\d+$") # strip display suffix like '-001'
# --- cell helpers ---------------------------------------------------------
def _attr_or_text(td, prefer: str = "data-sort") -> Optional[str]:
"""Return td[prefer] when present else trimmed textContent."""
if prefer and td is not None and td.get(prefer):
return td.get(prefer).strip()
return td.get_text(" ", strip=True) if td is not None else None
def normalise_date(raw: Optional[str]) -> Optional[str]:
"""'08/31/26-001' / '08/31/26' -> '2026-08-31'. Non-date -> None."""
if not raw:
return None
s = _DATE_SPLIT.sub("", raw.strip()).strip()
if not s:
return None
for fmt in ("%m/%d/%y", "%m/%d/%Y", "%Y-%m-%d"):
try:
return datetime.strptime(s, fmt).strftime("%Y-%m-%d")
except ValueError:
continue
return None
# --- section parsers ------------------------------------------------------
def _parse_hearings(soup, house, source, file_date=None):
rows = soup.select(HEARING_SELECTOR) or soup.select(HEARING_ROW)
records = []
for i, tr in enumerate(rows):
try:
date_td = tr.select_one("td.committee_hearing-date")
time_td = tr.select_one("td.committee_hearing-time")
comm_td = tr.select_one("td.committee_hearing-name")
loc_td = tr.select_one("td.committee_hearing-location")
menu = tr.select_one("td.committee_hearing-menu")
date = normalise_date(_attr_or_text(date_td))
committee = _attr_or_text(comm_td, "data-sort")
ics = None
agenda = None
if menu is not None:
ics_a = menu.select_one("a[href^='/api/ics/generate']")
if ics_a:
ics = ics_a.get("href")
ag_a = menu.select_one("a[href^='/api/dailyfile/agenda/']")
if ag_a:
agenda = ag_a.get("href", "").rsplit("/", 1)[-1]
if not committee and date is None:
raise ValueError("hearing row has neither committee nor date")
records.append(BillAction(
date=date,
house=house,
source=source,
record_type="hearing",
committee=committee,
hearing_time=_attr_or_text(time_td, "text") or None,
location=_attr_or_text(loc_td, "data-sort"),
ics_url=ics,
agenda_id=agenda,
raw_selector="committee-hearing-table",
))
except Exception as e: # noqa: BLE001
log.warning("skip malformed hearing row #%d: %s", i, e)
# backfill a missing per-row date from the file date
for h in records:
if not h.date and file_date:
h.date = file_date
return records
def _extract_bill_measure(measure_td):
"""Return (bill_number, bill_id, supplement) from a measure / number cell."""
supplement = None
a = measure_td.select_one("a")
bill_id = None
bill_number = None
if a is not None and a.get("href"):
m = re.search(r"bill_id=([A-Za-z0-9]+)", a["href"])
if m:
bill_id = m.group(1)
dec = parse_bill_id(bill_id)
if dec:
bill_number = dec["display"]
if not bill_number:
# Fallback: the display text before any
(e.g. "AB 28"). Only accept
# a clean "PREFIX N" form; anything with extra text is a malformed row
# and is left None so the caller can skip it.
display = a.get_text(" ", strip=True) if a is not None else measure_td.get_text(" ", strip=True)
m2 = re.match(r"^(AB|SB|ACR|SCR|HR|SJR|ABR)\s*(\d+)$", (display or "").strip())
if m2:
bill_number = f"{m2.group(1)} {int(m2.group(2))}"
bill_id = None # display only; leave bill_id None
parsed_supp = parse_supplement(measure_td.get_text(" ", strip=True))
if parsed_supp:
supplement = parsed_supp
return bill_number, bill_id, supplement
def _parse_floor_bills(soup, house, source, file_date):
rows = soup.select(FLOOR_SELECTOR) or soup.select(FLOOR_ROW)
records = []
for i, tr in enumerate(rows):
try:
measure = tr.select_one("td.floor_bill-measure")
if measure is None:
raise ValueError("missing floor_bill-measure cell")
bill_number, bill_id, supplement = _extract_bill_measure(measure)
if not (bill_number or bill_id):
raise ValueError("no bill number/id in floor row")
dec = parse_bill_id(bill_id) if bill_id else None
chamber_origin = dec["house"].upper() if dec else origin_house((bill_number or "").split(" ")[0])
session = dec["session"] if dec else None
view = tr.select_one("td.floor_bill-view-details")
description = None
if view is not None:
description = view.get_text(" ", strip=True) or None
item_td = tr.select_one("td.floor_bill-item")
item_text = _attr_or_text(item_td, "data-sort") or _attr_or_text(item_td, "text")
item_number = int(item_text.strip()) if item_text and item_text.strip().isdigit() else None
short_title = _attr_or_text(tr.select_one("td.floor_bill-subject"), "text")
records.append(BillAction(
date=file_date,
house=house,
source=source,
record_type="bill",
bill_number=bill_number,
bill_id=bill_id,
session=session,
chamber_origin=chamber_origin,
author=_attr_or_text(tr.select_one("td.floor_bill-author"), "data-sort"),
short_title=short_title,
description=description or short_title,
location=_attr_or_text(tr.select_one("td.floor_bill-location"), "data-sort"),
item_number=item_number,
supplement=supplement,
raw_selector="floor-bills-table",
))
except Exception as e: # noqa: BLE001
log.warning("skip malformed floor row #%d: %s", i, e)
return records
def _parse_committee_bills(soup, house, source, file_date):
rows = soup.select(COMMITTEE_SELECTOR) or soup.select(COMMITTEE_ROW)
records = []
for i, tr in enumerate(rows):
try:
num_td = tr.select_one("td.committee_bill-number")
if num_td is None:
raise ValueError("missing committee_bill-number cell")
bill_number, bill_id, _ = _extract_bill_measure(num_td)
if not (bill_number or bill_id):
raise ValueError("no bill number/id in committee-bill row")
dec = parse_bill_id(bill_id) if bill_id else None
chamber_origin = dec["house"].upper() if dec else origin_house((bill_number or "").split(" ")[0])
short_title = _attr_or_text(tr.select_one("td.committee_bill-subject"), "text")
records.append(BillAction(
date=file_date,
house=house,
source=source,
record_type="bill",
bill_number=bill_number,
bill_id=bill_id,
session=dec["session"] if dec else None,
chamber_origin=chamber_origin,
author=_attr_or_text(tr.select_one("td.committee_bill-author"), "data-sort"),
short_title=short_title,
description=short_title,
committee=_attr_or_text(tr.select_one("td.committee_bill-committee"), "data-sort"),
raw_selector="committee-bills-table",
))
except Exception as e: # noqa: BLE001
log.warning("skip malformed committee-bill row #%d: %s", i, e)
return records
def parse_html(html: str, *, house: str = "assembly",
source: str = ASSEMBLY_DAILY_FILE, parser: str = "lxml") -> List[BillAction]:
"""Parse a single HTML document into a list of BillAction records.
The date is derived from the committee-hearing table (the daily file's date)
and applied to the bill rows, per spec 3.2 (bills carry no per-row date).
"""
from bs4 import BeautifulSoup
engine = parser if parser in ("lxml", "html.parser") else "html.parser"
try:
soup = BeautifulSoup(html, engine)
except Exception as e: # noqa: BLE001
log.error("failed to parse document (%s): %s", type(e).__name__, e)
return []
hearings = _parse_hearings(soup, house, source, file_date=None)
file_date = next((h.date for h in hearings if h.date), None)
for h in hearings:
if not h.date:
h.date = file_date
# Bills first (floor, then committee), then hearings — so a parsed batch
# leads with bill records, each carrying bill_number/session/etc.
records = []
records.extend(_parse_floor_bills(soup, house, source, file_date))
records.extend(_parse_committee_bills(soup, house, source, file_date))
records.extend(hearings)
return records
def _read_text(path: str) -> str:
"""Read a file with charset detection + an encoding fallback."""
with open(path, "rb") as fh:
raw = fh.read()
head = raw[:2048].decode("utf-8", "ignore").lower()
charset = "utf-8"
meta = re.search(r"content-type=\"[^\"]*?charset=([a-z0-9_-]+)", head)
if meta:
charset = meta.group(1)
else:
decl = re.search(r'charset\s*=\s*["\']?\s*([a-z0-9_-]+)', head)
if decl:
charset = decl.group(1)
try:
text = raw.decode(charset)
except (UnicodeDecodeError, LookupError):
log.warning("bad charset %r in %s, falling back to UTF-8/replace", charset, path)
text = raw.decode("utf-8", "replace")
if "\ufffd" in text:
log.warning("replacement chars detected while decoding %s", path)
return text
def _resolve_inputs(input_path: str) -> List[str]:
"""Expand an input into HTML files: a file, a directory (all *.html), or a glob."""
if os.path.isdir(input_path):
return sorted(glob.glob(os.path.join(input_path, "*.html")))
if any(ch in input_path for ch in "*?[]"):
return sorted(glob.glob(input_path))
return [input_path]
def parse(input_path: str, *, house: str = "assembly",
source: str = ASSEMBLY_DAILY_FILE, parser: str = "lxml",
include_hearings: bool = True) -> List[BillAction]:
"""Parse one or more cached HTML files into a flat list of records.
Robust to malformed rows (logged + skipped), bad encodings (logged +
fallback), and multi-file/multi-page input. Never aborts the batch on a
single bad file or row.
"""
files = _resolve_inputs(input_path)
if not files:
log.warning("no input files matched %r", input_path)
return []
records: List[BillAction] = []
for path in files:
if not os.path.exists(path):
log.warning("input file not found: %s", path)
continue
try:
html = _read_text(path)
except OSError as e:
log.warning("cannot read %s: %s", path, e)
continue
doc_records = parse_html(html, house=house, source=source, parser=parser)
if not include_hearings:
doc_records = [r for r in doc_records if r.record_type != "hearing"]
records.extend(doc_records)
return records