318 lines
13 KiB
Python
318 lines
13 KiB
Python
"""HTML parsers: transform cached raw Daily File HTML into BillAction records.
|
|
|
|
Selector contracts follow DAILY_FILE_PARSE_SPEC.md section 3:
|
|
|
|
- Committee hearings : table.committee-hearing-table / tr.committee-hearing-details
|
|
- Bills on the floor : table.floor-bills-table / tr.floor-bill-details
|
|
- Bills in committee : table.committee-bills-table / tr[class~=committee-bills-details]
|
|
|
|
Robustness (task acceptance criteria):
|
|
* malformed rows (missing bill link, no number, decode failures) are skipped
|
|
with a logged warning and never abort the batch;
|
|
* encoding issues are handled by a charset decode with fallback + logging;
|
|
* multi-page / multi-file input is handled by iterating every *.html the
|
|
input resolves to.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import glob
|
|
import logging
|
|
import os
|
|
import re
|
|
from datetime import datetime
|
|
from typing import List, Optional
|
|
|
|
from .bill_id import parse_bill_id, parse_display, parse_supplement, origin_house
|
|
from .schema import BillAction, ASSEMBLY_DAILY_FILE
|
|
|
|
log = logging.getLogger("scraper.parsers")
|
|
|
|
# ---- section selectors ---------------------------------------------------
|
|
HEARING_SELECTOR = "table.committee-hearing-table tr.committee-hearing-details"
|
|
FLOOR_SELECTOR = "table.floor-bills-table tr.floor-bill-details"
|
|
COMMITTEE_SELECTOR = "table.committee-bills-table tr[class~=committee-bills-details]"
|
|
|
|
# Looser fallbacks in case a table class is missing but rows are still present.
|
|
HEARING_ROW = "tr[class*=committee-hearing-details]"
|
|
FLOOR_ROW = "tr[class*=floor-bill-details]"
|
|
COMMITTEE_ROW = "tr[class*=committee-bills-details]"
|
|
|
|
_SUPP = re.compile(r"Supplemental\s+(\d+)", re.IGNORECASE)
|
|
_DATE_SPLIT = re.compile(r"-\d+$") # strip display suffix like '-001'
|
|
|
|
|
|
# --- cell helpers ---------------------------------------------------------
|
|
def _attr_or_text(td, prefer: str = "data-sort") -> Optional[str]:
|
|
"""Return td[prefer] when present else trimmed textContent."""
|
|
if prefer and td is not None and td.get(prefer):
|
|
return td.get(prefer).strip()
|
|
return td.get_text(" ", strip=True) if td is not None else None
|
|
|
|
|
|
def normalise_date(raw: Optional[str]) -> Optional[str]:
|
|
"""'08/31/26-001' / '08/31/26' -> '2026-08-31'. Non-date -> None."""
|
|
if not raw:
|
|
return None
|
|
s = _DATE_SPLIT.sub("", raw.strip()).strip()
|
|
if not s:
|
|
return None
|
|
for fmt in ("%m/%d/%y", "%m/%d/%Y", "%Y-%m-%d"):
|
|
try:
|
|
return datetime.strptime(s, fmt).strftime("%Y-%m-%d")
|
|
except ValueError:
|
|
continue
|
|
return None
|
|
|
|
|
|
# --- section parsers ------------------------------------------------------
|
|
def _parse_hearings(soup, house, source, file_date=None):
|
|
rows = soup.select(HEARING_SELECTOR) or soup.select(HEARING_ROW)
|
|
records = []
|
|
for i, tr in enumerate(rows):
|
|
try:
|
|
date_td = tr.select_one("td.committee_hearing-date")
|
|
time_td = tr.select_one("td.committee_hearing-time")
|
|
comm_td = tr.select_one("td.committee_hearing-name")
|
|
loc_td = tr.select_one("td.committee_hearing-location")
|
|
menu = tr.select_one("td.committee_hearing-menu")
|
|
|
|
date = normalise_date(_attr_or_text(date_td))
|
|
committee = _attr_or_text(comm_td, "data-sort")
|
|
ics = None
|
|
agenda = None
|
|
if menu is not None:
|
|
ics_a = menu.select_one("a[href^='/api/ics/generate']")
|
|
if ics_a:
|
|
ics = ics_a.get("href")
|
|
ag_a = menu.select_one("a[href^='/api/dailyfile/agenda/']")
|
|
if ag_a:
|
|
agenda = ag_a.get("href", "").rsplit("/", 1)[-1]
|
|
|
|
if not committee and date is None:
|
|
raise ValueError("hearing row has neither committee nor date")
|
|
|
|
records.append(BillAction(
|
|
date=date,
|
|
house=house,
|
|
source=source,
|
|
record_type="hearing",
|
|
committee=committee,
|
|
hearing_time=_attr_or_text(time_td, "text") or None,
|
|
location=_attr_or_text(loc_td, "data-sort"),
|
|
ics_url=ics,
|
|
agenda_id=agenda,
|
|
raw_selector="committee-hearing-table",
|
|
))
|
|
except Exception as e: # noqa: BLE001
|
|
log.warning("skip malformed hearing row #%d: %s", i, e)
|
|
# backfill a missing per-row date from the file date
|
|
for h in records:
|
|
if not h.date and file_date:
|
|
h.date = file_date
|
|
return records
|
|
|
|
|
|
def _extract_bill_measure(measure_td):
|
|
"""Return (bill_number, bill_id, supplement) from a measure / number cell."""
|
|
supplement = None
|
|
a = measure_td.select_one("a")
|
|
bill_id = None
|
|
bill_number = None
|
|
if a is not None and a.get("href"):
|
|
m = re.search(r"bill_id=([A-Za-z0-9]+)", a["href"])
|
|
if m:
|
|
bill_id = m.group(1)
|
|
dec = parse_bill_id(bill_id)
|
|
if dec:
|
|
bill_number = dec["display"]
|
|
if not bill_number:
|
|
# Fallback: the display text before any <br> (e.g. "AB 28"). Only accept
|
|
# a clean "PREFIX N" form; anything with extra text is a malformed row
|
|
# and is left None so the caller can skip it.
|
|
display = a.get_text(" ", strip=True) if a is not None else measure_td.get_text(" ", strip=True)
|
|
m2 = re.match(r"^(AB|SB|ACR|SCR|HR|SJR|ABR)\s*(\d+)$", (display or "").strip())
|
|
if m2:
|
|
bill_number = f"{m2.group(1)} {int(m2.group(2))}"
|
|
bill_id = None # display only; leave bill_id None
|
|
parsed_supp = parse_supplement(measure_td.get_text(" ", strip=True))
|
|
if parsed_supp:
|
|
supplement = parsed_supp
|
|
return bill_number, bill_id, supplement
|
|
|
|
|
|
def _parse_floor_bills(soup, house, source, file_date):
|
|
rows = soup.select(FLOOR_SELECTOR) or soup.select(FLOOR_ROW)
|
|
records = []
|
|
for i, tr in enumerate(rows):
|
|
try:
|
|
measure = tr.select_one("td.floor_bill-measure")
|
|
if measure is None:
|
|
raise ValueError("missing floor_bill-measure cell")
|
|
bill_number, bill_id, supplement = _extract_bill_measure(measure)
|
|
if not (bill_number or bill_id):
|
|
raise ValueError("no bill number/id in floor row")
|
|
|
|
dec = parse_bill_id(bill_id) if bill_id else None
|
|
chamber_origin = dec["house"].upper() if dec else origin_house((bill_number or "").split(" ")[0])
|
|
session = dec["session"] if dec else None
|
|
|
|
view = tr.select_one("td.floor_bill-view-details")
|
|
description = None
|
|
if view is not None:
|
|
description = view.get_text(" ", strip=True) or None
|
|
|
|
item_td = tr.select_one("td.floor_bill-item")
|
|
item_text = _attr_or_text(item_td, "data-sort") or _attr_or_text(item_td, "text")
|
|
item_number = int(item_text.strip()) if item_text and item_text.strip().isdigit() else None
|
|
|
|
short_title = _attr_or_text(tr.select_one("td.floor_bill-subject"), "text")
|
|
records.append(BillAction(
|
|
date=file_date,
|
|
house=house,
|
|
source=source,
|
|
record_type="bill",
|
|
bill_number=bill_number,
|
|
bill_id=bill_id,
|
|
session=session,
|
|
chamber_origin=chamber_origin,
|
|
author=_attr_or_text(tr.select_one("td.floor_bill-author"), "data-sort"),
|
|
short_title=short_title,
|
|
description=description or short_title,
|
|
location=_attr_or_text(tr.select_one("td.floor_bill-location"), "data-sort"),
|
|
item_number=item_number,
|
|
supplement=supplement,
|
|
raw_selector="floor-bills-table",
|
|
))
|
|
except Exception as e: # noqa: BLE001
|
|
log.warning("skip malformed floor row #%d: %s", i, e)
|
|
return records
|
|
|
|
|
|
def _parse_committee_bills(soup, house, source, file_date):
|
|
rows = soup.select(COMMITTEE_SELECTOR) or soup.select(COMMITTEE_ROW)
|
|
records = []
|
|
for i, tr in enumerate(rows):
|
|
try:
|
|
num_td = tr.select_one("td.committee_bill-number")
|
|
if num_td is None:
|
|
raise ValueError("missing committee_bill-number cell")
|
|
bill_number, bill_id, _ = _extract_bill_measure(num_td)
|
|
if not (bill_number or bill_id):
|
|
raise ValueError("no bill number/id in committee-bill row")
|
|
|
|
dec = parse_bill_id(bill_id) if bill_id else None
|
|
chamber_origin = dec["house"].upper() if dec else origin_house((bill_number or "").split(" ")[0])
|
|
short_title = _attr_or_text(tr.select_one("td.committee_bill-subject"), "text")
|
|
records.append(BillAction(
|
|
date=file_date,
|
|
house=house,
|
|
source=source,
|
|
record_type="bill",
|
|
bill_number=bill_number,
|
|
bill_id=bill_id,
|
|
session=dec["session"] if dec else None,
|
|
chamber_origin=chamber_origin,
|
|
author=_attr_or_text(tr.select_one("td.committee_bill-author"), "data-sort"),
|
|
short_title=short_title,
|
|
description=short_title,
|
|
committee=_attr_or_text(tr.select_one("td.committee_bill-committee"), "data-sort"),
|
|
raw_selector="committee-bills-table",
|
|
))
|
|
except Exception as e: # noqa: BLE001
|
|
log.warning("skip malformed committee-bill row #%d: %s", i, e)
|
|
return records
|
|
|
|
|
|
def parse_html(html: str, *, house: str = "assembly",
|
|
source: str = ASSEMBLY_DAILY_FILE, parser: str = "lxml") -> List[BillAction]:
|
|
"""Parse a single HTML document into a list of BillAction records.
|
|
|
|
The date is derived from the committee-hearing table (the daily file's date)
|
|
and applied to the bill rows, per spec 3.2 (bills carry no per-row date).
|
|
"""
|
|
from bs4 import BeautifulSoup
|
|
|
|
engine = parser if parser in ("lxml", "html.parser") else "html.parser"
|
|
try:
|
|
soup = BeautifulSoup(html, engine)
|
|
except Exception as e: # noqa: BLE001
|
|
log.error("failed to parse document (%s): %s", type(e).__name__, e)
|
|
return []
|
|
|
|
hearings = _parse_hearings(soup, house, source, file_date=None)
|
|
file_date = next((h.date for h in hearings if h.date), None)
|
|
for h in hearings:
|
|
if not h.date:
|
|
h.date = file_date
|
|
|
|
# Bills first (floor, then committee), then hearings — so a parsed batch
|
|
# leads with bill records, each carrying bill_number/session/etc.
|
|
records = []
|
|
records.extend(_parse_floor_bills(soup, house, source, file_date))
|
|
records.extend(_parse_committee_bills(soup, house, source, file_date))
|
|
records.extend(hearings)
|
|
return records
|
|
|
|
|
|
def _read_text(path: str) -> str:
|
|
"""Read a file with charset detection + an encoding fallback."""
|
|
with open(path, "rb") as fh:
|
|
raw = fh.read()
|
|
head = raw[:2048].decode("utf-8", "ignore").lower()
|
|
charset = "utf-8"
|
|
meta = re.search(r"content-type=\"[^\"]*?charset=([a-z0-9_-]+)", head)
|
|
if meta:
|
|
charset = meta.group(1)
|
|
else:
|
|
decl = re.search(r'charset\s*=\s*["\']?\s*([a-z0-9_-]+)', head)
|
|
if decl:
|
|
charset = decl.group(1)
|
|
try:
|
|
text = raw.decode(charset)
|
|
except (UnicodeDecodeError, LookupError):
|
|
log.warning("bad charset %r in %s, falling back to UTF-8/replace", charset, path)
|
|
text = raw.decode("utf-8", "replace")
|
|
if "\ufffd" in text:
|
|
log.warning("replacement chars detected while decoding %s", path)
|
|
return text
|
|
|
|
|
|
def _resolve_inputs(input_path: str) -> List[str]:
|
|
"""Expand an input into HTML files: a file, a directory (all *.html), or a glob."""
|
|
if os.path.isdir(input_path):
|
|
return sorted(glob.glob(os.path.join(input_path, "*.html")))
|
|
if any(ch in input_path for ch in "*?[]"):
|
|
return sorted(glob.glob(input_path))
|
|
return [input_path]
|
|
|
|
|
|
def parse(input_path: str, *, house: str = "assembly",
|
|
source: str = ASSEMBLY_DAILY_FILE, parser: str = "lxml",
|
|
include_hearings: bool = True) -> List[BillAction]:
|
|
"""Parse one or more cached HTML files into a flat list of records.
|
|
|
|
Robust to malformed rows (logged + skipped), bad encodings (logged +
|
|
fallback), and multi-file/multi-page input. Never aborts the batch on a
|
|
single bad file or row.
|
|
"""
|
|
files = _resolve_inputs(input_path)
|
|
if not files:
|
|
log.warning("no input files matched %r", input_path)
|
|
return []
|
|
records: List[BillAction] = []
|
|
for path in files:
|
|
if not os.path.exists(path):
|
|
log.warning("input file not found: %s", path)
|
|
continue
|
|
try:
|
|
html = _read_text(path)
|
|
except OSError as e:
|
|
log.warning("cannot read %s: %s", path, e)
|
|
continue
|
|
doc_records = parse_html(html, house=house, source=source, parser=parser)
|
|
if not include_hearings:
|
|
doc_records = [r for r in doc_records if r.record_type != "hearing"]
|
|
records.extend(doc_records)
|
|
return records
|