feat(dashboard): add forward "what's next" signal + fix upcoming floor
The daily-file bill queue is a backward snapshot, so "upcoming" could only resurface the latest past day (the 08-31 snapshot read as today's queue) and there was no "what is scheduled next." leginfo bill status/history are backward-only, so the genuine future-dated signal is the chambers' "upcoming events" calendars. - api.py: floor the "upcoming" view at *today* (not the DB max date) when no as_of is pinned; surface next_legislative_day + scheduled_events (next 7 days) on /bills. Add _today_iso / _next_legislative_day / _scheduled_events, all degrading to a recess marker / [] (never 5xx). - relational.py: schema v2 adds the scheduled_events table + store_scheduled_events (idempotent per-day refresh), load_scheduled_events, next_legislative_day. - parsers.py: parse_events_page + classify_event_kind for the chamber events calendar; skip empty-title rows (decompose the <time> before the fallback). - ingest.py + cli.py: --store-events lane (cache-first, recess-safe) that parses the calendar HTML into scheduled_events. - sources.py + config.yaml: register assembly_events / senate_events (kind: events, empty_is_ok: true). - run_ingest.sh: run the events lane after the daily-file lane as non-fatal (skippable via CALI_SCRAPER_SKIP_EVENTS=1); daily-file exit still gates cron. - frontend: "Next legislative day" line + "What's next · next 7 days" band with honest recess empty states. - DASHBOARD.md: document the forward signal + known limitations (leginfo backward-only gap, recess empty state, 08-31 snapshot is analysis-only). - tests: 5 new API tests + 18 parse-events tests; 170 pass, 92% coverage.
This commit is contained in:
1 parent
1919d43bf0
commit
03eeb6bb62
16 files changed
+941
-70
No files matched your search
@@ -315,3 +315,109 @@ def parse(input_path: str, *, house: str = "assembly",
|
||||
doc_records = [r for r in doc_records if r.record_type != "hearing"]
|
||||
records.extend(doc_records)
|
||||
return records
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Chamber "upcoming events" calendar (scheduled_events source, v2)
|
||||
#
|
||||
# The chamber calendar (assembly.ca.gov/events/upcoming, senate.ca.gov/
|
||||
# calendar) is a Drupal "views" table (<table class="cols-3"> with Date /
|
||||
# Title / Video columns). Each row is a scheduled hearing / floor session --
|
||||
# the only *forward-looking* signal (the daily file is a backward snapshot).
|
||||
# Rows are stored in scheduled_events (relational.store_scheduled_events).
|
||||
# An empty page (a recess) is a legitimate empty result, not an error.
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
# Title keywords that classify an event's kind (best-effort, never crash).
|
||||
_EV_KIND_PATTERNS = (
|
||||
(re.compile(r"\bfloor\b", re.I), "FLOOR"),
|
||||
(re.compile(r"\bcommittee\b", re.I), "COMMITTEE"),
|
||||
)
|
||||
|
||||
|
||||
def classify_event_kind(title):
|
||||
"""Map an event title to a kind: FLOOR / COMMITTEE / OTHER."""
|
||||
t = title or ""
|
||||
for pat, kind in _EV_KIND_PATTERNS:
|
||||
if pat.search(t):
|
||||
return kind
|
||||
return "OTHER"
|
||||
|
||||
|
||||
def parse_events_page(html, chamber="assembly", source="assembly_events",
|
||||
base_url="", parser="lxml"):
|
||||
"""Parse a chamber events calendar page into scheduled-event dicts.
|
||||
|
||||
Returns a list of dicts with keys: date (ISO YYYY-MM-DD), chamber
|
||||
(upper), kind (FLOOR|COMMITTEE|OTHER), title, source_url. An empty /
|
||||
recess page yields []. Rows without a resolvable date are skipped
|
||||
(logged). Header rows (<th>) are ignored.
|
||||
"""
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
engine = parser if parser in ("lxml", "html.parser") else "html.parser"
|
||||
try:
|
||||
soup = BeautifulSoup(html, engine)
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.error("events: failed to parse document (%s): %s",
|
||||
type(e).__name__, e)
|
||||
return []
|
||||
|
||||
out = []
|
||||
# Fall back to any <tr> carrying a <time> cell when table.cols-3 is
|
||||
# absent, so a layout change still yields rows rather than a silent
|
||||
# empty page.
|
||||
base = (base_url or "").rstrip("/")
|
||||
host = base.rsplit("/", 2)[0] if base else "https://www.assembly.ca.gov"
|
||||
tables = soup.select("table.cols-3")
|
||||
rows = [tr for tbl in tables for tr in tbl.find_all("tr")]
|
||||
if not rows:
|
||||
rows = [tr for tr in soup.find_all("tr") if tr.find("time")]
|
||||
for i, tr in enumerate(rows):
|
||||
# Skip header rows (column labels render as <th>).
|
||||
if tr.find("th"):
|
||||
continue
|
||||
try:
|
||||
time_el = tr.find("time")
|
||||
if time_el is None:
|
||||
raise ValueError("event row has no date cell")
|
||||
# Prefer the datetime attr: "2026-09-01T00:00:00-07:00".
|
||||
dt = time_el.get("datetime") or ""
|
||||
date = dt[:10] if len(dt) >= 10 and dt[4:5] == "-" else None
|
||||
if not date:
|
||||
date = normalise_date(time_el.get_text(strip=True))
|
||||
|
||||
title_td = tr.select_one("td.views-field-title")
|
||||
if title_td is not None:
|
||||
title = title_td.get_text(" ", strip=True)
|
||||
else:
|
||||
t = tr.find("time")
|
||||
if t is not None:
|
||||
t.decompose()
|
||||
title = tr.get_text(" ", strip=True)
|
||||
if not date or not title:
|
||||
raise ValueError("event row has no date or title")
|
||||
|
||||
# source_url: the row's own link (event / video detail) when
|
||||
# present, else the calendar page (base_url) as provenance.
|
||||
a = tr.find("a", href=True)
|
||||
source_url = None
|
||||
if a is not None:
|
||||
href = (a.get("href") or "").strip()
|
||||
if href.startswith("http"):
|
||||
source_url = href
|
||||
elif href.startswith("/"):
|
||||
source_url = host + href
|
||||
if not source_url:
|
||||
source_url = base or None
|
||||
|
||||
out.append({
|
||||
"date": date,
|
||||
"chamber": chamber.strip().upper(),
|
||||
"kind": classify_event_kind(title),
|
||||
"title": title,
|
||||
"source_url": source_url,
|
||||
"source": source,
|
||||
})
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("skip malformed event row #%d: %s", i, e)
|
||||
return out
|
||||
Reference in new issue
Block a user