feat(dashboard): add forward "what's next" signal + fix upcoming floor

The daily-file bill queue is a backward snapshot, so "upcoming" could only
resurface the latest past day (the 08-31 snapshot read as today's queue) and
there was no "what is scheduled next." leginfo bill status/history are
backward-only, so the genuine future-dated signal is the chambers' "upcoming
events" calendars.

- api.py: floor the "upcoming" view at *today* (not the DB max date) when no
  as_of is pinned; surface next_legislative_day + scheduled_events (next 7
  days) on /bills. Add _today_iso / _next_legislative_day / _scheduled_events,
  all degrading to a recess marker / [] (never 5xx).
- relational.py: schema v2 adds the scheduled_events table +
  store_scheduled_events (idempotent per-day refresh), load_scheduled_events,
  next_legislative_day.
- parsers.py: parse_events_page + classify_event_kind for the chamber events
  calendar; skip empty-title rows (decompose the <time> before the fallback).
- ingest.py + cli.py: --store-events lane (cache-first, recess-safe) that
  parses the calendar HTML into scheduled_events.
- sources.py + config.yaml: register assembly_events / senate_events
  (kind: events, empty_is_ok: true).
- run_ingest.sh: run the events lane after the daily-file lane as non-fatal
  (skippable via CALI_SCRAPER_SKIP_EVENTS=1); daily-file exit still gates cron.
- frontend: "Next legislative day" line + "What's next · next 7 days" band
  with honest recess empty states.
- DASHBOARD.md: document the forward signal + known limitations (leginfo
  backward-only gap, recess empty state, 08-31 snapshot is analysis-only).
- tests: 5 new API tests + 18 parse-events tests; 170 pass, 92% coverage.
This commit is contained in:
kfj001 committed 2026-09-06 19:50:02 -07:00
1 parent 1919d43bf0
commit 03eeb6bb62
16 files changed
+941 -70

No files matched your search

+86 -17
View File
@@ -13,6 +13,7 @@ from . import data as _data
from . import ranking
from . import resolve as _resolve
from . import bill_status
from .. import relational
from .data import (load_tag_cloud, list_events,
load_legislators, committee_roster_for)
from .ranking import hearing_sort, full_queue, personal_band
@@ -117,17 +118,25 @@ def _truthy(v):
def handle_bills(req):
q = req.query
# "Upcoming" is a FORWARD view: when the caller asks for upcoming without
# pinning an as_of, floor it at *today*, not the most-recent data date.
# Flooring at today is the fix for "upcoming shows the latest PAST day";
# a None as_of would fall back to the DB max date and re-introduce it.
upcoming = _truthy(q.get("upcoming"))
as_of = q.get("as_of")
if upcoming and not as_of:
as_of = _today_iso()
events = list_events(
db_path=req.svc.db_path,
upcoming=_truthy(q.get("upcoming")),
from_date=q.get("from") or q.get("date_from"),
to_date=q.get("to") or q.get("date_to"),
chamber=q.get("chamber"),
tags=q_list("tags", q),
q=q.get("q"),
author=q.get("author"),
as_of=q.get("as_of"),
)
db_path=req.svc.db_path,
upcoming=upcoming,
from_date=q.get("from") or q.get("date_from"),
to_date=q.get("to") or q.get("date_to"),
chamber=q.get("chamber"),
tags=q_list("tags", q),
q=q.get("q"),
author=q.get("author"),
as_of=as_of,
)
if _truthy(q.get("personal")):
uid, err = req.require_auth()
if err:
@@ -143,7 +152,7 @@ def handle_bills(req):
events = full_queue(events)
else:
events = personal_band(
events, int(q.get("band_n") or ranking.PERSONAL_BAND_N))
events, int(q.get("band_n") or ranking.PERSONAL_BAND_N))
session = _detect_session(events, req.svc.db_path)
items = [_client_bill(e) for e in events]
groups = {}
@@ -151,16 +160,76 @@ def handle_bills(req):
key = e.get("date") or "TBD"
groups[key] = groups.get(key, 0) + 1
chamber_avail = _chamber_availability(items, req.svc.db_path)
# The bill queue is a backward daily-file snapshot; the chamber "upcoming
# events" calendar (scheduled_events, schema v2) is the forward-looking
# signal. Surface "next legislative day" + the next-7-days events so a bill
# like AB-1099 shows what is scheduled next, not just the past. A None
# as_of (the "All"/history view) still answers "next" from today onward.
floor = as_of if as_of else _today_iso()
next_day = _next_legislative_day(req.svc.db_path, floor, q.get("chamber"))
scheduled = _scheduled_events(req.svc.db_path, floor, q.get("chamber"))
body = {
"session": session,
"as_of": q.get("as_of") or _now_iso(),
"chamber_availability": chamber_avail,
"items": items,
"groups": groups,
}
"session": session,
"as_of": as_of or floor,
"chamber_availability": chamber_avail,
"next_legislative_day": next_day,
"scheduled_events": scheduled,
"items": items,
"groups": groups,
}
return 200, body
def _today_iso() -> str:
import datetime
return datetime.date.today().isoformat()
def _next_legislative_day(db_path, as_of, chamber=None):
"""Earliest scheduled chamber event on/after as_of (DESIGN section 6.1).
Degrades gracefully: when the scheduled_events table is absent (pre-v2 DB)
or empty for the window (a recess), returns ``{"date": None, "recess":
True}`` so the UI can show "next legislative day unknown". Never raises.
"""
try:
return relational.next_legislative_day(db_path, as_of=as_of,
chamber=chamber)
except Exception: # noqa: BLE001 - forward source must not break /bills
return {"date": None, "recess": True}
def _scheduled_events(db_path, as_of, chamber=None, window_days=7):
"""Near-term scheduled chamber events (the "next 7 days" band).
Returns events from as_of through as_of+window_days, de-duplicated by
(date, kind, title). Degrades to [] when the table is absent / empty (a
recess). Never raises.
"""
try:
import datetime
start = _today_iso() if not as_of else str(as_of)
d0 = datetime.date.fromisoformat(start[:10])
end = (d0 + datetime.timedelta(days=window_days)).isoformat()
evs = relational.load_scheduled_events(db_path, as_of=start,
chamber=chamber)
seen = set()
out = []
for e in evs:
if e.get("date") and e["date"] > end:
continue
k = (e.get("date"), e.get("kind"), e.get("title"))
if k in seen:
continue
seen.add(k)
out.append(e)
out.sort(key=lambda e: (str(e.get("date") or "z"),
str(e.get("kind") or ""),
str(e.get("title") or "")))
return out
except Exception: # noqa: BLE001
return []
def _client_bill(ev):
out = {k: ev.get(k) for k in _data._EVENT_FIELDS}
out["hearing_sort"] = hearing_sort(ev.get("hearing_time"))