feat(dashboard): add forward "what's next" signal + fix upcoming floor
The daily-file bill queue is a backward snapshot, so "upcoming" could only resurface the latest past day (the 08-31 snapshot read as today's queue) and there was no "what is scheduled next." leginfo bill status/history are backward-only, so the genuine future-dated signal is the chambers' "upcoming events" calendars. - api.py: floor the "upcoming" view at *today* (not the DB max date) when no as_of is pinned; surface next_legislative_day + scheduled_events (next 7 days) on /bills. Add _today_iso / _next_legislative_day / _scheduled_events, all degrading to a recess marker / [] (never 5xx). - relational.py: schema v2 adds the scheduled_events table + store_scheduled_events (idempotent per-day refresh), load_scheduled_events, next_legislative_day. - parsers.py: parse_events_page + classify_event_kind for the chamber events calendar; skip empty-title rows (decompose the <time> before the fallback). - ingest.py + cli.py: --store-events lane (cache-first, recess-safe) that parses the calendar HTML into scheduled_events. - sources.py + config.yaml: register assembly_events / senate_events (kind: events, empty_is_ok: true). - run_ingest.sh: run the events lane after the daily-file lane as non-fatal (skippable via CALI_SCRAPER_SKIP_EVENTS=1); daily-file exit still gates cron. - frontend: "Next legislative day" line + "What's next · next 7 days" band with honest recess empty states. - DASHBOARD.md: document the forward signal + known limitations (leginfo backward-only gap, recess empty state, 08-31 snapshot is analysis-only). - tests: 5 new API tests + 18 parse-events tests; 170 pass, 92% coverage.
This commit is contained in:
1 parent
1919d43bf0
commit
03eeb6bb62
16 files changed
+941
-70
No files matched your search
+86
-17
@@ -13,6 +13,7 @@ from . import data as _data
|
||||
from . import ranking
|
||||
from . import resolve as _resolve
|
||||
from . import bill_status
|
||||
from .. import relational
|
||||
from .data import (load_tag_cloud, list_events,
|
||||
load_legislators, committee_roster_for)
|
||||
from .ranking import hearing_sort, full_queue, personal_band
|
||||
@@ -117,17 +118,25 @@ def _truthy(v):
|
||||
|
||||
def handle_bills(req):
|
||||
q = req.query
|
||||
# "Upcoming" is a FORWARD view: when the caller asks for upcoming without
|
||||
# pinning an as_of, floor it at *today*, not the most-recent data date.
|
||||
# Flooring at today is the fix for "upcoming shows the latest PAST day";
|
||||
# a None as_of would fall back to the DB max date and re-introduce it.
|
||||
upcoming = _truthy(q.get("upcoming"))
|
||||
as_of = q.get("as_of")
|
||||
if upcoming and not as_of:
|
||||
as_of = _today_iso()
|
||||
events = list_events(
|
||||
db_path=req.svc.db_path,
|
||||
upcoming=_truthy(q.get("upcoming")),
|
||||
from_date=q.get("from") or q.get("date_from"),
|
||||
to_date=q.get("to") or q.get("date_to"),
|
||||
chamber=q.get("chamber"),
|
||||
tags=q_list("tags", q),
|
||||
q=q.get("q"),
|
||||
author=q.get("author"),
|
||||
as_of=q.get("as_of"),
|
||||
)
|
||||
db_path=req.svc.db_path,
|
||||
upcoming=upcoming,
|
||||
from_date=q.get("from") or q.get("date_from"),
|
||||
to_date=q.get("to") or q.get("date_to"),
|
||||
chamber=q.get("chamber"),
|
||||
tags=q_list("tags", q),
|
||||
q=q.get("q"),
|
||||
author=q.get("author"),
|
||||
as_of=as_of,
|
||||
)
|
||||
if _truthy(q.get("personal")):
|
||||
uid, err = req.require_auth()
|
||||
if err:
|
||||
@@ -143,7 +152,7 @@ def handle_bills(req):
|
||||
events = full_queue(events)
|
||||
else:
|
||||
events = personal_band(
|
||||
events, int(q.get("band_n") or ranking.PERSONAL_BAND_N))
|
||||
events, int(q.get("band_n") or ranking.PERSONAL_BAND_N))
|
||||
session = _detect_session(events, req.svc.db_path)
|
||||
items = [_client_bill(e) for e in events]
|
||||
groups = {}
|
||||
@@ -151,16 +160,76 @@ def handle_bills(req):
|
||||
key = e.get("date") or "TBD"
|
||||
groups[key] = groups.get(key, 0) + 1
|
||||
chamber_avail = _chamber_availability(items, req.svc.db_path)
|
||||
# The bill queue is a backward daily-file snapshot; the chamber "upcoming
|
||||
# events" calendar (scheduled_events, schema v2) is the forward-looking
|
||||
# signal. Surface "next legislative day" + the next-7-days events so a bill
|
||||
# like AB-1099 shows what is scheduled next, not just the past. A None
|
||||
# as_of (the "All"/history view) still answers "next" from today onward.
|
||||
floor = as_of if as_of else _today_iso()
|
||||
next_day = _next_legislative_day(req.svc.db_path, floor, q.get("chamber"))
|
||||
scheduled = _scheduled_events(req.svc.db_path, floor, q.get("chamber"))
|
||||
body = {
|
||||
"session": session,
|
||||
"as_of": q.get("as_of") or _now_iso(),
|
||||
"chamber_availability": chamber_avail,
|
||||
"items": items,
|
||||
"groups": groups,
|
||||
}
|
||||
"session": session,
|
||||
"as_of": as_of or floor,
|
||||
"chamber_availability": chamber_avail,
|
||||
"next_legislative_day": next_day,
|
||||
"scheduled_events": scheduled,
|
||||
"items": items,
|
||||
"groups": groups,
|
||||
}
|
||||
return 200, body
|
||||
|
||||
|
||||
def _today_iso() -> str:
|
||||
import datetime
|
||||
return datetime.date.today().isoformat()
|
||||
|
||||
|
||||
def _next_legislative_day(db_path, as_of, chamber=None):
|
||||
"""Earliest scheduled chamber event on/after as_of (DESIGN section 6.1).
|
||||
|
||||
Degrades gracefully: when the scheduled_events table is absent (pre-v2 DB)
|
||||
or empty for the window (a recess), returns ``{"date": None, "recess":
|
||||
True}`` so the UI can show "next legislative day unknown". Never raises.
|
||||
"""
|
||||
try:
|
||||
return relational.next_legislative_day(db_path, as_of=as_of,
|
||||
chamber=chamber)
|
||||
except Exception: # noqa: BLE001 - forward source must not break /bills
|
||||
return {"date": None, "recess": True}
|
||||
|
||||
|
||||
def _scheduled_events(db_path, as_of, chamber=None, window_days=7):
|
||||
"""Near-term scheduled chamber events (the "next 7 days" band).
|
||||
|
||||
Returns events from as_of through as_of+window_days, de-duplicated by
|
||||
(date, kind, title). Degrades to [] when the table is absent / empty (a
|
||||
recess). Never raises.
|
||||
"""
|
||||
try:
|
||||
import datetime
|
||||
start = _today_iso() if not as_of else str(as_of)
|
||||
d0 = datetime.date.fromisoformat(start[:10])
|
||||
end = (d0 + datetime.timedelta(days=window_days)).isoformat()
|
||||
evs = relational.load_scheduled_events(db_path, as_of=start,
|
||||
chamber=chamber)
|
||||
seen = set()
|
||||
out = []
|
||||
for e in evs:
|
||||
if e.get("date") and e["date"] > end:
|
||||
continue
|
||||
k = (e.get("date"), e.get("kind"), e.get("title"))
|
||||
if k in seen:
|
||||
continue
|
||||
seen.add(k)
|
||||
out.append(e)
|
||||
out.sort(key=lambda e: (str(e.get("date") or "z"),
|
||||
str(e.get("kind") or ""),
|
||||
str(e.get("title") or "")))
|
||||
return out
|
||||
except Exception: # noqa: BLE001
|
||||
return []
|
||||
|
||||
def _client_bill(ev):
|
||||
out = {k: ev.get(k) for k in _data._EVENT_FIELDS}
|
||||
out["hearing_sort"] = hearing_sort(ev.get("hearing_time"))
|
||||
|
||||
Reference in new issue
Block a user