feat(dashboard): add forward "what's next" signal + fix upcoming floor

The daily-file bill queue is a backward snapshot, so "upcoming" could only
resurface the latest past day (the 08-31 snapshot read as today's queue) and
there was no "what is scheduled next." leginfo bill status/history are
backward-only, so the genuine future-dated signal is the chambers' "upcoming
events" calendars.

- api.py: floor the "upcoming" view at *today* (not the DB max date) when no
  as_of is pinned; surface next_legislative_day + scheduled_events (next 7
  days) on /bills. Add _today_iso / _next_legislative_day / _scheduled_events,
  all degrading to a recess marker / [] (never 5xx).
- relational.py: schema v2 adds the scheduled_events table +
  store_scheduled_events (idempotent per-day refresh), load_scheduled_events,
  next_legislative_day.
- parsers.py: parse_events_page + classify_event_kind for the chamber events
  calendar; skip empty-title rows (decompose the <time> before the fallback).
- ingest.py + cli.py: --store-events lane (cache-first, recess-safe) that
  parses the calendar HTML into scheduled_events.
- sources.py + config.yaml: register assembly_events / senate_events
  (kind: events, empty_is_ok: true).
- run_ingest.sh: run the events lane after the daily-file lane as non-fatal
  (skippable via CALI_SCRAPER_SKIP_EVENTS=1); daily-file exit still gates cron.
- frontend: "Next legislative day" line + "What's next · next 7 days" band
  with honest recess empty states.
- DASHBOARD.md: document the forward signal + known limitations (leginfo
  backward-only gap, recess empty state, 08-31 snapshot is analysis-only).
- tests: 5 new API tests + 18 parse-events tests; 170 pass, 92% coverage.
This commit is contained in:
kfj001 committed 2026-09-06 19:50:02 -07:00
1 parent 1919d43bf0
commit 03eeb6bb62
16 files changed
+941 -70

No files matched your search

+88
View File
@@ -214,6 +214,94 @@ def test_empty_no_tagged_bills_and_chamber_pending(tmp_path, services):
assert status == 200 and body["items"] == []
# ---- forward "upcoming" source: scheduled_events + next_legislative_day ----
def _events_db(tmp_path):
"""A small DB whose flat snapshot is in the PAST but whose chamber
scheduled_events table holds future-dated events (the forward source)."""
p = str(tmp_path / "fwd.sqlite")
write_sqlite([
BillAction(date="2026-08-31", house="assembly", record_type="bill",
bill_number="AB 1099", bill_id="202520260AB1099",
session="2025-2026", chamber_origin="assembly",
author="Bryan", short_title="Developmental services",
description="Read third time. Passed. Ordered to the Assembly.",
source="ASSEMBLY_DAILY_FILE")], p, table=relational.FLAT_TABLE)
relational.migrate(p)
relational.store_scheduled_events([
{"date": "2026-09-08", "chamber": "ASSEMBLY", "kind": "FLOOR",
"title": "Assembly Floor Session",
"source_url": "https://www.assembly.ca.gov/events/upcoming",
"source": "assembly_events"},
{"date": "2026-09-09", "chamber": "ASSEMBLY", "kind": "COMMITTEE",
"title": "Judiciary Committee", "source": "assembly_events"},
], path=p, fetched_at="2026-09-06T00:00:00Z")
return p
def test_upcoming_defaults_to_not_past(tmp_path):
"""The bug that motivated this work: with upcoming=1 and no as_of, the
bill queue must floor at *today* (the past snapshot drops out), not at the
most-recent data date (which would re-surface yesterday as "upcoming")."""
fwd = _events_db(tmp_path)
svc = Services(fwd, PrefsStore(str(tmp_path / "p.json")),
LocalAuth(str(tmp_path / "a.json")))
status, body = api.run(_req("GET", "/api/v1/bills", svc,
query={"upcoming": "1"}))
assert status == 200, body
# the 08-31 snapshot is in the past relative to today -> not "upcoming"
assert body["items"] == []
# ...but the chamber calendar still answers "next":
assert body["next_legislative_day"]["date"] is not None
assert body["next_legislative_day"]["recess"] is False
assert len(body["scheduled_events"]) >= 1
def test_envelope_carries_forward_fields(tmp_path):
"""Envelope must carry the forward fields with a 7-day window."""
fwd = _events_db(tmp_path)
svc = Services(fwd, PrefsStore(str(tmp_path / "p.json")),
LocalAuth(str(tmp_path / "a.json")))
status, body = api.run(_req("GET", "/api/v1/bills", svc,
query={"upcoming": "1", "as_of": "2026-09-06"}))
assert status == 200, body
for key in ("next_legislative_day", "scheduled_events"):
assert key in body
nd = body["next_legislative_day"]
assert nd["date"] == "2026-09-08"
assert nd["kind"] == "FLOOR"
# 7-day window from 09-06 reaches 09-08 and 09-09, not further:
dates = {e["date"] for e in body["scheduled_events"]}
assert "2026-09-08" in dates and "2026-09-09" in dates
def test_next_day_recess_when_no_forward_events(tmp_path):
"""A pre-v2-style DB with no scheduled_events degrades to a recess marker
(never a 5xx): next_legislative_day is a {date: None, recess: True}."""
p = str(tmp_path / "empty.sqlite")
relational.migrate(p)
svc = Services(p, PrefsStore(str(tmp_path / "p.json")),
LocalAuth(str(tmp_path / "a.json")))
status, body = api.run(_req("GET", "/api/v1/bills", svc,
query={"upcoming": "1"}))
assert status == 200, body
assert body["next_legislative_day"] == {"date": None, "recess": True}
assert body["scheduled_events"] == []
def test_all_view_still_returns_history(tmp_path):
"""Omitting upcoming (the "All"/history view) must still return the past
snapshot rows -- the forward fix only changes the *upcoming* view."""
fwd = _events_db(tmp_path)
svc = Services(fwd, PrefsStore(str(tmp_path / "p.json")),
LocalAuth(str(tmp_path / "a.json")))
status, body = api.run(_req("GET", "/api/v1/bills", svc, query={}))
assert status == 200
assert len(body["items"]) >= 1
assert any(i["bill_number"] == "AB 1099" for i in body["items"])
# ---- /tags, /legislators, /districts (§4.1 / §3.2 / §4.2) -----------------
def test_tags_cloud(services):