feat(dashboard): add forward "what's next" signal + fix upcoming floor
The daily-file bill queue is a backward snapshot, so "upcoming" could only resurface the latest past day (the 08-31 snapshot read as today's queue) and there was no "what is scheduled next." leginfo bill status/history are backward-only, so the genuine future-dated signal is the chambers' "upcoming events" calendars. - api.py: floor the "upcoming" view at *today* (not the DB max date) when no as_of is pinned; surface next_legislative_day + scheduled_events (next 7 days) on /bills. Add _today_iso / _next_legislative_day / _scheduled_events, all degrading to a recess marker / [] (never 5xx). - relational.py: schema v2 adds the scheduled_events table + store_scheduled_events (idempotent per-day refresh), load_scheduled_events, next_legislative_day. - parsers.py: parse_events_page + classify_event_kind for the chamber events calendar; skip empty-title rows (decompose the <time> before the fallback). - ingest.py + cli.py: --store-events lane (cache-first, recess-safe) that parses the calendar HTML into scheduled_events. - sources.py + config.yaml: register assembly_events / senate_events (kind: events, empty_is_ok: true). - run_ingest.sh: run the events lane after the daily-file lane as non-fatal (skippable via CALI_SCRAPER_SKIP_EVENTS=1); daily-file exit still gates cron. - frontend: "Next legislative day" line + "What's next · next 7 days" band with honest recess empty states. - DASHBOARD.md: document the forward signal + known limitations (leginfo backward-only gap, recess empty state, 08-31 snapshot is analysis-only). - tests: 5 new API tests + 18 parse-events tests; 170 pass, 92% coverage.
This commit is contained in:
1 parent
1919d43bf0
commit
03eeb6bb62
16 files changed
+941
-70
No files matched your search
@@ -214,6 +214,94 @@ def test_empty_no_tagged_bills_and_chamber_pending(tmp_path, services):
|
||||
assert status == 200 and body["items"] == []
|
||||
|
||||
|
||||
|
||||
# ---- forward "upcoming" source: scheduled_events + next_legislative_day ----
|
||||
|
||||
|
||||
def _events_db(tmp_path):
|
||||
"""A small DB whose flat snapshot is in the PAST but whose chamber
|
||||
scheduled_events table holds future-dated events (the forward source)."""
|
||||
p = str(tmp_path / "fwd.sqlite")
|
||||
write_sqlite([
|
||||
BillAction(date="2026-08-31", house="assembly", record_type="bill",
|
||||
bill_number="AB 1099", bill_id="202520260AB1099",
|
||||
session="2025-2026", chamber_origin="assembly",
|
||||
author="Bryan", short_title="Developmental services",
|
||||
description="Read third time. Passed. Ordered to the Assembly.",
|
||||
source="ASSEMBLY_DAILY_FILE")], p, table=relational.FLAT_TABLE)
|
||||
relational.migrate(p)
|
||||
relational.store_scheduled_events([
|
||||
{"date": "2026-09-08", "chamber": "ASSEMBLY", "kind": "FLOOR",
|
||||
"title": "Assembly Floor Session",
|
||||
"source_url": "https://www.assembly.ca.gov/events/upcoming",
|
||||
"source": "assembly_events"},
|
||||
{"date": "2026-09-09", "chamber": "ASSEMBLY", "kind": "COMMITTEE",
|
||||
"title": "Judiciary Committee", "source": "assembly_events"},
|
||||
], path=p, fetched_at="2026-09-06T00:00:00Z")
|
||||
return p
|
||||
|
||||
|
||||
def test_upcoming_defaults_to_not_past(tmp_path):
|
||||
"""The bug that motivated this work: with upcoming=1 and no as_of, the
|
||||
bill queue must floor at *today* (the past snapshot drops out), not at the
|
||||
most-recent data date (which would re-surface yesterday as "upcoming")."""
|
||||
fwd = _events_db(tmp_path)
|
||||
svc = Services(fwd, PrefsStore(str(tmp_path / "p.json")),
|
||||
LocalAuth(str(tmp_path / "a.json")))
|
||||
status, body = api.run(_req("GET", "/api/v1/bills", svc,
|
||||
query={"upcoming": "1"}))
|
||||
assert status == 200, body
|
||||
# the 08-31 snapshot is in the past relative to today -> not "upcoming"
|
||||
assert body["items"] == []
|
||||
# ...but the chamber calendar still answers "next":
|
||||
assert body["next_legislative_day"]["date"] is not None
|
||||
assert body["next_legislative_day"]["recess"] is False
|
||||
assert len(body["scheduled_events"]) >= 1
|
||||
|
||||
|
||||
def test_envelope_carries_forward_fields(tmp_path):
|
||||
"""Envelope must carry the forward fields with a 7-day window."""
|
||||
fwd = _events_db(tmp_path)
|
||||
svc = Services(fwd, PrefsStore(str(tmp_path / "p.json")),
|
||||
LocalAuth(str(tmp_path / "a.json")))
|
||||
status, body = api.run(_req("GET", "/api/v1/bills", svc,
|
||||
query={"upcoming": "1", "as_of": "2026-09-06"}))
|
||||
assert status == 200, body
|
||||
for key in ("next_legislative_day", "scheduled_events"):
|
||||
assert key in body
|
||||
nd = body["next_legislative_day"]
|
||||
assert nd["date"] == "2026-09-08"
|
||||
assert nd["kind"] == "FLOOR"
|
||||
# 7-day window from 09-06 reaches 09-08 and 09-09, not further:
|
||||
dates = {e["date"] for e in body["scheduled_events"]}
|
||||
assert "2026-09-08" in dates and "2026-09-09" in dates
|
||||
|
||||
|
||||
def test_next_day_recess_when_no_forward_events(tmp_path):
|
||||
"""A pre-v2-style DB with no scheduled_events degrades to a recess marker
|
||||
(never a 5xx): next_legislative_day is a {date: None, recess: True}."""
|
||||
p = str(tmp_path / "empty.sqlite")
|
||||
relational.migrate(p)
|
||||
svc = Services(p, PrefsStore(str(tmp_path / "p.json")),
|
||||
LocalAuth(str(tmp_path / "a.json")))
|
||||
status, body = api.run(_req("GET", "/api/v1/bills", svc,
|
||||
query={"upcoming": "1"}))
|
||||
assert status == 200, body
|
||||
assert body["next_legislative_day"] == {"date": None, "recess": True}
|
||||
assert body["scheduled_events"] == []
|
||||
|
||||
|
||||
def test_all_view_still_returns_history(tmp_path):
|
||||
"""Omitting upcoming (the "All"/history view) must still return the past
|
||||
snapshot rows -- the forward fix only changes the *upcoming* view."""
|
||||
fwd = _events_db(tmp_path)
|
||||
svc = Services(fwd, PrefsStore(str(tmp_path / "p.json")),
|
||||
LocalAuth(str(tmp_path / "a.json")))
|
||||
status, body = api.run(_req("GET", "/api/v1/bills", svc, query={}))
|
||||
assert status == 200
|
||||
assert len(body["items"]) >= 1
|
||||
assert any(i["bill_number"] == "AB 1099" for i in body["items"])
|
||||
|
||||
# ---- /tags, /legislators, /districts (§4.1 / §3.2 / §4.2) -----------------
|
||||
|
||||
def test_tags_cloud(services):
|
||||
|
||||
Reference in new issue
Block a user