feat(dashboard): add forward "what's next" signal + fix upcoming floor
The daily-file bill queue is a backward snapshot, so "upcoming" could only resurface the latest past day (the 08-31 snapshot read as today's queue) and there was no "what is scheduled next." leginfo bill status/history are backward-only, so the genuine future-dated signal is the chambers' "upcoming events" calendars. - api.py: floor the "upcoming" view at *today* (not the DB max date) when no as_of is pinned; surface next_legislative_day + scheduled_events (next 7 days) on /bills. Add _today_iso / _next_legislative_day / _scheduled_events, all degrading to a recess marker / [] (never 5xx). - relational.py: schema v2 adds the scheduled_events table + store_scheduled_events (idempotent per-day refresh), load_scheduled_events, next_legislative_day. - parsers.py: parse_events_page + classify_event_kind for the chamber events calendar; skip empty-title rows (decompose the <time> before the fallback). - ingest.py + cli.py: --store-events lane (cache-first, recess-safe) that parses the calendar HTML into scheduled_events. - sources.py + config.yaml: register assembly_events / senate_events (kind: events, empty_is_ok: true). - run_ingest.sh: run the events lane after the daily-file lane as non-fatal (skippable via CALI_SCRAPER_SKIP_EVENTS=1); daily-file exit still gates cron. - frontend: "Next legislative day" line + "What's next · next 7 days" band with honest recess empty states. - DASHBOARD.md: document the forward signal + known limitations (leginfo backward-only gap, recess empty state, 08-31 snapshot is analysis-only). - tests: 5 new API tests + 18 parse-events tests; 170 pass, 92% coverage.
This commit is contained in:
kfj001 committed
1 parent
1919d43bf0
commit
03eeb6bb62
16 files changed
+941
-70
No files matched your search
@@ -214,6 +214,94 @@ def test_empty_no_tagged_bills_and_chamber_pending(tmp_path, services):
|
||||
assert status == 200 and body["items"] == []
|
||||
|
||||
|
||||
|
||||
# ---- forward "upcoming" source: scheduled_events + next_legislative_day ----
|
||||
|
||||
|
||||
def _events_db(tmp_path):
|
||||
"""A small DB whose flat snapshot is in the PAST but whose chamber
|
||||
scheduled_events table holds future-dated events (the forward source)."""
|
||||
p = str(tmp_path / "fwd.sqlite")
|
||||
write_sqlite([
|
||||
BillAction(date="2026-08-31", house="assembly", record_type="bill",
|
||||
bill_number="AB 1099", bill_id="202520260AB1099",
|
||||
session="2025-2026", chamber_origin="assembly",
|
||||
author="Bryan", short_title="Developmental services",
|
||||
description="Read third time. Passed. Ordered to the Assembly.",
|
||||
source="ASSEMBLY_DAILY_FILE")], p, table=relational.FLAT_TABLE)
|
||||
relational.migrate(p)
|
||||
relational.store_scheduled_events([
|
||||
{"date": "2026-09-08", "chamber": "ASSEMBLY", "kind": "FLOOR",
|
||||
"title": "Assembly Floor Session",
|
||||
"source_url": "https://www.assembly.ca.gov/events/upcoming",
|
||||
"source": "assembly_events"},
|
||||
{"date": "2026-09-09", "chamber": "ASSEMBLY", "kind": "COMMITTEE",
|
||||
"title": "Judiciary Committee", "source": "assembly_events"},
|
||||
], path=p, fetched_at="2026-09-06T00:00:00Z")
|
||||
return p
|
||||
|
||||
|
||||
def test_upcoming_defaults_to_not_past(tmp_path):
|
||||
"""The bug that motivated this work: with upcoming=1 and no as_of, the
|
||||
bill queue must floor at *today* (the past snapshot drops out), not at the
|
||||
most-recent data date (which would re-surface yesterday as "upcoming")."""
|
||||
fwd = _events_db(tmp_path)
|
||||
svc = Services(fwd, PrefsStore(str(tmp_path / "p.json")),
|
||||
LocalAuth(str(tmp_path / "a.json")))
|
||||
status, body = api.run(_req("GET", "/api/v1/bills", svc,
|
||||
query={"upcoming": "1"}))
|
||||
assert status == 200, body
|
||||
# the 08-31 snapshot is in the past relative to today -> not "upcoming"
|
||||
assert body["items"] == []
|
||||
# ...but the chamber calendar still answers "next":
|
||||
assert body["next_legislative_day"]["date"] is not None
|
||||
assert body["next_legislative_day"]["recess"] is False
|
||||
assert len(body["scheduled_events"]) >= 1
|
||||
|
||||
|
||||
def test_envelope_carries_forward_fields(tmp_path):
|
||||
"""Envelope must carry the forward fields with a 7-day window."""
|
||||
fwd = _events_db(tmp_path)
|
||||
svc = Services(fwd, PrefsStore(str(tmp_path / "p.json")),
|
||||
LocalAuth(str(tmp_path / "a.json")))
|
||||
status, body = api.run(_req("GET", "/api/v1/bills", svc,
|
||||
query={"upcoming": "1", "as_of": "2026-09-06"}))
|
||||
assert status == 200, body
|
||||
for key in ("next_legislative_day", "scheduled_events"):
|
||||
assert key in body
|
||||
nd = body["next_legislative_day"]
|
||||
assert nd["date"] == "2026-09-08"
|
||||
assert nd["kind"] == "FLOOR"
|
||||
# 7-day window from 09-06 reaches 09-08 and 09-09, not further:
|
||||
dates = {e["date"] for e in body["scheduled_events"]}
|
||||
assert "2026-09-08" in dates and "2026-09-09" in dates
|
||||
|
||||
|
||||
def test_next_day_recess_when_no_forward_events(tmp_path):
|
||||
"""A pre-v2-style DB with no scheduled_events degrades to a recess marker
|
||||
(never a 5xx): next_legislative_day is a {date: None, recess: True}."""
|
||||
p = str(tmp_path / "empty.sqlite")
|
||||
relational.migrate(p)
|
||||
svc = Services(p, PrefsStore(str(tmp_path / "p.json")),
|
||||
LocalAuth(str(tmp_path / "a.json")))
|
||||
status, body = api.run(_req("GET", "/api/v1/bills", svc,
|
||||
query={"upcoming": "1"}))
|
||||
assert status == 200, body
|
||||
assert body["next_legislative_day"] == {"date": None, "recess": True}
|
||||
assert body["scheduled_events"] == []
|
||||
|
||||
|
||||
def test_all_view_still_returns_history(tmp_path):
|
||||
"""Omitting upcoming (the "All"/history view) must still return the past
|
||||
snapshot rows -- the forward fix only changes the *upcoming* view."""
|
||||
fwd = _events_db(tmp_path)
|
||||
svc = Services(fwd, PrefsStore(str(tmp_path / "p.json")),
|
||||
LocalAuth(str(tmp_path / "a.json")))
|
||||
status, body = api.run(_req("GET", "/api/v1/bills", svc, query={}))
|
||||
assert status == 200
|
||||
assert len(body["items"]) >= 1
|
||||
assert any(i["bill_number"] == "AB 1099" for i in body["items"])
|
||||
|
||||
# ---- /tags, /legislators, /districts (§4.1 / §3.2 / §4.2) -----------------
|
||||
|
||||
def test_tags_cloud(services):
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
"""Unit tests for scraper.parsers.parse_events_page (the chamber "upcoming"
|
||||
events calendar parser, schema v2 / scheduled_events forward source).
|
||||
|
||||
The bill queue (daily file) is a backward snapshot, so the chamber calendar is
|
||||
the only forward-looking signal. These tests cover the new parser branches:
|
||||
date-from-<time> attribute, kind classification, source-url resolution, header
|
||||
row skipping, the <tr>-with-<time> fallback, and the empty/recess case.
|
||||
|
||||
The parser is the coverage-gated module, so these aim for complete branch
|
||||
coverage of parse_events_page + classify_event_kind.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
from scraper import parsers
|
||||
from scraper.parsers import parse_events_page, classify_event_kind
|
||||
|
||||
|
||||
# ---- a populated "views" table, matching the live assembly.ca.gov layout ----
|
||||
POPULATED_HTML = """
|
||||
<html><body>
|
||||
<div class="view-content">
|
||||
<table class="cols-3">
|
||||
<thead><tr>
|
||||
<th class="views-field views-field-media-source-pub-date">Date</th>
|
||||
<th class="views-field views-field-title">Title</th>
|
||||
<th class="views-field">Video</th>
|
||||
</tr></thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td class="views-field views-field-media-source-pub-date">
|
||||
<time datetime="2026-09-10T00:00:00-07:00">September 10, 2026</time>
|
||||
</td>
|
||||
<td class="views-field views-field-title">
|
||||
Assembly Floor Session</td>
|
||||
<td class="views-field">
|
||||
<a href="/media/assembly-floor-session-20260910">watch now</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="views-field views-field-media-source-pub-date">
|
||||
<time datetime="2026-09-11T00:00:00-07:00">September 11, 2026</time>
|
||||
</td>
|
||||
<td class="views-field views-field-title">
|
||||
Judiciary Committee</td>
|
||||
<td class="views-field"></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="views-field views-field-media-source-pub-date">
|
||||
<time datetime="2026-09-12T00:00:00-07:00">September 12, 2026</time>
|
||||
</td>
|
||||
<td class="views-field views-field-title">
|
||||
Press Conference</td>
|
||||
<td class="views-field"><a href="http://example.com/x">link</a></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
# A recess: the table is present but carries no data rows (empty view-content).
|
||||
RECESS_HTML = """
|
||||
<html><body>
|
||||
<div class="view-content">
|
||||
<table class="cols-3">
|
||||
<thead><tr>
|
||||
<th class="views-field views-field-media-source-pub-date">Date</th>
|
||||
<th class="views-field views-field-title">Title</th>
|
||||
</tr></thead>
|
||||
<tbody></tbody>
|
||||
</table>
|
||||
<p>There are currently no upcoming committee hearings scheduled.</p>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
# Layout with no <table class="cols-3">: rows must be recovered by <time>.
|
||||
NO_COLS3_HTML = """
|
||||
<html><body>
|
||||
<tr>
|
||||
<td><time datetime="2026-09-13T00:00:00-07:00">September 13, 2026</time></td>
|
||||
<td>Budget Committee</td>
|
||||
</tr>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
|
||||
# ---- kind classification ---------------------------------------------------
|
||||
|
||||
def test_classify_floor():
|
||||
assert classify_event_kind("Assembly Floor Session") == "FLOOR"
|
||||
|
||||
|
||||
def test_classify_committee():
|
||||
assert classify_event_kind("Judiciary Committee") == "COMMITTEE"
|
||||
|
||||
|
||||
def test_classify_other():
|
||||
assert classify_event_kind("Press Conference") == "OTHER"
|
||||
|
||||
|
||||
def test_classify_none():
|
||||
assert classify_event_kind(None) == "OTHER"
|
||||
|
||||
|
||||
# ---- populated page --------------------------------------------------------
|
||||
|
||||
def test_populated_page_parses_three_events():
|
||||
evs = parse_events_page(POPULATED_HTML, chamber="assembly",
|
||||
source="assembly_events",
|
||||
base_url="https://www.assembly.ca.gov/events/upcoming")
|
||||
assert len(evs) == 3
|
||||
by_date = {e["date"]: e for e in evs}
|
||||
assert by_date["2026-09-10"]["kind"] == "FLOOR"
|
||||
assert by_date["2026-09-11"]["kind"] == "COMMITTEE"
|
||||
assert by_date["2026-09-12"]["kind"] == "OTHER"
|
||||
|
||||
|
||||
def test_dates_come_from_time_attribute():
|
||||
evs = parse_events_page(POPULATED_HTML)
|
||||
dates = sorted(e["date"] for e in evs)
|
||||
assert dates == ["2026-09-10", "2026-09-11", "2026-09-12"]
|
||||
|
||||
|
||||
def test_chamber_normalised_to_upper():
|
||||
evs = parse_events_page(POPULATED_HTML, chamber=" Assembly ")
|
||||
assert all(e["chamber"] == "ASSEMBLY" for e in evs)
|
||||
|
||||
|
||||
def test_relative_link_resolves_against_host():
|
||||
evs = parse_events_page(POPULATED_HTML,
|
||||
base_url="https://www.assembly.ca.gov/events/upcoming")
|
||||
floor = next(e for e in evs if e["kind"] == "FLOOR")
|
||||
assert floor["source_url"] == \
|
||||
"https://www.assembly.ca.gov/media/assembly-floor-session-20260910"
|
||||
|
||||
|
||||
def test_row_without_link_falls_back_to_base_url():
|
||||
evs = parse_events_page(POPULATED_HTML,
|
||||
base_url="https://www.assembly.ca.gov/events/upcoming")
|
||||
comm = next(e for e in evs if e["kind"] == "COMMITTEE")
|
||||
assert comm["source_url"] == "https://www.assembly.ca.gov/events/upcoming"
|
||||
|
||||
|
||||
def test_external_link_kept_verbatim():
|
||||
evs = parse_events_page(POPULATED_HTML,
|
||||
base_url="https://www.assembly.ca.gov/events/upcoming")
|
||||
other = next(e for e in evs if e["kind"] == "OTHER")
|
||||
assert other["source_url"] == "http://example.com/x"
|
||||
|
||||
|
||||
def test_header_row_skipped():
|
||||
evs = parse_events_page(POPULATED_HTML)
|
||||
assert all(e["title"] not in ("Date", "Title") for e in evs)
|
||||
|
||||
|
||||
# ---- empty / recess --------------------------------------------------------
|
||||
|
||||
def test_recess_yields_no_events():
|
||||
assert parse_events_page(RECESS_HTML) == []
|
||||
|
||||
|
||||
def test_empty_string_yields_no_events():
|
||||
assert parse_events_page("") == []
|
||||
|
||||
|
||||
def test_html_with_no_time_cells_yields_no_events():
|
||||
assert parse_events_page("<html><body><p>Nothing here.</p></body></html>") == []
|
||||
|
||||
|
||||
# ---- layout fallback + bad rows -------------------------------------------
|
||||
|
||||
class _Broken:
|
||||
"""A soup stand-in whose find() raises, exercised via parse error path."""
|
||||
pass
|
||||
|
||||
|
||||
def test_no_cols3_falls_back_to_time_rows():
|
||||
evs = parse_events_page(NO_COLS3_HTML, chamber="assembly")
|
||||
assert len(evs) == 1
|
||||
assert evs[0]["date"] == "2026-09-13"
|
||||
assert evs[0]["kind"] == "COMMITTEE"
|
||||
|
||||
|
||||
def test_malformed_row_skipped_not_fatal():
|
||||
# A row with a <time> but no title cell is skipped, the good row survives.
|
||||
html = """
|
||||
<html><body><table class="cols-3"><tbody>
|
||||
<tr><td><time datetime="2026-09-14">Sep 14</time></td><td></td></tr>
|
||||
<tr><td><time datetime="2026-09-15">Sep 15</time></td>
|
||||
<td class="views-field views-field-title">Rules Committee</td></tr>
|
||||
</tbody></table></body></html>
|
||||
"""
|
||||
evs = parse_events_page(html, chamber="assembly")
|
||||
assert len(evs) == 1
|
||||
assert evs[0]["title"] == "Rules Committee"
|
||||
|
||||
|
||||
def test_broken_soup_returns_empty():
|
||||
# A document that cannot be parsed at all yields [] (never raises).
|
||||
result = parse_events_page(None, parser="lxml")
|
||||
assert result == []
|
||||
|
||||
|
||||
# ---- live sample (regression, network-independent) -------------------------
|
||||
|
||||
def test_sample_past_page_parses(tmp_path):
|
||||
sample = os.path.join(os.path.dirname(os.path.dirname(__file__)),
|
||||
"samples", "assembly_events_past.html")
|
||||
if not os.path.exists(sample):
|
||||
pytest.skip("sample not present")
|
||||
import os as _os
|
||||
html = open(sample, encoding="utf-8").read()
|
||||
evs = parse_events_page(html, chamber="assembly", source="assembly_events")
|
||||
assert len(evs) >= 2
|
||||
assert all(e["chamber"] == "ASSEMBLY" for e in evs)
|
||||
assert all(e["date"] and len(e["date"]) == 10 for e in evs)
|
||||
Reference in new issue
Block a user