feat(dashboard): add forward "what's next" signal + fix upcoming floor
The daily-file bill queue is a backward snapshot, so "upcoming" could only resurface the latest past day (the 08-31 snapshot read as today's queue) and there was no "what is scheduled next." leginfo bill status/history are backward-only, so the genuine future-dated signal is the chambers' "upcoming events" calendars. - api.py: floor the "upcoming" view at *today* (not the DB max date) when no as_of is pinned; surface next_legislative_day + scheduled_events (next 7 days) on /bills. Add _today_iso / _next_legislative_day / _scheduled_events, all degrading to a recess marker / [] (never 5xx). - relational.py: schema v2 adds the scheduled_events table + store_scheduled_events (idempotent per-day refresh), load_scheduled_events, next_legislative_day. - parsers.py: parse_events_page + classify_event_kind for the chamber events calendar; skip empty-title rows (decompose the <time> before the fallback). - ingest.py + cli.py: --store-events lane (cache-first, recess-safe) that parses the calendar HTML into scheduled_events. - sources.py + config.yaml: register assembly_events / senate_events (kind: events, empty_is_ok: true). - run_ingest.sh: run the events lane after the daily-file lane as non-fatal (skippable via CALI_SCRAPER_SKIP_EVENTS=1); daily-file exit still gates cron. - frontend: "Next legislative day" line + "What's next · next 7 days" band with honest recess empty states. - DASHBOARD.md: document the forward signal + known limitations (leginfo backward-only gap, recess empty state, 08-31 snapshot is analysis-only). - tests: 5 new API tests + 18 parse-events tests; 170 pass, 92% coverage.
This commit is contained in:
kfj001 committed
1 parent
1919d43bf0
commit
03eeb6bb62
16 files changed
+941
-70
No files matched your search
@@ -0,0 +1,217 @@
|
||||
"""Unit tests for scraper.parsers.parse_events_page (the chamber "upcoming"
|
||||
events calendar parser, schema v2 / scheduled_events forward source).
|
||||
|
||||
The bill queue (daily file) is a backward snapshot, so the chamber calendar is
|
||||
the only forward-looking signal. These tests cover the new parser branches:
|
||||
date-from-<time> attribute, kind classification, source-url resolution, header
|
||||
row skipping, the <tr>-with-<time> fallback, and the empty/recess case.
|
||||
|
||||
The parser is the coverage-gated module, so these aim for complete branch
|
||||
coverage of parse_events_page + classify_event_kind.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
from scraper import parsers
|
||||
from scraper.parsers import parse_events_page, classify_event_kind
|
||||
|
||||
|
||||
# ---- a populated "views" table, matching the live assembly.ca.gov layout ----
|
||||
POPULATED_HTML = """
|
||||
<html><body>
|
||||
<div class="view-content">
|
||||
<table class="cols-3">
|
||||
<thead><tr>
|
||||
<th class="views-field views-field-media-source-pub-date">Date</th>
|
||||
<th class="views-field views-field-title">Title</th>
|
||||
<th class="views-field">Video</th>
|
||||
</tr></thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td class="views-field views-field-media-source-pub-date">
|
||||
<time datetime="2026-09-10T00:00:00-07:00">September 10, 2026</time>
|
||||
</td>
|
||||
<td class="views-field views-field-title">
|
||||
Assembly Floor Session</td>
|
||||
<td class="views-field">
|
||||
<a href="/media/assembly-floor-session-20260910">watch now</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="views-field views-field-media-source-pub-date">
|
||||
<time datetime="2026-09-11T00:00:00-07:00">September 11, 2026</time>
|
||||
</td>
|
||||
<td class="views-field views-field-title">
|
||||
Judiciary Committee</td>
|
||||
<td class="views-field"></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="views-field views-field-media-source-pub-date">
|
||||
<time datetime="2026-09-12T00:00:00-07:00">September 12, 2026</time>
|
||||
</td>
|
||||
<td class="views-field views-field-title">
|
||||
Press Conference</td>
|
||||
<td class="views-field"><a href="http://example.com/x">link</a></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
# A recess: the table is present but carries no data rows (empty view-content).
|
||||
RECESS_HTML = """
|
||||
<html><body>
|
||||
<div class="view-content">
|
||||
<table class="cols-3">
|
||||
<thead><tr>
|
||||
<th class="views-field views-field-media-source-pub-date">Date</th>
|
||||
<th class="views-field views-field-title">Title</th>
|
||||
</tr></thead>
|
||||
<tbody></tbody>
|
||||
</table>
|
||||
<p>There are currently no upcoming committee hearings scheduled.</p>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
# Layout with no <table class="cols-3">: rows must be recovered by <time>.
|
||||
NO_COLS3_HTML = """
|
||||
<html><body>
|
||||
<tr>
|
||||
<td><time datetime="2026-09-13T00:00:00-07:00">September 13, 2026</time></td>
|
||||
<td>Budget Committee</td>
|
||||
</tr>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
|
||||
# ---- kind classification ---------------------------------------------------
|
||||
|
||||
def test_classify_floor():
|
||||
assert classify_event_kind("Assembly Floor Session") == "FLOOR"
|
||||
|
||||
|
||||
def test_classify_committee():
|
||||
assert classify_event_kind("Judiciary Committee") == "COMMITTEE"
|
||||
|
||||
|
||||
def test_classify_other():
|
||||
assert classify_event_kind("Press Conference") == "OTHER"
|
||||
|
||||
|
||||
def test_classify_none():
|
||||
assert classify_event_kind(None) == "OTHER"
|
||||
|
||||
|
||||
# ---- populated page --------------------------------------------------------
|
||||
|
||||
def test_populated_page_parses_three_events():
|
||||
evs = parse_events_page(POPULATED_HTML, chamber="assembly",
|
||||
source="assembly_events",
|
||||
base_url="https://www.assembly.ca.gov/events/upcoming")
|
||||
assert len(evs) == 3
|
||||
by_date = {e["date"]: e for e in evs}
|
||||
assert by_date["2026-09-10"]["kind"] == "FLOOR"
|
||||
assert by_date["2026-09-11"]["kind"] == "COMMITTEE"
|
||||
assert by_date["2026-09-12"]["kind"] == "OTHER"
|
||||
|
||||
|
||||
def test_dates_come_from_time_attribute():
|
||||
evs = parse_events_page(POPULATED_HTML)
|
||||
dates = sorted(e["date"] for e in evs)
|
||||
assert dates == ["2026-09-10", "2026-09-11", "2026-09-12"]
|
||||
|
||||
|
||||
def test_chamber_normalised_to_upper():
|
||||
evs = parse_events_page(POPULATED_HTML, chamber=" Assembly ")
|
||||
assert all(e["chamber"] == "ASSEMBLY" for e in evs)
|
||||
|
||||
|
||||
def test_relative_link_resolves_against_host():
|
||||
evs = parse_events_page(POPULATED_HTML,
|
||||
base_url="https://www.assembly.ca.gov/events/upcoming")
|
||||
floor = next(e for e in evs if e["kind"] == "FLOOR")
|
||||
assert floor["source_url"] == \
|
||||
"https://www.assembly.ca.gov/media/assembly-floor-session-20260910"
|
||||
|
||||
|
||||
def test_row_without_link_falls_back_to_base_url():
|
||||
evs = parse_events_page(POPULATED_HTML,
|
||||
base_url="https://www.assembly.ca.gov/events/upcoming")
|
||||
comm = next(e for e in evs if e["kind"] == "COMMITTEE")
|
||||
assert comm["source_url"] == "https://www.assembly.ca.gov/events/upcoming"
|
||||
|
||||
|
||||
def test_external_link_kept_verbatim():
|
||||
evs = parse_events_page(POPULATED_HTML,
|
||||
base_url="https://www.assembly.ca.gov/events/upcoming")
|
||||
other = next(e for e in evs if e["kind"] == "OTHER")
|
||||
assert other["source_url"] == "http://example.com/x"
|
||||
|
||||
|
||||
def test_header_row_skipped():
|
||||
evs = parse_events_page(POPULATED_HTML)
|
||||
assert all(e["title"] not in ("Date", "Title") for e in evs)
|
||||
|
||||
|
||||
# ---- empty / recess --------------------------------------------------------
|
||||
|
||||
def test_recess_yields_no_events():
|
||||
assert parse_events_page(RECESS_HTML) == []
|
||||
|
||||
|
||||
def test_empty_string_yields_no_events():
|
||||
assert parse_events_page("") == []
|
||||
|
||||
|
||||
def test_html_with_no_time_cells_yields_no_events():
|
||||
assert parse_events_page("<html><body><p>Nothing here.</p></body></html>") == []
|
||||
|
||||
|
||||
# ---- layout fallback + bad rows -------------------------------------------
|
||||
|
||||
class _Broken:
|
||||
"""A soup stand-in whose find() raises, exercised via parse error path."""
|
||||
pass
|
||||
|
||||
|
||||
def test_no_cols3_falls_back_to_time_rows():
|
||||
evs = parse_events_page(NO_COLS3_HTML, chamber="assembly")
|
||||
assert len(evs) == 1
|
||||
assert evs[0]["date"] == "2026-09-13"
|
||||
assert evs[0]["kind"] == "COMMITTEE"
|
||||
|
||||
|
||||
def test_malformed_row_skipped_not_fatal():
|
||||
# A row with a <time> but no title cell is skipped, the good row survives.
|
||||
html = """
|
||||
<html><body><table class="cols-3"><tbody>
|
||||
<tr><td><time datetime="2026-09-14">Sep 14</time></td><td></td></tr>
|
||||
<tr><td><time datetime="2026-09-15">Sep 15</time></td>
|
||||
<td class="views-field views-field-title">Rules Committee</td></tr>
|
||||
</tbody></table></body></html>
|
||||
"""
|
||||
evs = parse_events_page(html, chamber="assembly")
|
||||
assert len(evs) == 1
|
||||
assert evs[0]["title"] == "Rules Committee"
|
||||
|
||||
|
||||
def test_broken_soup_returns_empty():
|
||||
# A document that cannot be parsed at all yields [] (never raises).
|
||||
result = parse_events_page(None, parser="lxml")
|
||||
assert result == []
|
||||
|
||||
|
||||
# ---- live sample (regression, network-independent) -------------------------
|
||||
|
||||
def test_sample_past_page_parses(tmp_path):
|
||||
sample = os.path.join(os.path.dirname(os.path.dirname(__file__)),
|
||||
"samples", "assembly_events_past.html")
|
||||
if not os.path.exists(sample):
|
||||
pytest.skip("sample not present")
|
||||
import os as _os
|
||||
html = open(sample, encoding="utf-8").read()
|
||||
evs = parse_events_page(html, chamber="assembly", source="assembly_events")
|
||||
assert len(evs) >= 2
|
||||
assert all(e["chamber"] == "ASSEMBLY" for e in evs)
|
||||
assert all(e["date"] and len(e["date"]) == 10 for e in evs)
|
||||
Reference in new issue
Block a user