kfj001
03eeb6bb62
The daily-file bill queue is a backward snapshot, so "upcoming" could only resurface the latest past day (the 08-31 snapshot read as today's queue) and there was no "what is scheduled next." leginfo bill status/history are backward-only, so the genuine future-dated signal is the chambers' "upcoming events" calendars. - api.py: floor the "upcoming" view at *today* (not the DB max date) when no as_of is pinned; surface next_legislative_day + scheduled_events (next 7 days) on /bills. Add _today_iso / _next_legislative_day / _scheduled_events, all degrading to a recess marker / [] (never 5xx). - relational.py: schema v2 adds the scheduled_events table + store_scheduled_events (idempotent per-day refresh), load_scheduled_events, next_legislative_day. - parsers.py: parse_events_page + classify_event_kind for the chamber events calendar; skip empty-title rows (decompose the <time> before the fallback). - ingest.py + cli.py: --store-events lane (cache-first, recess-safe) that parses the calendar HTML into scheduled_events. - sources.py + config.yaml: register assembly_events / senate_events (kind: events, empty_is_ok: true). - run_ingest.sh: run the events lane after the daily-file lane as non-fatal (skippable via CALI_SCRAPER_SKIP_EVENTS=1); daily-file exit still gates cron. - frontend: "Next legislative day" line + "What's next · next 7 days" band with honest recess empty states. - DASHBOARD.md: document the forward signal + known limitations (leginfo backward-only gap, recess empty state, 08-31 snapshot is analysis-only). - tests: 5 new API tests + 18 parse-events tests; 170 pass, 92% coverage.
218 lines
7.4 KiB
Python
218 lines
7.4 KiB
Python
"""Unit tests for scraper.parsers.parse_events_page (the chamber "upcoming"
|
|
events calendar parser, schema v2 / scheduled_events forward source).
|
|
|
|
The bill queue (daily file) is a backward snapshot, so the chamber calendar is
|
|
the only forward-looking signal. These tests cover the new parser branches:
|
|
date-from-<time> attribute, kind classification, source-url resolution, header
|
|
row skipping, the <tr>-with-<time> fallback, and the empty/recess case.
|
|
|
|
The parser is the coverage-gated module, so these aim for complete branch
|
|
coverage of parse_events_page + classify_event_kind.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
|
|
from scraper import parsers
|
|
from scraper.parsers import parse_events_page, classify_event_kind
|
|
|
|
|
|
# ---- a populated "views" table, matching the live assembly.ca.gov layout ----
|
|
POPULATED_HTML = """
|
|
<html><body>
|
|
<div class="view-content">
|
|
<table class="cols-3">
|
|
<thead><tr>
|
|
<th class="views-field views-field-media-source-pub-date">Date</th>
|
|
<th class="views-field views-field-title">Title</th>
|
|
<th class="views-field">Video</th>
|
|
</tr></thead>
|
|
<tbody>
|
|
<tr>
|
|
<td class="views-field views-field-media-source-pub-date">
|
|
<time datetime="2026-09-10T00:00:00-07:00">September 10, 2026</time>
|
|
</td>
|
|
<td class="views-field views-field-title">
|
|
Assembly Floor Session</td>
|
|
<td class="views-field">
|
|
<a href="/media/assembly-floor-session-20260910">watch now</a></td>
|
|
</tr>
|
|
<tr>
|
|
<td class="views-field views-field-media-source-pub-date">
|
|
<time datetime="2026-09-11T00:00:00-07:00">September 11, 2026</time>
|
|
</td>
|
|
<td class="views-field views-field-title">
|
|
Judiciary Committee</td>
|
|
<td class="views-field"></td>
|
|
</tr>
|
|
<tr>
|
|
<td class="views-field views-field-media-source-pub-date">
|
|
<time datetime="2026-09-12T00:00:00-07:00">September 12, 2026</time>
|
|
</td>
|
|
<td class="views-field views-field-title">
|
|
Press Conference</td>
|
|
<td class="views-field"><a href="http://example.com/x">link</a></td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
</body></html>
|
|
"""
|
|
|
|
# A recess: the table is present but carries no data rows (empty view-content).
|
|
RECESS_HTML = """
|
|
<html><body>
|
|
<div class="view-content">
|
|
<table class="cols-3">
|
|
<thead><tr>
|
|
<th class="views-field views-field-media-source-pub-date">Date</th>
|
|
<th class="views-field views-field-title">Title</th>
|
|
</tr></thead>
|
|
<tbody></tbody>
|
|
</table>
|
|
<p>There are currently no upcoming committee hearings scheduled.</p>
|
|
</body></html>
|
|
"""
|
|
|
|
# Layout with no <table class="cols-3">: rows must be recovered by <time>.
|
|
NO_COLS3_HTML = """
|
|
<html><body>
|
|
<tr>
|
|
<td><time datetime="2026-09-13T00:00:00-07:00">September 13, 2026</time></td>
|
|
<td>Budget Committee</td>
|
|
</tr>
|
|
</body></html>
|
|
"""
|
|
|
|
|
|
# ---- kind classification ---------------------------------------------------
|
|
|
|
def test_classify_floor():
|
|
assert classify_event_kind("Assembly Floor Session") == "FLOOR"
|
|
|
|
|
|
def test_classify_committee():
|
|
assert classify_event_kind("Judiciary Committee") == "COMMITTEE"
|
|
|
|
|
|
def test_classify_other():
|
|
assert classify_event_kind("Press Conference") == "OTHER"
|
|
|
|
|
|
def test_classify_none():
|
|
assert classify_event_kind(None) == "OTHER"
|
|
|
|
|
|
# ---- populated page --------------------------------------------------------
|
|
|
|
def test_populated_page_parses_three_events():
|
|
evs = parse_events_page(POPULATED_HTML, chamber="assembly",
|
|
source="assembly_events",
|
|
base_url="https://www.assembly.ca.gov/events/upcoming")
|
|
assert len(evs) == 3
|
|
by_date = {e["date"]: e for e in evs}
|
|
assert by_date["2026-09-10"]["kind"] == "FLOOR"
|
|
assert by_date["2026-09-11"]["kind"] == "COMMITTEE"
|
|
assert by_date["2026-09-12"]["kind"] == "OTHER"
|
|
|
|
|
|
def test_dates_come_from_time_attribute():
|
|
evs = parse_events_page(POPULATED_HTML)
|
|
dates = sorted(e["date"] for e in evs)
|
|
assert dates == ["2026-09-10", "2026-09-11", "2026-09-12"]
|
|
|
|
|
|
def test_chamber_normalised_to_upper():
|
|
evs = parse_events_page(POPULATED_HTML, chamber=" Assembly ")
|
|
assert all(e["chamber"] == "ASSEMBLY" for e in evs)
|
|
|
|
|
|
def test_relative_link_resolves_against_host():
|
|
evs = parse_events_page(POPULATED_HTML,
|
|
base_url="https://www.assembly.ca.gov/events/upcoming")
|
|
floor = next(e for e in evs if e["kind"] == "FLOOR")
|
|
assert floor["source_url"] == \
|
|
"https://www.assembly.ca.gov/media/assembly-floor-session-20260910"
|
|
|
|
|
|
def test_row_without_link_falls_back_to_base_url():
|
|
evs = parse_events_page(POPULATED_HTML,
|
|
base_url="https://www.assembly.ca.gov/events/upcoming")
|
|
comm = next(e for e in evs if e["kind"] == "COMMITTEE")
|
|
assert comm["source_url"] == "https://www.assembly.ca.gov/events/upcoming"
|
|
|
|
|
|
def test_external_link_kept_verbatim():
|
|
evs = parse_events_page(POPULATED_HTML,
|
|
base_url="https://www.assembly.ca.gov/events/upcoming")
|
|
other = next(e for e in evs if e["kind"] == "OTHER")
|
|
assert other["source_url"] == "http://example.com/x"
|
|
|
|
|
|
def test_header_row_skipped():
|
|
evs = parse_events_page(POPULATED_HTML)
|
|
assert all(e["title"] not in ("Date", "Title") for e in evs)
|
|
|
|
|
|
# ---- empty / recess --------------------------------------------------------
|
|
|
|
def test_recess_yields_no_events():
|
|
assert parse_events_page(RECESS_HTML) == []
|
|
|
|
|
|
def test_empty_string_yields_no_events():
|
|
assert parse_events_page("") == []
|
|
|
|
|
|
def test_html_with_no_time_cells_yields_no_events():
|
|
assert parse_events_page("<html><body><p>Nothing here.</p></body></html>") == []
|
|
|
|
|
|
# ---- layout fallback + bad rows -------------------------------------------
|
|
|
|
class _Broken:
|
|
"""A soup stand-in whose find() raises, exercised via parse error path."""
|
|
pass
|
|
|
|
|
|
def test_no_cols3_falls_back_to_time_rows():
|
|
evs = parse_events_page(NO_COLS3_HTML, chamber="assembly")
|
|
assert len(evs) == 1
|
|
assert evs[0]["date"] == "2026-09-13"
|
|
assert evs[0]["kind"] == "COMMITTEE"
|
|
|
|
|
|
def test_malformed_row_skipped_not_fatal():
|
|
# A row with a <time> but no title cell is skipped, the good row survives.
|
|
html = """
|
|
<html><body><table class="cols-3"><tbody>
|
|
<tr><td><time datetime="2026-09-14">Sep 14</time></td><td></td></tr>
|
|
<tr><td><time datetime="2026-09-15">Sep 15</time></td>
|
|
<td class="views-field views-field-title">Rules Committee</td></tr>
|
|
</tbody></table></body></html>
|
|
"""
|
|
evs = parse_events_page(html, chamber="assembly")
|
|
assert len(evs) == 1
|
|
assert evs[0]["title"] == "Rules Committee"
|
|
|
|
|
|
def test_broken_soup_returns_empty():
|
|
# A document that cannot be parsed at all yields [] (never raises).
|
|
result = parse_events_page(None, parser="lxml")
|
|
assert result == []
|
|
|
|
|
|
# ---- live sample (regression, network-independent) -------------------------
|
|
|
|
def test_sample_past_page_parses(tmp_path):
|
|
sample = os.path.join(os.path.dirname(os.path.dirname(__file__)),
|
|
"samples", "assembly_events_past.html")
|
|
if not os.path.exists(sample):
|
|
pytest.skip("sample not present")
|
|
import os as _os
|
|
html = open(sample, encoding="utf-8").read()
|
|
evs = parse_events_page(html, chamber="assembly", source="assembly_events")
|
|
assert len(evs) >= 2
|
|
assert all(e["chamber"] == "ASSEMBLY" for e in evs)
|
|
assert all(e["date"] and len(e["date"]) == 10 for e in evs)
|