WesternTechnologies/cali_scraper
2
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(dashboard): add forward "what's next" signal + fix upcoming floor

Browse Source 
The daily-file bill queue is a backward snapshot, so "upcoming" could only
resurface the latest past day (the 08-31 snapshot read as today's queue) and
there was no "what is scheduled next." leginfo bill status/history are
backward-only, so the genuine future-dated signal is the chambers' "upcoming
events" calendars.

- api.py: floor the "upcoming" view at *today* (not the DB max date) when no
  as_of is pinned; surface next_legislative_day + scheduled_events (next 7
  days) on /bills. Add _today_iso / _next_legislative_day / _scheduled_events,
  all degrading to a recess marker / [] (never 5xx).
- relational.py: schema v2 adds the scheduled_events table +
  store_scheduled_events (idempotent per-day refresh), load_scheduled_events,
  next_legislative_day.
- parsers.py: parse_events_page + classify_event_kind for the chamber events
  calendar; skip empty-title rows (decompose the <time> before the fallback).
- ingest.py + cli.py: --store-events lane (cache-first, recess-safe) that
  parses the calendar HTML into scheduled_events.
- sources.py + config.yaml: register assembly_events / senate_events
  (kind: events, empty_is_ok: true).
- run_ingest.sh: run the events lane after the daily-file lane as non-fatal
  (skippable via CALI_SCRAPER_SKIP_EVENTS=1); daily-file exit still gates cron.
- frontend: "Next legislative day" line + "What's next · next 7 days" band
  with honest recess empty states.
- DASHBOARD.md: document the forward signal + known limitations (leginfo
  backward-only gap, recess empty state, 08-31 snapshot is analysis-only).
- tests: 5 new API tests + 18 parse-events tests; 170 pass, 92% coverage.
This commit is contained in:
kfj001 committed 2026-09-06 19:50:02 -07:00
1 parent 1919d43bf0
commit 03eeb6bb62
16 files changed
+941 -70
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scraper/relational.py
+159 -17
View File
@@ -51,7 +51,11 @@ DEFAULT_DB = os.path.join(_REPO_ROOT, "db", "daily_file.sqlite")
FLAT_TABLE = "daily_file_records"
#: Schema version stamped into ``schema_migrations`` by :func:`migrate`.
SCHEMA_VERSION = 1
# v1: bills/committees/members/committee_members/bill_committees + flat view.
# v2: adds scheduled_events (the chamber "upcoming events" forward source — the
# only genuine future-dated signal; see design/API_CONTRACT.md §5 + the
# "what's next" work). Kept OUT of the frozen flat daily_file_records view.
SCHEMA_VERSION = 2
#: The flat columns QA + downstream rely on. Imported so the flat-preserving
#: view is byte-for-byte aligned with the contract the rest of the pipeline
@@ -211,9 +215,27 @@ def _create_schema(conn: sqlite3.Connection) -> None:
applied_at TEXT,
note TEXT,
PRIMARY KEY (version)
);
"""
)
);
-- v2: the chamber "upcoming events" forward source. This is the ONLY
-- genuine *future-dated* signal (the daily file is a backward snapshot;
-- leginfo bill status/history is backward-only). It holds the next
-- scheduled floor/committee events parsed from the chamber events
-- calendar (assembly.ca.gov/events, senate.ca.gov/calendar). It is
-- deliberately OUT of the frozen flat daily_file_records view.
CREATE TABLE IF NOT EXISTS scheduled_events (
id INTEGER PRIMARY KEY,
date TEXT NOT NULL, -- ISO YYYY-MM-DD (event day)
chamber TEXT NOT NULL, -- ASSEMBLY | SENATE
kind TEXT, -- FLOOR | COMMITTEE | OTHER
title TEXT NOT NULL, -- human event title
source_url TEXT, -- where the event came from
fetched_at TEXT, -- when this page was read
source TEXT, -- assembly_events | senate_events
UNIQUE (date, chamber, kind, title, source)
);
"""
)
# Flat-preserving view: a drop-in mirror of the table QA/UI read. Column
# order and row set are identical to daily_file_records, ordered
# deterministically so the "flat contract unchanged" test is stable.
@@ -229,13 +251,13 @@ def _create_schema(conn: sqlite3.Connection) -> None:
def _stamp_migration(conn: sqlite3.Connection, version: int = SCHEMA_VERSION) -> None:
conn.execute(
"INSERT OR IGNORE INTO schema_migrations (version, applied_at, note) "
"VALUES (?, ?, ?)",
(version,
__import__("datetime").date.today().isoformat(),
"relational schema: bills/committees/members/committee_members/"
"bill_committees + flat view"),
)
"INSERT OR IGNORE INTO schema_migrations (version, applied_at, note) "
"VALUES (?, ?, ?)",
(version,
__import__("datetime").date.today().isoformat(),
"relational schema: bills/committees/members/committee_members/"
"bill_committees + flat view + scheduled_events (v2)"),
)
conn.commit()
@@ -514,12 +536,132 @@ def sample_selects(path: str = DEFAULT_DB, limit: int = 5) -> Dict[str, Any]:
"JOIN committees c ON c.name_key = cm.committee "
"JOIN members m ON m.name_norm = cm.member "
"ORDER BY c.name_raw, m.full_name LIMIT ?", (limit,)),
"committees_without_roster": q(
"SELECT name_raw, chamber, session FROM committees c "
"WHERE NOT EXISTS (SELECT 1 FROM committee_members cm "
" WHERE cm.committee = c.name_key "
" AND cm.chamber = c.chamber "
" AND cm.session = c.session)"),
"committees_without_roster": q(
"SELECT name_raw, chamber, session FROM committees c "
"WHERE NOT EXISTS (SELECT 1 FROM committee_members cm "
" WHERE cm.committee = c.name_key "
" AND cm.chamber = c.chamber "
" AND cm.session = c.session)"),
}
finally:
conn.close()
# ---------------------------------------------------------------------------
# scheduled_events: the chamber "upcoming events" forward source (v2)
# ---------------------------------------------------------------------------
def _scheduled_events_table(conn: sqlite3.Connection, name: str = "scheduled_events") -> bool:
cur = conn.execute(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?",
(name,))
return cur.fetchone() is not None
def store_scheduled_events(events: Iterable[Dict[str, Any]],
path: str = DEFAULT_DB,
fetched_at: str = "") -> Dict[str, Any]:
"""Insert parsed chamber events into ``scheduled_events`` idempotently.
Each ``event`` is a dict with keys ``date`` (ISO), ``chamber`` (upper),
``kind`` (``FLOOR``/``COMMITTEE``/``OTHER``), ``title``, ``source_url``,
``source``. The natural key ``(date, chamber, kind, title, source)`` makes
re-runs a no-op (``INSERT OR IGNORE``), so the existing-cron behaviour
("re-parse doesn't re-hit") holds: a second parse of the same calendar day
adds no rows. Ensures the table exists (via :func:`_create_schema`). Returns
a small summary ``{stored, cleared_stale, total, fetched_at}``.
"""
conn = connect(path)
try:
_create_schema(conn) # CREATE IF NOT EXISTS; cheap + idempotent
# Drop a stale day's rows first so a re-published calendar (a moved or
# renamed hearing) reflects the latest read rather than a frozen
# snapshot. Only that day is cleared; other days are untouched.
days = {str(e.get("date")) for e in events if e.get("date")}
deleted = 0
for d in days:
cur = conn.execute(
"DELETE FROM scheduled_events WHERE date=? AND fetched_at<>?",
(d, fetched_at))
deleted += cur.rowcount
rows = []
for e in events:
if not e.get("date") or not e.get("title"):
continue
rows.append((
str(e["date"]).strip(),
(e.get("chamber") or "unknown").strip().upper(),
(e.get("kind") or "OTHER").strip().upper() or "OTHER",
str(e["title"]).strip(),
e.get("source_url") or None,
fetched_at or None,
(e.get("source") or "assembly_events").strip()
or "assembly_events",
))
if rows:
cur = conn.executemany(
"INSERT OR IGNORE INTO scheduled_events "
"(date, chamber, kind, title, source_url, fetched_at, "
"source) VALUES (?,?,?,?,?,?,?)",
rows)
stored = cur.rowcount
else:
stored = 0
conn.commit()
total = conn.execute(
"SELECT COUNT(*) FROM scheduled_events").fetchone()[0]
return {"stored": stored, "cleared_stale": deleted,
"total": total, "fetched_at": fetched_at or None}
finally:
conn.close()
def load_scheduled_events(path: str = DEFAULT_DB,
as_of: Optional[str] = None,
chamber: Optional[str] = None,
limit: int = 200) -> List[Dict[str, Any]]:
"""Return scheduled (chamber) events, optionally future-dated from ``as_of``.
A ``None``/absent ``as_of`` keeps every event; a date string keeps
``date >= as_of`` (a T-1 event is excluded, matching API_CONTRACT §5).
Degrades to ``[]`` when the table does not exist (pre-v2 DBs / fresh schema).
"""
conn = connect(path, foreign_keys=False)
try:
if not _scheduled_events_table(conn):
return []
sql = ("SELECT date, chamber, kind, title, source_url, fetched_at, "
"source FROM scheduled_events WHERE 1=1")
clauses: list = []
args: list = []
if as_of:
clauses.append("date >= ?")
args.append(str(as_of))
if chamber:
clauses.append("UPPER(chamber) = ?")
args.append(chamber.strip().upper())
if clauses:
sql += " AND " + " AND ".join(clauses)
sql += " ORDER BY date, chamber, kind, title LIMIT ?"
args.append(int(limit))
rows = conn.execute(sql, args).fetchall()
finally:
conn.close()
return [dict(r) for r in rows]
def next_legislative_day(path: str = DEFAULT_DB,
as_of: Optional[str] = None,
chamber: Optional[str] = None) -> Dict[str, Any]:
"""The earliest scheduled event on/after ``as_of`` (DESIGN §6.1 "next
legislative day"), or a ``{date: None, recess: True}`` marker when nothing is
scheduled (a genuine recess).
This is the dashboard's "what's next" answer when the daily-file bill queue
has no future rows: it surfaces the chamber events calendar. ``recess`` is
True when there is no event after ``as_of``.
"""
events = load_scheduled_events(path, as_of=as_of, chamber=chamber, limit=1)
if not events:
return {"date": None, "recess": True}
return {**events[0], "recess": False}