\ No newline at end of file
diff --git a/samples/assembly_events_upcoming.html b/samples/assembly_events_upcoming.html
new file mode 100644
index 0000000..78cfff6
--- /dev/null
+++ b/samples/assembly_events_upcoming.html
@@ -0,0 +1,3 @@
+Events | California State Assembly Skip to main content
\ No newline at end of file
diff --git a/scraper/cli.py b/scraper/cli.py
index d99a5ef..1d5afe5 100644
--- a/scraper/cli.py
+++ b/scraper/cli.py
@@ -32,10 +32,21 @@ def cmd_ingest(args) -> int:
try:
ing = Ingestor(config_path=args.config, force=args.force)
sources = _sources_arg(args)
- result = ing.ingest(date, sources=sources)
+ if getattr(args, "store_events", False):
+ # Forward-looking chamber calendar -> scheduled_events table. Reuses
+ # the same cache-first pipeline; a recess (empty calendar) is a
+ # no-op, not a failure. Default to the events sources when none are
+ # named (the daily-file sources are a different lane).
+ if not sources:
+ sources = [n for n, s in ing.cfg.sources.items()
+ if s.get("kind") == "events"]
+ result = ing.ingest_events(date, sources=sources,
+ db_path=getattr(args, "db", None))
+ else:
+ result = ing.ingest(date, sources=sources)
ing.close()
except SystemExit as exc:
- # config / argparse errors surface here
+ # config / argparse errors surface here
print("ERROR: %s" % exc, file=sys.stderr)
return int(exc.code or 2)
@@ -172,11 +183,18 @@ def build_parser() -> argparse.ArgumentParser:
pi.add_argument("--date", help="YYYY-MM-DD (default: today)")
pi.add_argument("--source",
help="Chamber name(s), comma-separated "
- "(assembly_daily_file, senate_daily_file). Default: all.")
+ "(assembly_daily_file, senate_daily_file). Default: all.")
pi.add_argument("--force", action="store_true",
help="Re-fetch even if already cached.")
pi.add_argument("--json", action="store_true",
help="Emit the result as JSON.")
+ pi.add_argument("--store-events", dest="store_events", action="store_true",
+ help="Fetch + parse the chamber 'upcoming events' calendar "
+ "and store it in the scheduled_events table "
+ "(forward-looking; the daily-file queue is backward).")
+ pi.add_argument("--db", dest="db", default=None,
+ help="SQLite path for --store-events "
+ "(default: the relational lane's db/daily_file.sqlite).")
pl = sub.add_parser("list", help="List configured sources + URLs")
pc = sub.add_parser("check", help="Cache status for a date (no network)")
diff --git a/scraper/dashboard/DASHBOARD.md b/scraper/dashboard/DASHBOARD.md
index 65feece..e40fdc6 100644
--- a/scraper/dashboard/DASHBOARD.md
+++ b/scraper/dashboard/DASHBOARD.md
@@ -75,16 +75,42 @@ list:
marker and a "Floated: ..." reason line
- Zero-scored bills remain in their date-grouped order below
+## Forward signal: "what's next" (scheduled_events, schema v2)
+
+The daily-file bill queue is a *backward* snapshot — leginfo's bill status and
+bill history are past-action logs, so they cannot answer "what is scheduled
+next." The only genuine future-dated signal is the chambers' "upcoming events"
+calendars (`assembly.ca.gov/events/upcoming`, `senate.ca.gov/calendar`). These
+are fetched (`python3 -m scraper ingest --store-events`), parsed into the
+`scheduled_events` table (relational schema v2), and surfaced on `/bills` as two
+new envelope keys:
+
+- `next_legislative_day` — `{date, kind, chamber, title, source}` (the earliest
+ event on/after `as_of`), or `{date: null, recess: true}` when nothing is
+ scheduled.
+- `scheduled_events` — the next 7 days of chamber events.
+
+The "Upcoming" filter is a *forward* view: with `upcoming=1` and no explicit
+`as_of`, the bill queue floors at **today**, so a stale/older daily-file
+snapshot can no longer present itself as "upcoming." Omitting `upcoming`
+(the "All" / history view) still returns every row.
+
+The dashboard renders a "Next legislative day" line in the freshness band and a
+"What's next · next 7 days" band. A recess (empty calendar) is an *honest*
+empty state, not an error.
+
## Tests
```bash
-python3 -m pytest tests/test_dashboard_ranking.py tests/test_dashboard_api.py
+python3 -m pytest tests/test_dashboard_ranking.py tests/test_dashboard_api.py \
+ tests/test_parse_events.py
```
-32 tests (12 ranking + 20 API), all passing. Pure ranking logic is tested in
-isolation; API handlers are tested with no sockets via the
+55 tests (12 ranking + 25 API + 18 parse-events), all passing. Pure ranking
+logic is tested in isolation; API handlers are tested with no sockets via the
`handler(req) -> (status, body)` pattern; a live `ThreadingHTTPServer` smoke test
-confirms static + API endpoints serve.
+confirms static + API endpoints serve. The event parser is tested against
+inline fixtures plus the saved `samples/assembly_events_*.html` pages.
## DASHBOARD.md
@@ -110,3 +136,14 @@ cannot be auto-approved).
no roster data is supplied.
- Dev auth is a JSON-file local account, not production auth.
- ZIP -> district uses a documented demo dataset until roster ETL lands.
+- **Forward signal gap:** leginfo's bill status/history are *backward-only*
+ (past actions + status narrative, no future dates), so there is no per-bill
+ "next step" label. The dashboard's forward answer comes solely from the
+ chamber events calendars (`scheduled_events`).
+- **Recess empty state:** while the Assembly (and at times the Senate) is in
+ final recess, `/events/upcoming` legitimately returns **0 events**. The
+ "What's next" band and `next_legislative_day` then show an honest "recess /
+ unknown" state rather than fabricating a date. This is expected, not a
+ parsing failure.
+- The 2026-08-31 daily-file snapshot is analysis-only; it is *not* re-surfaced
+ as "upcoming" because the upcoming view floors at today.
diff --git a/scraper/dashboard/api.py b/scraper/dashboard/api.py
index 748116d..2e0f3fa 100644
--- a/scraper/dashboard/api.py
+++ b/scraper/dashboard/api.py
@@ -13,6 +13,7 @@ from . import data as _data
from . import ranking
from . import resolve as _resolve
from . import bill_status
+from .. import relational
from .data import (load_tag_cloud, list_events,
load_legislators, committee_roster_for)
from .ranking import hearing_sort, full_queue, personal_band
@@ -117,17 +118,25 @@ def _truthy(v):
def handle_bills(req):
q = req.query
+ # "Upcoming" is a FORWARD view: when the caller asks for upcoming without
+ # pinning an as_of, floor it at *today*, not the most-recent data date.
+ # Flooring at today is the fix for "upcoming shows the latest PAST day";
+ # a None as_of would fall back to the DB max date and re-introduce it.
+ upcoming = _truthy(q.get("upcoming"))
+ as_of = q.get("as_of")
+ if upcoming and not as_of:
+ as_of = _today_iso()
events = list_events(
- db_path=req.svc.db_path,
- upcoming=_truthy(q.get("upcoming")),
- from_date=q.get("from") or q.get("date_from"),
- to_date=q.get("to") or q.get("date_to"),
- chamber=q.get("chamber"),
- tags=q_list("tags", q),
- q=q.get("q"),
- author=q.get("author"),
- as_of=q.get("as_of"),
- )
+ db_path=req.svc.db_path,
+ upcoming=upcoming,
+ from_date=q.get("from") or q.get("date_from"),
+ to_date=q.get("to") or q.get("date_to"),
+ chamber=q.get("chamber"),
+ tags=q_list("tags", q),
+ q=q.get("q"),
+ author=q.get("author"),
+ as_of=as_of,
+ )
if _truthy(q.get("personal")):
uid, err = req.require_auth()
if err:
@@ -143,7 +152,7 @@ def handle_bills(req):
events = full_queue(events)
else:
events = personal_band(
- events, int(q.get("band_n") or ranking.PERSONAL_BAND_N))
+ events, int(q.get("band_n") or ranking.PERSONAL_BAND_N))
session = _detect_session(events, req.svc.db_path)
items = [_client_bill(e) for e in events]
groups = {}
@@ -151,16 +160,76 @@ def handle_bills(req):
key = e.get("date") or "TBD"
groups[key] = groups.get(key, 0) + 1
chamber_avail = _chamber_availability(items, req.svc.db_path)
+ # The bill queue is a backward daily-file snapshot; the chamber "upcoming
+ # events" calendar (scheduled_events, schema v2) is the forward-looking
+ # signal. Surface "next legislative day" + the next-7-days events so a bill
+ # like AB-1099 shows what is scheduled next, not just the past. A None
+ # as_of (the "All"/history view) still answers "next" from today onward.
+ floor = as_of if as_of else _today_iso()
+ next_day = _next_legislative_day(req.svc.db_path, floor, q.get("chamber"))
+ scheduled = _scheduled_events(req.svc.db_path, floor, q.get("chamber"))
body = {
- "session": session,
- "as_of": q.get("as_of") or _now_iso(),
- "chamber_availability": chamber_avail,
- "items": items,
- "groups": groups,
- }
+ "session": session,
+ "as_of": as_of or floor,
+ "chamber_availability": chamber_avail,
+ "next_legislative_day": next_day,
+ "scheduled_events": scheduled,
+ "items": items,
+ "groups": groups,
+ }
return 200, body
+def _today_iso() -> str:
+ import datetime
+ return datetime.date.today().isoformat()
+
+
+def _next_legislative_day(db_path, as_of, chamber=None):
+ """Earliest scheduled chamber event on/after as_of (DESIGN section 6.1).
+
+ Degrades gracefully: when the scheduled_events table is absent (pre-v2 DB)
+ or empty for the window (a recess), returns ``{"date": None, "recess":
+ True}`` so the UI can show "next legislative day unknown". Never raises.
+ """
+ try:
+ return relational.next_legislative_day(db_path, as_of=as_of,
+ chamber=chamber)
+ except Exception: # noqa: BLE001 - forward source must not break /bills
+ return {"date": None, "recess": True}
+
+
+def _scheduled_events(db_path, as_of, chamber=None, window_days=7):
+ """Near-term scheduled chamber events (the "next 7 days" band).
+
+ Returns events from as_of through as_of+window_days, de-duplicated by
+ (date, kind, title). Degrades to [] when the table is absent / empty (a
+ recess). Never raises.
+ """
+ try:
+ import datetime
+ start = _today_iso() if not as_of else str(as_of)
+ d0 = datetime.date.fromisoformat(start[:10])
+ end = (d0 + datetime.timedelta(days=window_days)).isoformat()
+ evs = relational.load_scheduled_events(db_path, as_of=start,
+ chamber=chamber)
+ seen = set()
+ out = []
+ for e in evs:
+ if e.get("date") and e["date"] > end:
+ continue
+ k = (e.get("date"), e.get("kind"), e.get("title"))
+ if k in seen:
+ continue
+ seen.add(k)
+ out.append(e)
+ out.sort(key=lambda e: (str(e.get("date") or "z"),
+ str(e.get("kind") or ""),
+ str(e.get("title") or "")))
+ return out
+ except Exception: # noqa: BLE001
+ return []
+
def _client_bill(ev):
out = {k: ev.get(k) for k in _data._EVENT_FIELDS}
out["hearing_sort"] = hearing_sort(ev.get("hearing_time"))
diff --git a/scraper/dashboard/static/app.js b/scraper/dashboard/static/app.js
index 3ab5192..9bcce37 100644
--- a/scraper/dashboard/static/app.js
+++ b/scraper/dashboard/static/app.js
@@ -175,16 +175,74 @@
wireRows(container);
}
- function niceDate(iso) {
- try {
- var d = new Date(iso + "T00:00:00");
- return d.toLocaleDateString(undefined, {
- weekday: "short", month: "short", day: "numeric",
- year: "numeric" });
- } catch (e) { return iso; }
- }
+ function niceDate(iso) {
+ try {
+ var d = new Date(iso + "T00:00:00");
+ return d.toLocaleDateString(undefined, {
+ weekday: "short", month: "short", day: "numeric",
+ year: "numeric" });
+ } catch (e) { return iso; }
+ }
- /* ---------- empty states (DESIGN §6) ---------- */
+ /* ---------- forward "what's next" (scheduled_events, schema v2) -------
+ The daily-file queue is a backward snapshot; the chamber "upcoming
+ events" calendar is the only genuine future-dated signal. A recess
+ (recess: true / empty) is an honest empty state, not an error. */
+ function eventRowHTML(ev) {
+ var url = ev.source_url || "";
+ var kindLabel = ev.kind && ev.kind !== "OTHER"
+ ? esc(ev.kind)
+ : (ev.chamber ? ev.chamber : "");
+ return '
diff --git a/scraper/dashboard/static/style.css b/scraper/dashboard/static/style.css
index cfd3abe..5eef23f 100644
--- a/scraper/dashboard/static/style.css
+++ b/scraper/dashboard/static/style.css
@@ -139,6 +139,19 @@ button.chip-toggle { min-width: 120px; }
.band-title .count { color: var(--ink); font-weight: 700; }
#personal-band .band-title { color: var(--accent); }
+/* ---- forward "what's next" band (scheduled_events) ---- */
+.next-band { background: #fbfcff; }
+.next-band .band-title { color: var(--accent); }
+.ev-row {
+ display: grid; grid-template-columns: 120px 1fr auto;
+ gap: 10px; align-items: baseline;
+ padding: 8px 8px 8px 0; border-bottom: 1px solid var(--border);
+}
+.ev-date { color: var(--muted); font-size: 13px; white-space: nowrap; }
+.ev-main { font-weight: 600; }
+.chip.ev-kind { background: #eef3fb; color: var(--asm); border-color: #cdd9ee; }
+#freshness-next-day { white-space: nowrap; }
+
.day-heading {
font-size: 13px; color: var(--muted); margin: 14px 0 6px;
padding-bottom: 4px; border-bottom: 1px solid var(--border); }
diff --git a/scraper/ingest.py b/scraper/ingest.py
index 50ea08c..9972728 100644
--- a/scraper/ingest.py
+++ b/scraper/ingest.py
@@ -97,20 +97,76 @@ class Ingestor:
timespec="seconds"))
self.log.record(event)
- # -- main API --
+ # -- main API --
def ingest(self, date_iso, source=None, sources=None):
names = sources or ([source] if source else self.cfg.source_names())
result = IngestResult(date=date_iso)
for name in names:
if name not in self.cfg.sources:
raise ValueError(
- "Unknown source '%s'. Known: %s"
- % (name, self.cfg.source_names()))
+ "Unknown source '%s'. Known: %s"
+ % (name, self.cfg.source_names()))
cr = self._ingest_chamber(name, date_iso)
result.chambers.append(cr)
return result
- # -- per-chamber --
+ # -- chamber events forward source (scheduled_events, schema v2) --
+ def ingest_events(self, date_iso, source=None, sources=None, db_path=None):
+ """Fetch + cache + parse + store the chamber "upcoming events" source.
+
+ Like :meth:`ingest`, this is cache-first (a date already on disk is served
+ with no network) and reuses the same source planner (the events source is
+ ``kind: events``). It parses the calendar HTML into scheduled-event rows
+ and stores them in the ``scheduled_events`` table.
+
+ ``db_path`` defaults to the relational lane DEFAULT_DB so the dashboard
+ can read what this writes. A recess (an empty calendar) is a legitimate
+ no-op, not an error: nothing is cleared and nothing is stored.
+ """
+ from . import relational
+ from . import parsers
+
+ db = db_path or relational.DEFAULT_DB
+ relational.migrate(db) # ensure the scheduled_events table exists
+ names = sources or ([source] if source else self.cfg.source_names())
+ result = IngestResult(date=date_iso)
+ for name in names:
+ cfg = self.cfg.source(name)
+ if cfg.get("kind") != "events":
+ raise ValueError(
+ "ingest_events needs an events source; %r is kind %r"
+ % (name, cfg.get("kind")))
+ cr = self._ingest_chamber(name, date_iso)
+
+ events = []
+ if cr.status == "ok" and cr.path is not None:
+ try:
+ html = cr.path.read_text(encoding="utf-8",
+ errors="replace")
+ except OSError as e:
+ html = None
+ cr.note += " | cannot read cached page: %s" % e
+ if html:
+ base = getattr(cr, "url", "") or ""
+ events = parsers.parse_events_page(
+ html, chamber=cfg.get("chamber") or cr.source, source=
+ name, base_url=base,)
+
+ # Store when a page was read (fresh or served from cache). A fresh
+ # timestamp lets a re-published calendar refresh its day on the next
+ # run while an unchanged read stays a no-op. An empty parse (recess)
+ # clears no days, so existing scheduled rows are preserved.
+ when = datetime.datetime.now(datetime.timezone.utc).strftime(
+ "%Y-%m-%dT%H:%M:%SZ")
+ summary = relational.store_scheduled_events(events, path=db,
+ fetched_at=when)
+ cr.note += (" | events: %d stored, %d stale cleared, %d total"
+ % (summary["stored"], summary["cleared_stale"],
+ summary["total"]))
+ result.chambers.append(cr)
+ return result
+
+ # -- per-chamber --
def _ingest_chamber(self, name, date_iso):
cfg = self.cfg.source(name)
plan = build_source_plan(name, date_iso, cfg)
diff --git a/scraper/parsers.py b/scraper/parsers.py
index 3428786..5a16084 100644
--- a/scraper/parsers.py
+++ b/scraper/parsers.py
@@ -315,3 +315,109 @@ def parse(input_path: str, *, house: str = "assembly",
doc_records = [r for r in doc_records if r.record_type != "hearing"]
records.extend(doc_records)
return records
+
+# -------------------------------------------------------------------
+# Chamber "upcoming events" calendar (scheduled_events source, v2)
+#
+# The chamber calendar (assembly.ca.gov/events/upcoming, senate.ca.gov/
+# calendar) is a Drupal "views" table (
with Date /
+# Title / Video columns). Each row is a scheduled hearing / floor session --
+# the only *forward-looking* signal (the daily file is a backward snapshot).
+# Rows are stored in scheduled_events (relational.store_scheduled_events).
+# An empty page (a recess) is a legitimate empty result, not an error.
+# -------------------------------------------------------------------
+
+# Title keywords that classify an event's kind (best-effort, never crash).
+_EV_KIND_PATTERNS = (
+ (re.compile(r"\bfloor\b", re.I), "FLOOR"),
+ (re.compile(r"\bcommittee\b", re.I), "COMMITTEE"),
+)
+
+
+def classify_event_kind(title):
+ """Map an event title to a kind: FLOOR / COMMITTEE / OTHER."""
+ t = title or ""
+ for pat, kind in _EV_KIND_PATTERNS:
+ if pat.search(t):
+ return kind
+ return "OTHER"
+
+
+def parse_events_page(html, chamber="assembly", source="assembly_events",
+ base_url="", parser="lxml"):
+ """Parse a chamber events calendar page into scheduled-event dicts.
+
+ Returns a list of dicts with keys: date (ISO YYYY-MM-DD), chamber
+ (upper), kind (FLOOR|COMMITTEE|OTHER), title, source_url. An empty /
+ recess page yields []. Rows without a resolvable date are skipped
+ (logged). Header rows (
) are ignored.
+ """
+ from bs4 import BeautifulSoup
+
+ engine = parser if parser in ("lxml", "html.parser") else "html.parser"
+ try:
+ soup = BeautifulSoup(html, engine)
+ except Exception as e: # noqa: BLE001
+ log.error("events: failed to parse document (%s): %s",
+ type(e).__name__, e)
+ return []
+
+ out = []
+ # Fall back to any
