kfj001
03eeb6bb62
The daily-file bill queue is a backward snapshot, so "upcoming" could only resurface the latest past day (the 08-31 snapshot read as today's queue) and there was no "what is scheduled next." leginfo bill status/history are backward-only, so the genuine future-dated signal is the chambers' "upcoming events" calendars. - api.py: floor the "upcoming" view at *today* (not the DB max date) when no as_of is pinned; surface next_legislative_day + scheduled_events (next 7 days) on /bills. Add _today_iso / _next_legislative_day / _scheduled_events, all degrading to a recess marker / [] (never 5xx). - relational.py: schema v2 adds the scheduled_events table + store_scheduled_events (idempotent per-day refresh), load_scheduled_events, next_legislative_day. - parsers.py: parse_events_page + classify_event_kind for the chamber events calendar; skip empty-title rows (decompose the <time> before the fallback). - ingest.py + cli.py: --store-events lane (cache-first, recess-safe) that parses the calendar HTML into scheduled_events. - sources.py + config.yaml: register assembly_events / senate_events (kind: events, empty_is_ok: true). - run_ingest.sh: run the events lane after the daily-file lane as non-fatal (skippable via CALI_SCRAPER_SKIP_EVENTS=1); daily-file exit still gates cron. - frontend: "Next legislative day" line + "What's next · next 7 days" band with honest recess empty states. - DASHBOARD.md: document the forward signal + known limitations (leginfo backward-only gap, recess empty state, 08-31 snapshot is analysis-only). - tests: 5 new API tests + 18 parse-events tests; 170 pass, 92% coverage.
109 lines
3.8 KiB
Python
109 lines
3.8 KiB
Python
# Resolve a chamber source for a given date -> the URL(s) to fetch + cache file
|
|
# extension. Neither daily-file page is date-addressable by URL, so the date only
|
|
# drives the Senate PDF filename and the cache key; the Assembly URL is constant.
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from dataclasses import dataclass, field
|
|
from typing import Optional
|
|
|
|
# Senate daily-file PDF filename pattern: SEN_{type}_{year}_R_{YYYYMMDD}.PDF
|
|
SENATE_PDF_RE = re.compile(
|
|
r"SEN_(?P<type>[A-Z]+)_(?P<year>\d{4})_R_(?P<date>\d{8})\.PDF$"
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class SourcePlan:
|
|
source: str # "assembly_daily_file" | "senate_daily_file"
|
|
date: str # requested YYYY-MM-DD
|
|
kind: str # "html" | "pdf" | "events"
|
|
primary_url: str # URL that is actually fetched
|
|
ext: str # cache extension ("html" | "pdf")
|
|
pdf_url: Optional[str] = None
|
|
as_of: Optional[str] = None # date the live page itself represents
|
|
extra_urls: list = field(default_factory=list)
|
|
|
|
|
|
def build_source_plan(name: str, date_iso: str, src_cfg: dict) -> SourcePlan:
|
|
kind = src_cfg.get("kind")
|
|
if kind == "events":
|
|
# The chamber "upcoming events" calendar is a live HTML page (a
|
|
# forward-looking source, distinct from the daily file). Its URL is
|
|
# constant; the date only drives the cache key (the page may list
|
|
# future-dated events, none of which is the --date value itself).
|
|
url = src_cfg.get("url") or src_cfg.get("index_url")
|
|
return SourcePlan(
|
|
source=name,
|
|
date=date_iso,
|
|
kind="events",
|
|
primary_url=url,
|
|
ext="html",
|
|
as_of=date_iso,
|
|
)
|
|
|
|
if kind == "html":
|
|
# Assembly: constant URL, always the current live file.
|
|
return SourcePlan(
|
|
source=name,
|
|
date=date_iso,
|
|
kind="html",
|
|
primary_url=src_cfg["url"],
|
|
ext="html",
|
|
as_of=None, # filled by the Ingestor after reading the body
|
|
)
|
|
|
|
if kind == "pdf":
|
|
# Senate: index is live HTML; the actual file is a date-encoded PDF.
|
|
pdf_base = src_cfg.get("pdf_base_url", "").rstrip("/") + "/"
|
|
filename = _senate_pdf_filename(src_cfg, date_iso)
|
|
pdf_url = pdf_base + filename
|
|
return SourcePlan(
|
|
source=name,
|
|
date=date_iso,
|
|
kind="pdf",
|
|
primary_url=src_cfg["index_url"],
|
|
ext="pdf",
|
|
pdf_url=pdf_url,
|
|
as_of=date_iso, # the PDF filename encodes the date
|
|
)
|
|
|
|
raise ValueError("Unknown source kind '%s' for '%s'" % (kind, name))
|
|
|
|
|
|
def _senate_pdf_filename(cfg: dict, date_iso: str) -> str:
|
|
pattern = cfg.get("pdf_filename_pattern", "SEN_{type}_{year}_R_{date}.PDF")
|
|
ydate = date_iso.replace("-", "")
|
|
year = _session_year(cfg, date_iso)
|
|
ptype = cfg.get("default_pdf_type", "R")
|
|
return pattern.format(type=ptype, year=year, date=ydate)
|
|
|
|
|
|
def _session_year(cfg: dict, date_iso: str) -> str:
|
|
# For the 2025-2026 session, any date in 2026 uses session-start year 2025.
|
|
if not cfg.get("session_year_from_date"):
|
|
return date_iso[:4]
|
|
try:
|
|
y = int(date_iso[:4])
|
|
except (ValueError, TypeError):
|
|
return date_iso[:4]
|
|
if y % 2 == 0:
|
|
return str(y - 1)
|
|
return str(y)
|
|
|
|
|
|
def parse_senate_pdf_filename(name: str):
|
|
# Reverse the encode: filename -> {type, year, date_iso, raw}; None if nmatch.
|
|
m = SENATE_PDF_RE.search(name.split("/")[-1])
|
|
if not m:
|
|
return None
|
|
ydate = m.group("date")
|
|
date_iso = ydate[0:4] + "-" + ydate[4:6] + "-" + ydate[6:8]
|
|
return {
|
|
"type": m.group("type"),
|
|
"year": m.group("year"),
|
|
"date_iso": date_iso,
|
|
"raw": name,
|
|
}
|