# Resolve a chamber source for a given date -> the URL(s) to fetch + cache file # extension. Neither daily-file page is date-addressable by URL, so the date only # drives the Senate PDF filename and the cache key; the Assembly URL is constant. from __future__ import annotations import re from dataclasses import dataclass, field from typing import Optional # Senate daily-file PDF filename pattern: SEN_{type}_{year}_R_{YYYYMMDD}.PDF SENATE_PDF_RE = re.compile( r"SEN_(?P[A-Z]+)_(?P\d{4})_R_(?P\d{8})\.PDF$" ) @dataclass class SourcePlan: source: str # "assembly_daily_file" | "senate_daily_file" date: str # requested YYYY-MM-DD kind: str # "html" | "pdf" | "events" primary_url: str # URL that is actually fetched ext: str # cache extension ("html" | "pdf") pdf_url: Optional[str] = None as_of: Optional[str] = None # date the live page itself represents extra_urls: list = field(default_factory=list) def build_source_plan(name: str, date_iso: str, src_cfg: dict) -> SourcePlan: kind = src_cfg.get("kind") if kind == "events": # The chamber "upcoming events" calendar is a live HTML page (a # forward-looking source, distinct from the daily file). Its URL is # constant; the date only drives the cache key (the page may list # future-dated events, none of which is the --date value itself). url = src_cfg.get("url") or src_cfg.get("index_url") return SourcePlan( source=name, date=date_iso, kind="events", primary_url=url, ext="html", as_of=date_iso, ) if kind == "html": # Assembly: constant URL, always the current live file. return SourcePlan( source=name, date=date_iso, kind="html", primary_url=src_cfg["url"], ext="html", as_of=None, # filled by the Ingestor after reading the body ) if kind == "pdf": # Senate: index is live HTML; the actual file is a date-encoded PDF. pdf_base = src_cfg.get("pdf_base_url", "").rstrip("/") + "/" filename = _senate_pdf_filename(src_cfg, date_iso) pdf_url = pdf_base + filename return SourcePlan( source=name, date=date_iso, kind="pdf", primary_url=src_cfg["index_url"], ext="pdf", pdf_url=pdf_url, as_of=date_iso, # the PDF filename encodes the date ) raise ValueError("Unknown source kind '%s' for '%s'" % (kind, name)) def _senate_pdf_filename(cfg: dict, date_iso: str) -> str: pattern = cfg.get("pdf_filename_pattern", "SEN_{type}_{year}_R_{date}.PDF") ydate = date_iso.replace("-", "") year = _session_year(cfg, date_iso) ptype = cfg.get("default_pdf_type", "R") return pattern.format(type=ptype, year=year, date=ydate) def _session_year(cfg: dict, date_iso: str) -> str: # For the 2025-2026 session, any date in 2026 uses session-start year 2025. if not cfg.get("session_year_from_date"): return date_iso[:4] try: y = int(date_iso[:4]) except (ValueError, TypeError): return date_iso[:4] if y % 2 == 0: return str(y - 1) return str(y) def parse_senate_pdf_filename(name: str): # Reverse the encode: filename -> {type, year, date_iso, raw}; None if nmatch. m = SENATE_PDF_RE.search(name.split("/")[-1]) if not m: return None ydate = m.group("date") date_iso = ydate[0:4] + "-" + ydate[4:6] + "-" + ydate[6:8] return { "type": m.group("type"), "year": m.group("year"), "date_iso": date_iso, "raw": name, }