Initial commit

scraper/sources.py

@@ -0,0 +1,93 @@
+# Resolve a chamber source for a given date -> the URL(s) to fetch + cache file
+# extension. Neither daily-file page is date-addressable by URL, so the date only
+# drives the Senate PDF filename and the cache key; the Assembly URL is constant.
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import Optional
+
+# Senate daily-file PDF filename pattern: SEN_{type}_{year}_R_{YYYYMMDD}.PDF
+SENATE_PDF_RE = re.compile(
+     r"SEN_(?P<type>[A-Z]+)_(?P<year>\d{4})_R_(?P<date>\d{8})\.PDF$"
+)
+
+
+@dataclass
+class SourcePlan:
+    source: str                  # "assembly_daily_file" | "senate_daily_file"
+    date: str                    # requested YYYY-MM-DD
+    kind: str                    # "html" | "pdf"
+    primary_url: str             # URL that is actually fetched
+    ext: str                     # cache extension ("html" | "pdf")
+    pdf_url: Optional[str] = None
+    as_of: Optional[str] = None      # date the live page itself represents
+    extra_urls: list = field(default_factory=list)
+
+
+def build_source_plan(name: str, date_iso: str, src_cfg: dict) -> SourcePlan:
+    kind = src_cfg.get("kind")
+    if kind == "html":
+        # Assembly: constant URL, always the current live file.
+        return SourcePlan(
+            source=name,
+            date=date_iso,
+            kind="html",
+            primary_url=src_cfg["url"],
+            ext="html",
+            as_of=None,   # filled by the Ingestor after reading the body
+        )
+
+    if kind == "pdf":
+        # Senate: index is live HTML; the actual file is a date-encoded PDF.
+        pdf_base = src_cfg.get("pdf_base_url", "").rstrip("/") + "/"
+        filename = _senate_pdf_filename(src_cfg, date_iso)
+        pdf_url = pdf_base + filename
+        return SourcePlan(
+            source=name,
+            date=date_iso,
+            kind="pdf",
+            primary_url=src_cfg["index_url"],
+            ext="pdf",
+            pdf_url=pdf_url,
+            as_of=date_iso,   # the PDF filename encodes the date
+        )
+
+    raise ValueError("Unknown source kind '%s' for '%s'" % (kind, name))
+
+
+def _senate_pdf_filename(cfg: dict, date_iso: str) -> str:
+    pattern = cfg.get("pdf_filename_pattern", "SEN_{type}_{year}_R_{date}.PDF")
+    ydate = date_iso.replace("-", "")
+    year = _session_year(cfg, date_iso)
+    ptype = cfg.get("default_pdf_type", "R")
+    return pattern.format(type=ptype, year=year, date=ydate)
+
+
+def _session_year(cfg: dict, date_iso: str) -> str:
+    # For the 2025-2026 session, any date in 2026 uses session-start year 2025.
+    if not cfg.get("session_year_from_date"):
+        return date_iso[:4]
+    try:
+        y = int(date_iso[:4])
+    except (ValueError, TypeError):
+        return date_iso[:4]
+    if y % 2 == 0:
+        return str(y - 1)
+    return str(y)
+
+
+def parse_senate_pdf_filename(name: str):
+     # Reverse the encode: filename -> {type, year, date_iso, raw}; None if nmatch.
+    m = SENATE_PDF_RE.search(name.split("/")[-1])
+    if not m:
+        return None
+    ydate = m.group("date")
+    date_iso = ydate[0:4] + "-" + ydate[4:6] + "-" + ydate[6:8]
+    return {
+          "type": m.group("type"),
+          "year": m.group("year"),
+          "date_iso": date_iso,
+          "raw": name,
+    }