Initial commit

scraper/schema.py

@@ -0,0 +1,99 @@
+"""Normalised output schema for a Daily File entry.
+
+One :class:`BillAction` per record, matching DAILY_FILE_PARSE_SPEC.md §7.
+Records serialise cleanly to JSON Lines and to a flat SQLite table.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass, fields, asdict
+from typing import Any, Optional, List
+
+
+# Source identifiers (spec §7). Floor / committee bills come from the Assembly
+# daily file; Senate data would arrive from the PDF path or the leginfo alt.
+ASSEMBLY_DAILY_FILE = "ASSEMBLY_DAILY_FILE"
+SENATE_DAILY_FILE_PDF = "SENATE_DAILY_FILE_PDF"
+LEGINFO_BILL_RESULTS = "LEGINFO_BILL_RESULTS"
+
+# House = the publishing chamber; chamber_origin = the bill's origin (spec §8 #7).
+HOUSE = "assembly"  # default house for the Assembly Daily File HTML source
+
+
+@dataclass
+class BillAction:
+    """A single normalised daily-file record.
+
+    Field names mirror the §7 JSON schema. ``id`` is an optional deterministic
+    content hash (sha1 of ``house|bill_number|date|committee``); leave it ``None``
+    to auto-compute on serialise.
+    """
+
+    # --- required identity / context -------------------------------------
+    date: Optional[str] = None            # "2026-08-31" (normalised ISO) when available
+    house: str = HOUSE                    # ASSEMBLY / SENATE (publishing house)
+    source: str = ASSEMBLY_DAILY_FILE     # source tag (see module constants)
+    record_type: str = "bill"             # "bill" | "hearing"
+    # --- bill fields ------------------------------------------------------
+    bill_number: Optional[str] = None     # "AB 28" (4-digit normalised display)
+    bill_id: Optional[str] = None         # canonical "202520260AB28"
+    session: Optional[str] = None         # "2025-2026"
+    chamber_origin: Optional[str] = None  # ASSEMBLY / SENATE (origin of the bill)
+    author: Optional[str] = None
+    short_title: Optional[str] = None     # the bill's subject / short title
+    description: Optional[str] = None     # the action description for the day's item
+    committee: Optional[str] = None       # None for floor items
+    hearing_time: Optional[str] = None    # for hearing rows: "9:30am" / "Upon Call of the Chair"
+    location: Optional[str] = None
+    item_number: Optional[int] = None     # int, un-padded (floor items)
+    supplement: Optional[int] = None      # edition, from "Supplemental N"
+    ics_url: Optional[str] = None
+    agenda_id: Optional[str] = None
+    # --- provenance -------------------------------------------------------
+    id: Optional[str] = None             # auto content hash when None
+    raw_selector: Optional[str] = None   # which table the row came from (debug/provenance)
+
+    def with_id(self, content: Optional[str] = None) -> "BillAction":
+        """Return the record with ``id`` filled in from a content hash.
+
+        The spec's key is ``sha1(house|bill_number|date|committee)``. Missing
+        components are rendered as empty strings so the hash stays deterministic.
+        """
+        if content is None:
+            parts = [
+                self.house or "",
+                self.bill_number or "",
+                self.date or "",
+                self.committee or "",
+            ]
+            content = "|".join(parts)
+        self.id = hashlib.sha1(content.encode("utf-8")).hexdigest()
+        return self
+
+    def to_json(self) -> str:
+        """Serialise to a single-line JSON string (the JSON Lines record)."""
+        import json
+
+        with_id = self if self.id is not None else self.with_id()
+        return json.dumps(
+            {k: v for k, v in asdict(with_id).items() if v is not None},
+            ensure_ascii=False,
+            sort_keys=True,
+        )
+
+    def to_dict(self, *, fill_id: bool = True, include_nulls: bool = False) -> dict:
+        rec = self.with_id() if fill_id and self.id is None else self
+        data = asdict(rec)
+        if not include_nulls:
+            data = {k: v for k, v in data.items() if v is not None}
+        return data
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "BillAction":
+        known = {f.name for f in fields(cls)}
+        return cls(**{k: v for k, v in data.items() if k in known})
+
+    @staticmethod
+    def field_names() -> List[str]:
+        return [f.name for f in fields(BillAction)]