Adds more bill status visibility to the dashboard

scraper/dashboard/bill_status.py

@@ -0,0 +1,172 @@
+"""Derive a bill's current legislative *status* + a human-readable *reason*
+from the daily-file action history, so the dashboard can say WHY a bill is in
+the queue instead of leaving a past/terminal bill to read as a vague "pending".
+
+Background
+----------
+The daily file's floor/committee "view details" cell is a *chronological* log of
+every action a bill has seen this session (e.g.::
+
+    Read third time. Passed. Ordered to the Assembly. (Ayes 39. Noes 0.).
+    Aug. 30 - In Assembly. Concurrence in Senate amendments pending.
+
+). The dashboard's queue is keyed off the file's event *date*, not off that
+history, so a bill whose only recorded actions are historical still surfaces as
+an "upcoming item" with no explanation (see API_CONTRACT §5 "upcoming", item 2:
+the event must be "not in a terminal 'acted/closed' state the source marks").
+This module is that gate, made observable: instead of hiding a closed bill it
+labels it, and it emits ``possibly_closed`` when the source can't disambiguate
+(API_CONTRACT §5: *"default to including ... and mark ``possibly_closed: true``"*).
+
+Conventions
+-----------
+* Pure, I/O-free: ``derive_status(description)`` -> a plain dict. This is what
+  lets it be unit-tested with no database, mirroring :mod:`ranking`.
+* The *latest* matching action in the text wins (the log ends with the bill's
+  current state). A small priority table breaks ties so that, on the same span,
+  the more specific / current fact beats a generic one (e.g. "reconsideration
+  pending" beats "passed").
+* Nothing here is hidden or removed: the queue keeps every bill; this only
+  *labels* it. ``status`` is one of ``open`` / ``closed`` / ``unknown``;
+  ``reason`` is the short human string the UI shows; ``possibly_closed`` mirrors
+  the contract flag.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Dict, List, Optional, Tuple
+
+# Each rule: (compiled regex, human reason, status, possibly_closed, priority).
+# Higher priority wins when two rules match the *same* span of text, so the most
+# specific / current fact surfaces. ``open`` = an action still ahead; ``closed``
+# = a terminal fact the source marks (inactive file / refused / engrossed / ...);
+# its ``possibly_closed`` is True because the source may still have a later,
+# unstated next step, so the contract's default-to-including still applies.
+_RULES: List[Tuple[str, str, str, bool, int]] = [
+    # latest / current-state facts first (highest priority)
+    ("motion to reconsider continued",
+     "Reconsideration continuing", "open", False, 100),
+    ("motion to reconsider",
+     "Reconsideration pending", "open", False, 99),
+    (r"concurr[a-z]*\s+in\s+senate\s+amendments\s+pending",
+     "Awaiting concurrence in Senate amendments", "open", False, 96),
+    ("refused to concur",
+     "Refused to concur in Senate amendments", "open", False, 95),
+    ("unfinished business",
+     "On the unfinished business file", "open", False, 94),
+    ("suspense file",
+     "On suspense file", "open", False, 90),
+    (r"do pass\b",
+     "Do pass — out of committee", "open", False, 88),
+    ("ordered to third reading",
+     "Ordered to third reading", "open", False, 84),
+    ("ordered to second reading",
+     "Ordered to second reading", "open", False, 82),
+    (r"read [a-z]* time\b",
+     "Read — in progress", "open", False, 80),
+    (r"set for [a-z ]*hearing|scheduled for hearing|may be heard in committee",
+     "Hearing scheduled", "open", False, 76),
+    ("referred to com",
+     "In committee", "open", False, 74),
+    ("held at desk",
+     "Held at desk", "open", False, 72),
+    # terminal facts the source marks
+    ("signed by the governor|governor.s signature",
+     "Chambered to the Governor", "closed", True, 71),
+    ("engrossed",
+     "Engrossed", "closed", True, 70),
+    (r"inactive file",
+     "Inactive file (no longer active)", "closed", True, 69),
+    (r"on motion of",
+     "Inactive file (no longer active)", "closed", True, 68),
+    ("withdrawn",
+     "Withdrawn", "closed", True, 67),
+    # generic fallbacks (lowest priority)
+    ("passed",
+     "Passed in one chamber", "open", False, 40),
+    ("pending",
+     "Action pending", "open", False, 30),
+]
+
+_RE = [(re.compile(p, re.I), r, st, cl, pr)
+      for p, r, st, cl, pr in _RULES]
+
+# "View Details" is UI noise, not an action.
+_NOISE = re.compile(r"view details\s*$", re.I)
+
+# A *genuinely* content-free history is "Vote required: N YYYY Mon. DD -" (a
+# leading dated stub with nothing after the dash). Match the trailing stub only
+# if the text before it is just the "Vote required:" header / whitespace / a
+# year — so a real action like "To inactive file on motion of ..." sitting
+# before a trailing date is NOT mistaken for an empty line.
+_EMPTY_ACTION = re.compile(
+      r"^(vote required:\s*\d*|\d{4}\s*[A-Za-z]{3,}\.\s*\d{1,2}|\d{1,2})"
+      r"[\s-]*$",
+      re.I)
+
+
+def _clean(text: Optional[str]) -> str:
+    s = _NOISE.sub("", (text or "").strip())
+    return s.strip()
+
+
+def _unknown(bill_number: Optional[str]) -> Dict[str, Any]:
+     # API_CONTRACT §5: when the source can't confirm a state, *include* the
+     # bill but do not assert open/closed -> status unknown, flag None.
+    reason = "On file (no action logged)" if bill_number else "On file"
+    return {"status": "unknown", "reason": reason, "possibly_closed": None}
+
+
+def derive_status(description: Optional[str],
+                  bill_number: Optional[str] = None) -> Dict[str, Any]:
+    """Classify a bill and return ``{status, reason, possibly_closed}``.
+
+    Parameters
+    ----------
+    description: the daily-file floor/committee "view details" history (the
+        chronological action log). May be ``None`` / empty.
+    bill_number: optional (e.g. "AB 1099"); only used for the unknown fallback
+        label, never for classification.
+
+    Returns
+    -------
+    ``{"status": "open"|"closed"|"unknown",
+    "reason": <human string>, "possibly_closed": <bool|None>}``.
+
+    The *latest* matching action in the text wins (the log ends with the
+    bill's current state); on a tie the higher-priority rule wins. When no
+    action can be read the bill defaults to ``unknown`` and
+    ``possibly_closed`` is ``None`` (API_CONTRACT §5: include it, flag it)
+    so the UI degrades to a neutral "On file" rather than asserting a state
+    the source doesn't give.
+    """
+    text = _clean(description)
+    if not text:
+        return _unknown(bill_number)
+
+    # A genuinely content-free history ("Vote required: N 2026 May 27 -")
+    # has no action to read: default to including and flag (API_CONTRACT §5),
+    # never assert a state the source doesn't give.
+    if _EMPTY_ACTION.match(text):
+        return _unknown(bill_number)
+
+    best: Optional[Tuple[int, int, str, str, Optional[bool]]] = None
+    for rx, reason, status, closed, pr in _RE:
+        for m in rx.finditer(text):
+            key = (m.end(), pr)
+            if best is None or key > best[0]:
+                best = (key, pr, reason, status, closed)
+
+    if best is None:
+        # Content but no recognised action: default to including + flag.
+        return _unknown(bill_number)
+
+    _, _, reason, status, closed = best
+    return {"status": status, "reason": reason,
+             "possibly_closed": True if closed else False}
+
+
+def is_closed(result: Dict[str, Any]) -> bool:
+    """True when :func:`derive_status` marked a terminal/acted state."""
+    return result.get("status") == "closed"