Adds more bill status visibility to the dashboard

scraper/dashboard/api.py

@@ -12,6 +12,7 @@ from typing import Any, Dict, List, Optional, Tuple
 from . import data as _data
 from . import ranking
 from . import resolve as _resolve
+from . import bill_status
 from .data import (load_tag_cloud, list_events,
                    load_legislators, committee_roster_for)
 from .ranking import hearing_sort, full_queue, personal_band
@@ -164,6 +165,14 @@ def _client_bill(ev):
     out = {k: ev.get(k) for k in _data._EVENT_FIELDS}
     out["hearing_sort"] = hearing_sort(ev.get("hearing_time"))
     out["tags"] = ev.get("tags")
+    # API_CONTRACT §5 "upcoming" item 2: label the bill's current state (a
+    # past/terminal floor action shouldn't read as a vague "pending"), and
+    # flag possibly_closed when the source can't disambiguate.
+    st = bill_status.derive_status(ev.get("description"),
+                                    ev.get("bill_number"))
+    out["status"] = st["status"]
+    out["reason"] = st["reason"]
+    out["possibly_closed"] = st["possibly_closed"]
     if "relevance" in ev:
         out["relevance"] = ev["relevance"]
     return out

scraper/dashboard/bill_status.py

@@ -0,0 +1,172 @@
+"""Derive a bill's current legislative *status* + a human-readable *reason*
+from the daily-file action history, so the dashboard can say WHY a bill is in
+the queue instead of leaving a past/terminal bill to read as a vague "pending".
+
+Background
+----------
+The daily file's floor/committee "view details" cell is a *chronological* log of
+every action a bill has seen this session (e.g.::
+
+    Read third time. Passed. Ordered to the Assembly. (Ayes 39. Noes 0.).
+    Aug. 30 - In Assembly. Concurrence in Senate amendments pending.
+
+). The dashboard's queue is keyed off the file's event *date*, not off that
+history, so a bill whose only recorded actions are historical still surfaces as
+an "upcoming item" with no explanation (see API_CONTRACT §5 "upcoming", item 2:
+the event must be "not in a terminal 'acted/closed' state the source marks").
+This module is that gate, made observable: instead of hiding a closed bill it
+labels it, and it emits ``possibly_closed`` when the source can't disambiguate
+(API_CONTRACT §5: *"default to including ... and mark ``possibly_closed: true``"*).
+
+Conventions
+-----------
+* Pure, I/O-free: ``derive_status(description)`` -> a plain dict. This is what
+  lets it be unit-tested with no database, mirroring :mod:`ranking`.
+* The *latest* matching action in the text wins (the log ends with the bill's
+  current state). A small priority table breaks ties so that, on the same span,
+  the more specific / current fact beats a generic one (e.g. "reconsideration
+  pending" beats "passed").
+* Nothing here is hidden or removed: the queue keeps every bill; this only
+  *labels* it. ``status`` is one of ``open`` / ``closed`` / ``unknown``;
+  ``reason`` is the short human string the UI shows; ``possibly_closed`` mirrors
+  the contract flag.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Dict, List, Optional, Tuple
+
+# Each rule: (compiled regex, human reason, status, possibly_closed, priority).
+# Higher priority wins when two rules match the *same* span of text, so the most
+# specific / current fact surfaces. ``open`` = an action still ahead; ``closed``
+# = a terminal fact the source marks (inactive file / refused / engrossed / ...);
+# its ``possibly_closed`` is True because the source may still have a later,
+# unstated next step, so the contract's default-to-including still applies.
+_RULES: List[Tuple[str, str, str, bool, int]] = [
+    # latest / current-state facts first (highest priority)
+    ("motion to reconsider continued",
+     "Reconsideration continuing", "open", False, 100),
+    ("motion to reconsider",
+     "Reconsideration pending", "open", False, 99),
+    (r"concurr[a-z]*\s+in\s+senate\s+amendments\s+pending",
+     "Awaiting concurrence in Senate amendments", "open", False, 96),
+    ("refused to concur",
+     "Refused to concur in Senate amendments", "open", False, 95),
+    ("unfinished business",
+     "On the unfinished business file", "open", False, 94),
+    ("suspense file",
+     "On suspense file", "open", False, 90),
+    (r"do pass\b",
+     "Do pass — out of committee", "open", False, 88),
+    ("ordered to third reading",
+     "Ordered to third reading", "open", False, 84),
+    ("ordered to second reading",
+     "Ordered to second reading", "open", False, 82),
+    (r"read [a-z]* time\b",
+     "Read — in progress", "open", False, 80),
+    (r"set for [a-z ]*hearing|scheduled for hearing|may be heard in committee",
+     "Hearing scheduled", "open", False, 76),
+    ("referred to com",
+     "In committee", "open", False, 74),
+    ("held at desk",
+     "Held at desk", "open", False, 72),
+    # terminal facts the source marks
+    ("signed by the governor|governor.s signature",
+     "Chambered to the Governor", "closed", True, 71),
+    ("engrossed",
+     "Engrossed", "closed", True, 70),
+    (r"inactive file",
+     "Inactive file (no longer active)", "closed", True, 69),
+    (r"on motion of",
+     "Inactive file (no longer active)", "closed", True, 68),
+    ("withdrawn",
+     "Withdrawn", "closed", True, 67),
+    # generic fallbacks (lowest priority)
+    ("passed",
+     "Passed in one chamber", "open", False, 40),
+    ("pending",
+     "Action pending", "open", False, 30),
+]
+
+_RE = [(re.compile(p, re.I), r, st, cl, pr)
+      for p, r, st, cl, pr in _RULES]
+
+# "View Details" is UI noise, not an action.
+_NOISE = re.compile(r"view details\s*$", re.I)
+
+# A *genuinely* content-free history is "Vote required: N YYYY Mon. DD -" (a
+# leading dated stub with nothing after the dash). Match the trailing stub only
+# if the text before it is just the "Vote required:" header / whitespace / a
+# year — so a real action like "To inactive file on motion of ..." sitting
+# before a trailing date is NOT mistaken for an empty line.
+_EMPTY_ACTION = re.compile(
+      r"^(vote required:\s*\d*|\d{4}\s*[A-Za-z]{3,}\.\s*\d{1,2}|\d{1,2})"
+      r"[\s-]*$",
+      re.I)
+
+
+def _clean(text: Optional[str]) -> str:
+    s = _NOISE.sub("", (text or "").strip())
+    return s.strip()
+
+
+def _unknown(bill_number: Optional[str]) -> Dict[str, Any]:
+     # API_CONTRACT §5: when the source can't confirm a state, *include* the
+     # bill but do not assert open/closed -> status unknown, flag None.
+    reason = "On file (no action logged)" if bill_number else "On file"
+    return {"status": "unknown", "reason": reason, "possibly_closed": None}
+
+
+def derive_status(description: Optional[str],
+                  bill_number: Optional[str] = None) -> Dict[str, Any]:
+    """Classify a bill and return ``{status, reason, possibly_closed}``.
+
+    Parameters
+    ----------
+    description: the daily-file floor/committee "view details" history (the
+        chronological action log). May be ``None`` / empty.
+    bill_number: optional (e.g. "AB 1099"); only used for the unknown fallback
+        label, never for classification.
+
+    Returns
+    -------
+    ``{"status": "open"|"closed"|"unknown",
+    "reason": <human string>, "possibly_closed": <bool|None>}``.
+
+    The *latest* matching action in the text wins (the log ends with the
+    bill's current state); on a tie the higher-priority rule wins. When no
+    action can be read the bill defaults to ``unknown`` and
+    ``possibly_closed`` is ``None`` (API_CONTRACT §5: include it, flag it)
+    so the UI degrades to a neutral "On file" rather than asserting a state
+    the source doesn't give.
+    """
+    text = _clean(description)
+    if not text:
+        return _unknown(bill_number)
+
+    # A genuinely content-free history ("Vote required: N 2026 May 27 -")
+    # has no action to read: default to including and flag (API_CONTRACT §5),
+    # never assert a state the source doesn't give.
+    if _EMPTY_ACTION.match(text):
+        return _unknown(bill_number)
+
+    best: Optional[Tuple[int, int, str, str, Optional[bool]]] = None
+    for rx, reason, status, closed, pr in _RE:
+        for m in rx.finditer(text):
+            key = (m.end(), pr)
+            if best is None or key > best[0]:
+                best = (key, pr, reason, status, closed)
+
+    if best is None:
+        # Content but no recognised action: default to including + flag.
+        return _unknown(bill_number)
+
+    _, _, reason, status, closed = best
+    return {"status": status, "reason": reason,
+             "possibly_closed": True if closed else False}
+
+
+def is_closed(result: Dict[str, Any]) -> bool:
+    """True when :func:`derive_status` marked a terminal/acted state."""
+    return result.get("status") == "closed"

scraper/dashboard/static/app.js

@@ -60,7 +60,30 @@
         if (t) return "tag-match";
         if (l) return "leg-match";
         return "";
-    }
+     }
+
+    /* The bill's current legislative state — why it is in the queue at all
+       (API_CONTRACT §5 "upcoming" item 2: a closed/past bill shouldn't read as
+       a vague "pending"). Always shown; a closed bill is dimmed; possibly_closed
+       is flagged when the source can't confirm the state. */
+    function statusReasonHTML(ev) {
+        var st = ev.status;
+        var reason = ev.reason;
+        if (!st && !reason) return "";
+        var closed = st === "closed";
+        var label = closed ? "Closed" : "On file";
+        var cls = "chip status"
+             + (closed ? " closed" : " open")
+             + (ev.possibly_closed ? " pclosed" : "");
+        var why = reason ? esc(reason) : "On file";
+        var tail = ev.possibly_closed ? " (possibly closed)" : "";
+        return "<div class='row-reason'"
+             + (closed ? " data-closed='1'" : "") + ">"
+             + "<span class='" + cls + "'>" + esc(label) + "</span> "
+             + "<span class='row-reason-why'>" + why
+             + (tail ? " " + esc(tail) : "") + "</span>"
+             + "</div>";
+     }
 
     function rowHTML(ev, opts) {
         opts = opts || {};
@@ -78,9 +101,11 @@
                     (on ? 'on"' : '"') + ">" + esc(tg) + "</span>";
             }).join("");
         var marker = rel
-            ? '<span class="chip marker" title="Floated: relevant to you">'
-              + "Relevant to you</span>"
-            : "";
+             ? '<span class="chip marker" title="Floated: relevant to you">'
+                + "Relevant to you</span>"
+             : "";
+         // Legislative status / reason — always on, above the relevance toggle.
+        var statusBlock = statusReasonHTML(ev);
         var reasons = "";
         if (rel) {
             var parts = (ev.relevance.matched || []).map(function (x) {
@@ -98,11 +123,12 @@
              esc(ev.bill_id) + '">'
              + '<span class="num">' + chip(ev.chamber_origin) +
                    '<br>' + esc(ev.bill_number) + "</span>"
-             + '<div class="main"><span class="title">'
-                   + esc(ev.short_title || ev.bill_number) + "</span>"
-                   + "<div class='meta'>" + esc(meta) + "</div>"
-                   + "<a href='" + url + "' target='_blank'"
-                   + " rel='noopener'>View →</a> " + marker
+              + '<div class="main"><span class="title">'
+                    + esc(ev.short_title || ev.bill_number) + "</span>"
+                    + "<div class='meta'>" + esc(meta) + "</div>"
+                    + statusBlock
+                    + "<a href='" + url + "' target='_blank'"
+                    + " rel='noopener'>View →</a> " + marker
                    + tags
                    + reasons
                    + '</div>'

scraper/dashboard/static/style.css

@@ -169,6 +169,21 @@ button.chip-toggle { min-width: 120px; }
 .chip.sen { background: #f5eef9; color: var(--sen); border-color:#e2d2ea; }
 .chip.marker { background: var(--accent); color: #fff; border-color:
    var(--accent); }
+
+/* ---- legislative status / "why is this here" reason (always on) ---- */
+.row-reason {
+   margin: 4px 0; font-size: 12px; color: var(--muted);
+   display: flex; align-items: baseline; gap: 6px; }
+.chip.status { border-color: var(--border); }
+.chip.status.open { background: #eef3fb; color: var(--asm);
+   border-color: #cdd9ee; }
+.chip.status.closed { background: #f0f0f0; color: var(--warn);
+   border-color: #d8c9a8; }
+.chip.status.pclosed { font-style: italic; }
+.row-reason-why { color: var(--muted); }
+     /* a terminal/closed bill is dimmed so it doesn't read as an open upcoming */
+.row-reason[data-closed='1'] { opacity: .85; }
+.row-reason[data-closed='1'] .row-reason-why { color: var(--muted); }
 .tag { display: inline-block; font-size: 11px; color: var(--muted);
    border: 1px solid var(--border); border-radius: 12px;
    padding: 0 6px; margin: 0 4px 0 0; }

tests/test_dashboard_api.py

@@ -141,11 +141,29 @@ def test_bills_personal_returns_relevance_and_floats(services):
 
 def test_bills_personal_requires_auth(services):
     status, body = api.run(_req("GET", "/api/v1/bills", services,
-                                 query={"personal": "1"}))
+                                  query={"personal": "1"}))
     assert status == 401, body
     assert body["error"]["code"] == "not_authenticated"
 
 
+# ---- §5 "upcoming" item 2: each item carries status + reason -------------
+
+def test_bills_items_carry_status_reason_and_possibly_closed(services):
+     # Every item must expose the "why is this here?" triad so a closed/past
+     # bill doesn't read as a vague "pending" (API_CONTRACT §5 item 2).
+    status, body = api.run(_req("GET", "/api/v1/bills", services,
+                                 query={"upcoming": "1"}))
+    assert status == 200, body
+    items = body["items"]
+    assert items, "expected at least one item"
+    for it in items:
+        for key in ("status", "reason", "possibly_closed"):
+            assert key in it, key
+        assert it["status"] in ("open", "closed", "unknown")
+        assert isinstance(it["reason"], str) and it["reason"]
+        assert it["possibly_closed"] in (True, False, None)
+
+
 # ---- six empty states -> 200 items:[] (never 4xx/5xx) ---------------------
 
 def test_empty_no_next_hearing(services):

tests/test_dashboard_bill_status.py

@@ -0,0 +1,110 @@
+"""Tests for ``scraper.dashboard.bill_status`` — the "why is this bill here?"
+classifier (API_CONTRACT §5 "upcoming" item 2: a bill should not be a vague
+"pending" when its floor history already shows a terminal / current state).
+
+Pure, no DB: ``derive_status(description)`` -> ``{status, reason,
+possibly_closed}``. The *latest* matching action in the chronological log wins;
+the contract's "default to including, flag possibly_closed" applies when the
+source can't disambiguate.
+"""
+from __future__ import annotations
+
+from scraper.dashboard import bill_status
+from scraper.dashboard.bill_status import derive_status, is_closed
+
+
+# ---------------------------------------------------------------------------
+# the motivating case: a passed bill sitting in "unfinished business"
+# ---------------------------------------------------------------------------
+
+def test_awaiting_concurrence_in_senate_amendments_is_open():
+      # AB1099: passed the Assembly, now awaiting Senate-amendment concurrence.
+    r = derive_status(
+          "Vote required: 41 2026 Aug. 30 - Read third time. Passed. "
+          "Ordered to the Assembly. (Ayes 39. Noes 0.). Aug. 30 - In Assembly. "
+          "Concurrence in Senate amendments pending. View Details",
+          "AB 1099")
+    assert r["status"] == "open"
+    assert "concurrence" in r["reason"].lower()
+    assert r["possibly_closed"] is False
+    assert not is_closed(r)
+
+
+def test_passed_is_not_terminal_but_closed_terms_are():
+      # "Passed" (one chamber) is an in-progress step -> open, not dropped.
+    passed = derive_status("Read third time. Passed. Ordered to the Assembly.",
+                           "AB 1")
+    assert passed["status"] == "open"
+
+      # "To inactive file" is a terminal/acted state -> closed, possibly_closed.
+    inact = derive_status("2026 Aug. 25 - To inactive file on motion of "
+                           "Assembly Member Jackson. View Details", "AB 2")
+    assert inact["status"] == "closed"
+    assert inact["possibly_closed"] is True
+    assert is_closed(inact)
+    assert "inactive" in inact["reason"].lower()
+
+
+# ---------------------------------------------------------------------------
+# latest-action-wins + priority
+# ---------------------------------------------------------------------------
+
+def test_latest_action_in_log_wins():
+      # A bill that passed long ago but now has "Motion to reconsider
+      # continued" surfaces the *current* reconsideration state, not the old pass.
+    r = derive_status(
+          "2025 Sep. 13 - Read third time. Passed. Ordered to the Assembly. "
+          "Sep. 13 - In Assembly. Assembly refused to concur in Senate "
+          "amendments. (Ayes 34. Noes 27.) Motion to reconsider on the next "
+          "legislative day made. ... 2026 Aug. 30 - Motion to reconsider "
+          "continued. View Details",
+          "AB 1231")
+    assert "reconsider" in r["reason"].lower()
+    assert r["status"] == "open"
+
+
+def test_priority_breates_ties_on_same_span():
+      # "Motion to reconsider" must win over a bare "passed" even though both
+      # match; the specific/current fact dominates.
+    r = derive_status("Passed. Motion to reconsider made.", "AB 1")
+    assert "reconsider" in r["reason"].lower()
+
+
+# ---------------------------------------------------------------------------
+# unknown / content-free -> default to including + flag (API_CONTRACT §5)
+# ---------------------------------------------------------------------------
+
+def test_content_free_defaults_to_unknown_and_flags():
+      # A bare dated line with no action text -> "On file", never a false pass.
+    r = derive_status("Vote required: 41 2026 May 27 - View Details", "AB 3")
+    assert r["status"] == "unknown"
+    assert r["possibly_closed"] is None     # can't confirm -> not asserted open
+    assert "on file" in r["reason"].lower()
+
+
+def test_empty_description_is_unknown_not_crash():
+    for text in (None, "", "   "):
+        r = derive_status(text, "AB 4")
+        assert r["status"] == "unknown"
+        assert r["possibly_closed"] is None
+
+
+# ---------------------------------------------------------------------------
+# a few more real-world terminal / open signals
+# ---------------------------------------------------------------------------
+
+def test_engrossed_and_withdrawn_are_closed():
+    assert is_closed(derive_status("Engrossed. View Details", "AB 5"))
+    assert is_closed(derive_status("Withdrawn from committee. View Details",
+                                   "AB 6"))
+
+
+def test_in_committee_and_scheduled_hearing_are_open():
+    assert derive_status("Referred to Com. on Health. View Details", "AB 7")["status"] == "open"
+    assert derive_status("June 17 - Set for first hearing. View Details",
+                         "AB 8")["status"] == "open"
+
+
+def test_return_shape_is_stable():
+    r = derive_status("Pending. View Details", "AB 9")
+    assert set(r.keys()) == {"status", "reason", "possibly_closed"}