diff --git a/scraper/dashboard/api.py b/scraper/dashboard/api.py index 5a8aa6f..748116d 100644 --- a/scraper/dashboard/api.py +++ b/scraper/dashboard/api.py @@ -12,6 +12,7 @@ from typing import Any, Dict, List, Optional, Tuple from . import data as _data from . import ranking from . import resolve as _resolve +from . import bill_status from .data import (load_tag_cloud, list_events, load_legislators, committee_roster_for) from .ranking import hearing_sort, full_queue, personal_band @@ -164,6 +165,14 @@ def _client_bill(ev): out = {k: ev.get(k) for k in _data._EVENT_FIELDS} out["hearing_sort"] = hearing_sort(ev.get("hearing_time")) out["tags"] = ev.get("tags") + # API_CONTRACT §5 "upcoming" item 2: label the bill's current state (a + # past/terminal floor action shouldn't read as a vague "pending"), and + # flag possibly_closed when the source can't disambiguate. + st = bill_status.derive_status(ev.get("description"), + ev.get("bill_number")) + out["status"] = st["status"] + out["reason"] = st["reason"] + out["possibly_closed"] = st["possibly_closed"] if "relevance" in ev: out["relevance"] = ev["relevance"] return out diff --git a/scraper/dashboard/bill_status.py b/scraper/dashboard/bill_status.py new file mode 100644 index 0000000..855da4d --- /dev/null +++ b/scraper/dashboard/bill_status.py @@ -0,0 +1,172 @@ +"""Derive a bill's current legislative *status* + a human-readable *reason* +from the daily-file action history, so the dashboard can say WHY a bill is in +the queue instead of leaving a past/terminal bill to read as a vague "pending". + +Background +---------- +The daily file's floor/committee "view details" cell is a *chronological* log of +every action a bill has seen this session (e.g.:: + + Read third time. Passed. Ordered to the Assembly. (Ayes 39. Noes 0.). + Aug. 30 - In Assembly. Concurrence in Senate amendments pending. + +). The dashboard's queue is keyed off the file's event *date*, not off that +history, so a bill whose only recorded actions are historical still surfaces as +an "upcoming item" with no explanation (see API_CONTRACT §5 "upcoming", item 2: +the event must be "not in a terminal 'acted/closed' state the source marks"). +This module is that gate, made observable: instead of hiding a closed bill it +labels it, and it emits ``possibly_closed`` when the source can't disambiguate +(API_CONTRACT §5: *"default to including ... and mark ``possibly_closed: true``"*). + +Conventions +----------- +* Pure, I/O-free: ``derive_status(description)`` -> a plain dict. This is what + lets it be unit-tested with no database, mirroring :mod:`ranking`. +* The *latest* matching action in the text wins (the log ends with the bill's + current state). A small priority table breaks ties so that, on the same span, + the more specific / current fact beats a generic one (e.g. "reconsideration + pending" beats "passed"). +* Nothing here is hidden or removed: the queue keeps every bill; this only + *labels* it. ``status`` is one of ``open`` / ``closed`` / ``unknown``; + ``reason`` is the short human string the UI shows; ``possibly_closed`` mirrors + the contract flag. +""" + +from __future__ import annotations + +import re +from typing import Any, Dict, List, Optional, Tuple + +# Each rule: (compiled regex, human reason, status, possibly_closed, priority). +# Higher priority wins when two rules match the *same* span of text, so the most +# specific / current fact surfaces. ``open`` = an action still ahead; ``closed`` +# = a terminal fact the source marks (inactive file / refused / engrossed / ...); +# its ``possibly_closed`` is True because the source may still have a later, +# unstated next step, so the contract's default-to-including still applies. +_RULES: List[Tuple[str, str, str, bool, int]] = [ + # latest / current-state facts first (highest priority) + ("motion to reconsider continued", + "Reconsideration continuing", "open", False, 100), + ("motion to reconsider", + "Reconsideration pending", "open", False, 99), + (r"concurr[a-z]*\s+in\s+senate\s+amendments\s+pending", + "Awaiting concurrence in Senate amendments", "open", False, 96), + ("refused to concur", + "Refused to concur in Senate amendments", "open", False, 95), + ("unfinished business", + "On the unfinished business file", "open", False, 94), + ("suspense file", + "On suspense file", "open", False, 90), + (r"do pass\b", + "Do pass — out of committee", "open", False, 88), + ("ordered to third reading", + "Ordered to third reading", "open", False, 84), + ("ordered to second reading", + "Ordered to second reading", "open", False, 82), + (r"read [a-z]* time\b", + "Read — in progress", "open", False, 80), + (r"set for [a-z ]*hearing|scheduled for hearing|may be heard in committee", + "Hearing scheduled", "open", False, 76), + ("referred to com", + "In committee", "open", False, 74), + ("held at desk", + "Held at desk", "open", False, 72), + # terminal facts the source marks + ("signed by the governor|governor.s signature", + "Chambered to the Governor", "closed", True, 71), + ("engrossed", + "Engrossed", "closed", True, 70), + (r"inactive file", + "Inactive file (no longer active)", "closed", True, 69), + (r"on motion of", + "Inactive file (no longer active)", "closed", True, 68), + ("withdrawn", + "Withdrawn", "closed", True, 67), + # generic fallbacks (lowest priority) + ("passed", + "Passed in one chamber", "open", False, 40), + ("pending", + "Action pending", "open", False, 30), +] + +_RE = [(re.compile(p, re.I), r, st, cl, pr) + for p, r, st, cl, pr in _RULES] + +# "View Details" is UI noise, not an action. +_NOISE = re.compile(r"view details\s*$", re.I) + +# A *genuinely* content-free history is "Vote required: N YYYY Mon. DD -" (a +# leading dated stub with nothing after the dash). Match the trailing stub only +# if the text before it is just the "Vote required:" header / whitespace / a +# year — so a real action like "To inactive file on motion of ..." sitting +# before a trailing date is NOT mistaken for an empty line. +_EMPTY_ACTION = re.compile( + r"^(vote required:\s*\d*|\d{4}\s*[A-Za-z]{3,}\.\s*\d{1,2}|\d{1,2})" + r"[\s-]*$", + re.I) + + +def _clean(text: Optional[str]) -> str: + s = _NOISE.sub("", (text or "").strip()) + return s.strip() + + +def _unknown(bill_number: Optional[str]) -> Dict[str, Any]: + # API_CONTRACT §5: when the source can't confirm a state, *include* the + # bill but do not assert open/closed -> status unknown, flag None. + reason = "On file (no action logged)" if bill_number else "On file" + return {"status": "unknown", "reason": reason, "possibly_closed": None} + + +def derive_status(description: Optional[str], + bill_number: Optional[str] = None) -> Dict[str, Any]: + """Classify a bill and return ``{status, reason, possibly_closed}``. + + Parameters + ---------- + description: the daily-file floor/committee "view details" history (the + chronological action log). May be ``None`` / empty. + bill_number: optional (e.g. "AB 1099"); only used for the unknown fallback + label, never for classification. + + Returns + ------- + ``{"status": "open"|"closed"|"unknown", + "reason": , "possibly_closed": }``. + + The *latest* matching action in the text wins (the log ends with the + bill's current state); on a tie the higher-priority rule wins. When no + action can be read the bill defaults to ``unknown`` and + ``possibly_closed`` is ``None`` (API_CONTRACT §5: include it, flag it) + so the UI degrades to a neutral "On file" rather than asserting a state + the source doesn't give. + """ + text = _clean(description) + if not text: + return _unknown(bill_number) + + # A genuinely content-free history ("Vote required: N 2026 May 27 -") + # has no action to read: default to including and flag (API_CONTRACT §5), + # never assert a state the source doesn't give. + if _EMPTY_ACTION.match(text): + return _unknown(bill_number) + + best: Optional[Tuple[int, int, str, str, Optional[bool]]] = None + for rx, reason, status, closed, pr in _RE: + for m in rx.finditer(text): + key = (m.end(), pr) + if best is None or key > best[0]: + best = (key, pr, reason, status, closed) + + if best is None: + # Content but no recognised action: default to including + flag. + return _unknown(bill_number) + + _, _, reason, status, closed = best + return {"status": status, "reason": reason, + "possibly_closed": True if closed else False} + + +def is_closed(result: Dict[str, Any]) -> bool: + """True when :func:`derive_status` marked a terminal/acted state.""" + return result.get("status") == "closed" diff --git a/scraper/dashboard/static/app.js b/scraper/dashboard/static/app.js index 3dbec33..3ab5192 100644 --- a/scraper/dashboard/static/app.js +++ b/scraper/dashboard/static/app.js @@ -60,7 +60,30 @@ if (t) return "tag-match"; if (l) return "leg-match"; return ""; - } + } + + /* The bill's current legislative state — why it is in the queue at all + (API_CONTRACT §5 "upcoming" item 2: a closed/past bill shouldn't read as + a vague "pending"). Always shown; a closed bill is dimmed; possibly_closed + is flagged when the source can't confirm the state. */ + function statusReasonHTML(ev) { + var st = ev.status; + var reason = ev.reason; + if (!st && !reason) return ""; + var closed = st === "closed"; + var label = closed ? "Closed" : "On file"; + var cls = "chip status" + + (closed ? " closed" : " open") + + (ev.possibly_closed ? " pclosed" : ""); + var why = reason ? esc(reason) : "On file"; + var tail = ev.possibly_closed ? " (possibly closed)" : ""; + return "
" + + "" + esc(label) + " " + + "" + why + + (tail ? " " + esc(tail) : "") + "" + + "
"; + } function rowHTML(ev, opts) { opts = opts || {}; @@ -78,9 +101,11 @@ (on ? 'on"' : '"') + ">" + esc(tg) + ""; }).join(""); var marker = rel - ? '' - + "Relevant to you" - : ""; + ? '' + + "Relevant to you" + : ""; + // Legislative status / reason — always on, above the relevance toggle. + var statusBlock = statusReasonHTML(ev); var reasons = ""; if (rel) { var parts = (ev.relevance.matched || []).map(function (x) { @@ -98,11 +123,12 @@ esc(ev.bill_id) + '">' + '' + chip(ev.chamber_origin) + '
' + esc(ev.bill_number) + "" - + '
' - + esc(ev.short_title || ev.bill_number) + "" - + "
" + esc(meta) + "
" - + "View → " + marker + + '
' + + esc(ev.short_title || ev.bill_number) + "" + + "
" + esc(meta) + "
" + + statusBlock + + "View → " + marker + tags + reasons + '
' diff --git a/scraper/dashboard/static/style.css b/scraper/dashboard/static/style.css index 0a057f0..cfd3abe 100644 --- a/scraper/dashboard/static/style.css +++ b/scraper/dashboard/static/style.css @@ -169,6 +169,21 @@ button.chip-toggle { min-width: 120px; } .chip.sen { background: #f5eef9; color: var(--sen); border-color:#e2d2ea; } .chip.marker { background: var(--accent); color: #fff; border-color: var(--accent); } + +/* ---- legislative status / "why is this here" reason (always on) ---- */ +.row-reason { + margin: 4px 0; font-size: 12px; color: var(--muted); + display: flex; align-items: baseline; gap: 6px; } +.chip.status { border-color: var(--border); } +.chip.status.open { background: #eef3fb; color: var(--asm); + border-color: #cdd9ee; } +.chip.status.closed { background: #f0f0f0; color: var(--warn); + border-color: #d8c9a8; } +.chip.status.pclosed { font-style: italic; } +.row-reason-why { color: var(--muted); } + /* a terminal/closed bill is dimmed so it doesn't read as an open upcoming */ +.row-reason[data-closed='1'] { opacity: .85; } +.row-reason[data-closed='1'] .row-reason-why { color: var(--muted); } .tag { display: inline-block; font-size: 11px; color: var(--muted); border: 1px solid var(--border); border-radius: 12px; padding: 0 6px; margin: 0 4px 0 0; } diff --git a/tests/test_dashboard_api.py b/tests/test_dashboard_api.py index 92ff38c..2ebd41f 100644 --- a/tests/test_dashboard_api.py +++ b/tests/test_dashboard_api.py @@ -141,11 +141,29 @@ def test_bills_personal_returns_relevance_and_floats(services): def test_bills_personal_requires_auth(services): status, body = api.run(_req("GET", "/api/v1/bills", services, - query={"personal": "1"})) + query={"personal": "1"})) assert status == 401, body assert body["error"]["code"] == "not_authenticated" +# ---- §5 "upcoming" item 2: each item carries status + reason ------------- + +def test_bills_items_carry_status_reason_and_possibly_closed(services): + # Every item must expose the "why is this here?" triad so a closed/past + # bill doesn't read as a vague "pending" (API_CONTRACT §5 item 2). + status, body = api.run(_req("GET", "/api/v1/bills", services, + query={"upcoming": "1"})) + assert status == 200, body + items = body["items"] + assert items, "expected at least one item" + for it in items: + for key in ("status", "reason", "possibly_closed"): + assert key in it, key + assert it["status"] in ("open", "closed", "unknown") + assert isinstance(it["reason"], str) and it["reason"] + assert it["possibly_closed"] in (True, False, None) + + # ---- six empty states -> 200 items:[] (never 4xx/5xx) --------------------- def test_empty_no_next_hearing(services): diff --git a/tests/test_dashboard_bill_status.py b/tests/test_dashboard_bill_status.py new file mode 100644 index 0000000..d32747a --- /dev/null +++ b/tests/test_dashboard_bill_status.py @@ -0,0 +1,110 @@ +"""Tests for ``scraper.dashboard.bill_status`` — the "why is this bill here?" +classifier (API_CONTRACT §5 "upcoming" item 2: a bill should not be a vague +"pending" when its floor history already shows a terminal / current state). + +Pure, no DB: ``derive_status(description)`` -> ``{status, reason, +possibly_closed}``. The *latest* matching action in the chronological log wins; +the contract's "default to including, flag possibly_closed" applies when the +source can't disambiguate. +""" +from __future__ import annotations + +from scraper.dashboard import bill_status +from scraper.dashboard.bill_status import derive_status, is_closed + + +# --------------------------------------------------------------------------- +# the motivating case: a passed bill sitting in "unfinished business" +# --------------------------------------------------------------------------- + +def test_awaiting_concurrence_in_senate_amendments_is_open(): + # AB1099: passed the Assembly, now awaiting Senate-amendment concurrence. + r = derive_status( + "Vote required: 41 2026 Aug. 30 - Read third time. Passed. " + "Ordered to the Assembly. (Ayes 39. Noes 0.). Aug. 30 - In Assembly. " + "Concurrence in Senate amendments pending. View Details", + "AB 1099") + assert r["status"] == "open" + assert "concurrence" in r["reason"].lower() + assert r["possibly_closed"] is False + assert not is_closed(r) + + +def test_passed_is_not_terminal_but_closed_terms_are(): + # "Passed" (one chamber) is an in-progress step -> open, not dropped. + passed = derive_status("Read third time. Passed. Ordered to the Assembly.", + "AB 1") + assert passed["status"] == "open" + + # "To inactive file" is a terminal/acted state -> closed, possibly_closed. + inact = derive_status("2026 Aug. 25 - To inactive file on motion of " + "Assembly Member Jackson. View Details", "AB 2") + assert inact["status"] == "closed" + assert inact["possibly_closed"] is True + assert is_closed(inact) + assert "inactive" in inact["reason"].lower() + + +# --------------------------------------------------------------------------- +# latest-action-wins + priority +# --------------------------------------------------------------------------- + +def test_latest_action_in_log_wins(): + # A bill that passed long ago but now has "Motion to reconsider + # continued" surfaces the *current* reconsideration state, not the old pass. + r = derive_status( + "2025 Sep. 13 - Read third time. Passed. Ordered to the Assembly. " + "Sep. 13 - In Assembly. Assembly refused to concur in Senate " + "amendments. (Ayes 34. Noes 27.) Motion to reconsider on the next " + "legislative day made. ... 2026 Aug. 30 - Motion to reconsider " + "continued. View Details", + "AB 1231") + assert "reconsider" in r["reason"].lower() + assert r["status"] == "open" + + +def test_priority_breates_ties_on_same_span(): + # "Motion to reconsider" must win over a bare "passed" even though both + # match; the specific/current fact dominates. + r = derive_status("Passed. Motion to reconsider made.", "AB 1") + assert "reconsider" in r["reason"].lower() + + +# --------------------------------------------------------------------------- +# unknown / content-free -> default to including + flag (API_CONTRACT §5) +# --------------------------------------------------------------------------- + +def test_content_free_defaults_to_unknown_and_flags(): + # A bare dated line with no action text -> "On file", never a false pass. + r = derive_status("Vote required: 41 2026 May 27 - View Details", "AB 3") + assert r["status"] == "unknown" + assert r["possibly_closed"] is None # can't confirm -> not asserted open + assert "on file" in r["reason"].lower() + + +def test_empty_description_is_unknown_not_crash(): + for text in (None, "", " "): + r = derive_status(text, "AB 4") + assert r["status"] == "unknown" + assert r["possibly_closed"] is None + + +# --------------------------------------------------------------------------- +# a few more real-world terminal / open signals +# --------------------------------------------------------------------------- + +def test_engrossed_and_withdrawn_are_closed(): + assert is_closed(derive_status("Engrossed. View Details", "AB 5")) + assert is_closed(derive_status("Withdrawn from committee. View Details", + "AB 6")) + + +def test_in_committee_and_scheduled_hearing_are_open(): + assert derive_status("Referred to Com. on Health. View Details", "AB 7")["status"] == "open" + assert derive_status("June 17 - Set for first hearing. View Details", + "AB 8")["status"] == "open" + + +def test_return_shape_is_stable(): + r = derive_status("Pending. View Details", "AB 9") + assert set(r.keys()) == {"status", "reason", "possibly_closed"}