Initial commit
This commit is contained in:
kfj001 committed
commit
5f2db08b43
100 files changed
+16985
No files matched your search
@@ -0,0 +1,335 @@
|
||||
"""Read-only access to the dashboard's data sources (spec §5).
|
||||
|
||||
The dashboard never mutates the relational DB. It reads:
|
||||
|
||||
* the flat ``v_daily_file_records`` view (or ``daily_file_records``) for the
|
||||
upcoming-events queue — one bill event per ``(date, bill_id)``, enriched
|
||||
with per-bill ``tags`` from the LLM lane's ``bill_tags`` table;
|
||||
* ``bill_tags`` for the tag cloud (§4.1);
|
||||
* ``members`` / ``committees`` / ``bill_committees`` / ``committee_members`` for
|
||||
the legislators reference data (§3.2 / §4.2) and the
|
||||
committee-of-watched-legislator float path.
|
||||
|
||||
Everything here returns plain JSON-serialisable dicts so the API layer can
|
||||
ship them verbatim into the API_CONTRACT shapes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
from typing import Any, Dict, List, Optional, Sequence
|
||||
|
||||
from .. import relational
|
||||
|
||||
FLAT_TABLE = relational.FLAT_TABLE
|
||||
VIEW_NAME = "v_" + FLAT_TABLE
|
||||
|
||||
# Fields a bill event carries into the API_CONTRACT §1 Bill shape.
|
||||
# `tags` is injected separately (from bill_tags), not a stored column.
|
||||
_EVENT_FIELDS = (
|
||||
"bill_id", "bill_number", "chamber_origin", "author", "short_title",
|
||||
"description", "committee", "hearing_time", "location", "date",
|
||||
"session", "item_number", "supplement", "source",
|
||||
)
|
||||
|
||||
|
||||
def _slug(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
return re.sub(r"[^a-z0-9]+", "-", name.strip().lower()).strip("-") or "unknown"
|
||||
|
||||
|
||||
def _title_case(s: str) -> str:
|
||||
return " ".join(w[:1].upper() + w[1:] for w in s.split() if w)
|
||||
|
||||
|
||||
def _table_exists(conn: sqlite3.Connection, name: str) -> bool:
|
||||
cur = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (name,))
|
||||
return cur.fetchone() is not None
|
||||
|
||||
|
||||
def _view_exists(conn: sqlite3.Connection, name: str) -> bool:
|
||||
cur = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='view' AND name=?", (name,))
|
||||
return cur.fetchone() is not None
|
||||
|
||||
|
||||
def _flat_source(conn: sqlite3.Connection) -> str:
|
||||
"""Prefer the flat-preserving view (the QA contract); fall back to the table."""
|
||||
return VIEW_NAME if _view_exists(conn, VIEW_NAME) else FLAT_TABLE
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tags (LLM lane: bill_tags table, §4.1)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def load_bill_tags(db_path: str = relational.DEFAULT_DB) -> Dict[str, List[str]]:
|
||||
"""Map ``bill_id`` -> [tag, ...] from the ``bill_tags`` table.
|
||||
|
||||
Absent/empty table -> ``{}`` so a bill with no analysis degrades to
|
||||
``tags: []`` (DESIGN §6.6 — no chips, ranking by legislator only).
|
||||
"""
|
||||
tags: Dict[str, List[str]] = {}
|
||||
if not os.path.exists(db_path):
|
||||
return tags
|
||||
conn = relational.connect(db_path, foreign_keys=False)
|
||||
try:
|
||||
if not _table_exists(conn, "bill_tags"):
|
||||
return tags
|
||||
rows = conn.execute(
|
||||
"SELECT bill_id, tag FROM bill_tags "
|
||||
"WHERE bill_id IS NOT NULL AND tag IS NOT NULL AND tag != '' "
|
||||
"ORDER BY bill_id, tag").fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
for r in rows:
|
||||
bid = r["bill_id"]
|
||||
tag = r["tag"]
|
||||
bucket = tags.setdefault(bid, [])
|
||||
if tag not in bucket:
|
||||
bucket.append(tag)
|
||||
return tags
|
||||
|
||||
|
||||
def load_tag_cloud(db_path: str = relational.DEFAULT_DB,
|
||||
as_of: Optional[str] = None,
|
||||
top: Optional[int] = None) -> Dict[str, Any]:
|
||||
"""The /tags cloud (§4.1): ``[{tag_id, label, count}, ...]`` sorted desc.
|
||||
|
||||
Count = number of bills carrying the tag. May be sparse early (LLM lane
|
||||
warming) — the UI shows a "filling in" hint, never a 0-count chip.
|
||||
"""
|
||||
import datetime
|
||||
conn = relational.connect(db_path, foreign_keys=False)
|
||||
try:
|
||||
if not _table_exists(conn, "bill_tags"):
|
||||
data = []
|
||||
else:
|
||||
data = conn.execute(
|
||||
"SELECT tag, COUNT(DISTINCT bill_id) AS n FROM bill_tags "
|
||||
"WHERE tag IS NOT NULL AND tag != '' GROUP BY tag "
|
||||
"ORDER BY n DESC, tag").fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
out = []
|
||||
for r in data:
|
||||
out.append({
|
||||
"tag_id": "tag:" + str(r["tag"]),
|
||||
"label": _title_case(str(r["tag"])),
|
||||
"count": int(r["n"]),
|
||||
})
|
||||
if top:
|
||||
out = out[:top]
|
||||
return {
|
||||
"as_of": as_of or _now_iso(),
|
||||
"tags": out,
|
||||
}
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
import datetime
|
||||
return datetime.datetime.now(datetime.timezone.utc).strftime(
|
||||
"%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Upcoming-events queue (the daily-file calendar, §2 / §5 "upcoming")
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _in_window(date: Optional[str], frm: Optional[str], to: Optional[str],
|
||||
as_of: Optional[str]) -> bool:
|
||||
"""API_CONTRACT §5 'upcoming' gate on event_date (= date here).
|
||||
|
||||
* a T-1 day (date < as_of) is EXCLUDED when an as_of floor is set;
|
||||
* 'this week'/'this month' are from/to windows over event_date.
|
||||
Dates are ISO ``YYYY-MM-DD`` so string compare == chronological.
|
||||
"""
|
||||
if not date:
|
||||
return not frm and not to
|
||||
if frm and date < frm:
|
||||
return False
|
||||
if to and date > to:
|
||||
return False
|
||||
if as_of and date < as_of:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def list_events(db_path: str = relational.DEFAULT_DB,
|
||||
upcoming: bool = False,
|
||||
from_date: Optional[str] = None,
|
||||
to_date: Optional[str] = None,
|
||||
chamber: Optional[str] = None,
|
||||
tags: Optional[Sequence[str]] = None,
|
||||
q: Optional[str] = None,
|
||||
author: Optional[str] = None,
|
||||
as_of: Optional[str] = None) -> List[Dict[str, Any]]:
|
||||
"""Build the bill-event queue, one event per ``(date, bill_id)``.
|
||||
|
||||
* ``upcoming=True`` + ``as_of`` -> keep only future-dated events
|
||||
(API_CONTRACT §5). When the caller omits ``as_of`` the default is the
|
||||
latest date present, so a demo on a single historical day still shows its
|
||||
queue; pin ``as_of`` for a true "future-of-now" view.
|
||||
* dedupe *within* a date by ``bill_id`` (multiple agenda items the same day
|
||||
collapse to one); across dates they are distinct events.
|
||||
* enriched with per-bill ``tags`` from the LLM lane.
|
||||
* optional filters: chamber, tags (any-match), free-text ``q``, author.
|
||||
|
||||
Returns list of dicts matching API_CONTRACT §1 **without** the ``relevance``
|
||||
block (the API layer adds that for logged-in views).
|
||||
"""
|
||||
conn = relational.connect(db_path, foreign_keys=False)
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM " + _flat_source(conn) +
|
||||
" WHERE bill_id IS NOT NULL AND bill_id != '' "
|
||||
"ORDER BY date, house, bill_id, committee, id").fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
flat = [dict(r) for r in rows]
|
||||
|
||||
bill_tags = load_bill_tags(db_path)
|
||||
tag_filter = {t.strip().lower() for t in (tags or []) if str(t).strip()}
|
||||
ql = (q or "").strip().lower()
|
||||
al = (author or "").strip().lower()
|
||||
|
||||
if as_of is None:
|
||||
dates = sorted({r["date"] for r in flat if r.get("date")})
|
||||
as_of = dates[-1] if dates else None
|
||||
|
||||
# Collapse to one event per (date, bill_id); prefer a row carrying a hearing.
|
||||
seen: Dict[tuple, Dict[str, Any]] = {}
|
||||
order: List[tuple] = []
|
||||
for r in flat:
|
||||
key = (r.get("date"), r.get("bill_id"))
|
||||
if key in seen:
|
||||
if not seen[key].get("_has_time") and r.get("hearing_time"):
|
||||
seen[key] = r
|
||||
continue
|
||||
r["_has_time"] = bool(r.get("hearing_time"))
|
||||
seen[key] = r
|
||||
order.append(key)
|
||||
|
||||
events = []
|
||||
for key in order:
|
||||
r = seen[key]
|
||||
origin = str(r.get("chamber_origin") or r.get("house") or "").lower()
|
||||
if chamber and origin != chamber.lower():
|
||||
continue
|
||||
if not _in_window(r.get("date"), from_date, to_date, as_of):
|
||||
continue
|
||||
btags = bill_tags.get(r["bill_id"], [])
|
||||
if tag_filter and not (tag_filter & {t.lower() for t in btags}):
|
||||
continue
|
||||
if ql:
|
||||
hay = " ".join(str(r.get(f) or "") for f in
|
||||
("short_title", "description", "bill_number",
|
||||
"author", "committee")).lower()
|
||||
if ql not in hay:
|
||||
continue
|
||||
if al and al not in str(r.get("author") or "").lower():
|
||||
continue
|
||||
ev = {f: r.get(f) for f in _EVENT_FIELDS}
|
||||
for f in ("hearing_time", "location", "committee", "short_title",
|
||||
"author", "item_number", "supplement", "date", "session"):
|
||||
if ev.get(f) == "":
|
||||
ev[f] = None
|
||||
ev["tags"] = btags if btags else None
|
||||
ev["source"] = r.get("source") or "ASSEMBLY_DAILY_FILE"
|
||||
events.append(ev)
|
||||
return events
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Legislators + districts reference data (§3.2 / §4.2)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def load_legislators(db_path: str = relational.DEFAULT_DB,
|
||||
q: Optional[str] = None) -> List[Dict[str, Any]]:
|
||||
"""Browse legislators (members table) with their committees + domains.
|
||||
|
||||
``domains`` = the domain tags of the committees a member sits on
|
||||
(committee -> its bills' tags). Degrades to an empty list when the
|
||||
correlation lane has no roster (no committee_members).
|
||||
"""
|
||||
conn = relational.connect(db_path, foreign_keys=False)
|
||||
try:
|
||||
members = [dict(r) for r in conn.execute(
|
||||
"SELECT name_norm, full_name, chamber, kind "
|
||||
"FROM members ORDER BY full_name, chamber").fetchall()]
|
||||
rows = conn.execute(
|
||||
"SELECT cm.member, c.name_raw AS committee "
|
||||
"FROM committee_members cm "
|
||||
"JOIN committees c ON c.name_key=cm.committee "
|
||||
"AND c.chamber=cm.chamber AND c.session=cm.session "
|
||||
"ORDER BY cm.member, c.name_raw").fetchall()
|
||||
served: Dict[str, List[str]] = {}
|
||||
for r in rows:
|
||||
served.setdefault(r["member"], []).append(r["committee"])
|
||||
|
||||
if _table_exists(conn, "bill_tags"):
|
||||
cmt_rows = conn.execute(
|
||||
"SELECT c.name_raw AS cn, t.tag FROM bill_committees bc "
|
||||
"JOIN committees c ON c.name_key=bc.committee "
|
||||
"AND c.chamber=bc.chamber AND c.session=bc.session "
|
||||
"JOIN bill_tags t ON t.bill_id=bc.bill_id").fetchall()
|
||||
cmt_tags: Dict[str, set] = {}
|
||||
for r in cmt_rows:
|
||||
cmt_tags.setdefault(r["cn"], set()).add(r["tag"])
|
||||
else:
|
||||
cmt_tags = {}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
ql = (q or "").strip().lower()
|
||||
out = []
|
||||
for m in members:
|
||||
slug = m["name_norm"]
|
||||
committees = served.get(slug, [])
|
||||
domains = sorted({t for cn in committees for t in cmt_tags.get(cn, set())})
|
||||
if ql and ql not in str(m.get("full_name") or slug).lower():
|
||||
continue
|
||||
out.append({
|
||||
"legislator_id": slug,
|
||||
"name": m.get("full_name") or slug.title(),
|
||||
"party": None,
|
||||
"chamber": (m.get("chamber") or "").upper(),
|
||||
"district": None,
|
||||
"committees": committees,
|
||||
"domains": domains,
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def committee_roster_for(db_path: str = relational.DEFAULT_DB,
|
||||
legislator_ids=None) -> Dict[str, List[str]]:
|
||||
"""Map a watched legislator (by its ``member`` slug) to the committee
|
||||
name_keys that legislator is known to serve on, via the correlation lane's
|
||||
``committee_members`` join.
|
||||
|
||||
Used by the ranking to fire the ``committee-of-watched-legislator`` float
|
||||
path. When the correlation lane found no roster (``committee_members``
|
||||
empty — the honest default), this returns ``{}`` and that path does not
|
||||
fire, so nothing is fabricated.
|
||||
"""
|
||||
conn = relational.connect(db_path, foreign_keys=False)
|
||||
try:
|
||||
if not _table_exists(conn, "committee_members"):
|
||||
return {}
|
||||
rows = conn.execute(
|
||||
"SELECT DISTINCT cm.member AS m, cm.committee AS ck "
|
||||
"FROM committee_members cm").fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
roster: Dict[str, List[str]] = {}
|
||||
for r in rows:
|
||||
m = r["m"]
|
||||
if legislator_ids is None or m in legislator_ids:
|
||||
bucket = roster.setdefault(m, [])
|
||||
if r["ck"] not in bucket:
|
||||
bucket.append(r["ck"])
|
||||
return roster
|
||||
Reference in new issue
Block a user