Initial commit
This commit is contained in:
kfj001 committed
commit
5f2db08b43
100 files changed
+16985
No files matched your search
@@ -0,0 +1,18 @@
|
||||
"""cali_scraper: HTTP ingestion layer for the CA Legislature Daily File.
|
||||
|
||||
Public surface:
|
||||
from scraper.ingest import Ingestor
|
||||
ing = Ingestor()
|
||||
result = ing.ingest(date="2026-09-02")
|
||||
|
||||
CLI:
|
||||
python -m scraper ingest --date 2026-09-02
|
||||
python -m scraper ingest --date 2026-09-02 --source assembly --force
|
||||
python -m scraper ingest --list
|
||||
"""
|
||||
|
||||
from .ingest import Ingestor, IngestResult
|
||||
from .config import Config, load_config
|
||||
|
||||
__all__ = ["Ingestor", "IngestResult", "Config", "load_config"]
|
||||
__version__ = "0.1.0"
|
||||
@@ -0,0 +1,10 @@
|
||||
"""``python -m scraper`` entry point."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
|
||||
from .cli import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Alert registry / hooks.
|
||||
|
||||
The ingestion layer must "alert" rather than silently fail on 404 / 5xx / antibot.
|
||||
This module defines a tiny registry of named sink handlers (log/webhook/...) and a
|
||||
single `fire()` entry point that the rest of the package calls. Adding a new sink
|
||||
(e.g. an email notifier) is a two-line change here -- no edits to the caller.
|
||||
|
||||
Handler contract: handler(context: AlertContext) -> None
|
||||
context.kind : "not_found" | "transient" | "antibot" | "ok"
|
||||
context.level : "info" | "warning" | "error"
|
||||
context.message : human-readable one-liner
|
||||
context.source : source id ("assembly_daily_file", ...)
|
||||
context.date : requested date (ISO) or None
|
||||
context.url : the URL involved, if any
|
||||
context.extra : free-form dict (status, attempts, ...)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
|
||||
|
||||
log = logging.getLogger("scraper.alerts")
|
||||
|
||||
|
||||
@dataclass
|
||||
class AlertContext:
|
||||
kind: str
|
||||
level: str
|
||||
message: str
|
||||
source: Optional[str] = None
|
||||
date: Optional[str] = None
|
||||
url: Optional[str] = None
|
||||
extra: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
# Registry of sinks. A "sink" is any callable AlertContext -> None.
|
||||
LogHandler: Callable[[AlertContext], None] = lambda ctx: log.log(
|
||||
getattr(logging, ctx.level.upper(), logging.INFO),
|
||||
f"[{ctx.kind}] {ctx.message}" + (f" ({ctx.source} {ctx.date})" if ctx.source else ""),
|
||||
)
|
||||
|
||||
|
||||
def log_sink(ctx: AlertContext) -> None:
|
||||
"""The default sink: emit a structured line to stderr + the logging stream."""
|
||||
print(f"[alert:{ctx.level}] {ctx.kind}: {ctx.message}", file=sys.stderr)
|
||||
LogHandler(ctx)
|
||||
|
||||
|
||||
def webhook_sink(ctx: AlertContext) -> None: # pragma: no cover - no live webhook
|
||||
import os
|
||||
import json
|
||||
url = os.environ.get("CALI_SCRAPER_WEBHOOK", "")
|
||||
if not url:
|
||||
log_sink(ctx)
|
||||
return
|
||||
import httpx
|
||||
httpx.post(url, json=ctx.__dict__, timeout=10, verify=False)
|
||||
|
||||
|
||||
ALERT_REGISTRY: Dict[str, Callable[[AlertContext], None]] = {
|
||||
"log": log_sink,
|
||||
"webhook": webhook_sink,
|
||||
}
|
||||
|
||||
|
||||
def register(name: str, handler: Callable[[AlertContext], None]) -> None:
|
||||
ALERT_REGISTRY[name] = handler
|
||||
|
||||
|
||||
def fire(config_alerts: Dict[str, Any], ctx: AlertContext) -> None:
|
||||
"""Dispatch an alert through the configured sink(s).
|
||||
|
||||
The alert may be disabled entirely via config alerts.enabled=false, but "ok"
|
||||
and "not_found" are always at minimum logged so the run is auditable.
|
||||
"""
|
||||
if not config_alerts.get("enabled", True) and ctx.level == "info":
|
||||
return
|
||||
sink = config_alerts.get("sink", "log")
|
||||
handler = ALERT_REGISTRY.get(sink, log_sink)
|
||||
try:
|
||||
handler(ctx)
|
||||
except Exception as exc: # an alerting failure must never crash the run
|
||||
log.error("alert sink '%s' failed: %s", sink, exc)
|
||||
log_sink(ctx)
|
||||
@@ -0,0 +1,132 @@
|
||||
"""``scraper.analyze`` — spec §4 CLI subcommand for Ollama / fallback bill analysis.
|
||||
|
||||
Wires into the shared ``scraper/cli.py`` via the guarded ``register()`` pattern
|
||||
(see ``scraper/parse_cli.py`` and ``scraper/relate_cli.py``):
|
||||
|
||||
* The ``analyze`` subcommand reads ``config.yaml``'s ``llm:`` section for the
|
||||
endpoint + model knobs.
|
||||
* When ``llm.enabled`` is false (or a server is unreachable / model empty) the
|
||||
run uses the deterministic built-in tagger (zero network).
|
||||
* Tags persist to ``bill_tags`` (``scraper.llm``); the tag cloud is written to
|
||||
``out/tag_cloud.json`` (or ``--out``), plus a CLI summary.
|
||||
* ``--force-llm`` overrides ``enabled: false`` to force the live server path.
|
||||
* ``--list-config`` prints the resolved llm: settings and exits 0.
|
||||
* ``--db`` overrides ``relational.DEFAULT_DB``.
|
||||
* ``--roster`` is ignored (accepted for CLI compatibility with the correlate lane;
|
||||
this lane does no roster matching).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
from . import llm, relational
|
||||
from .config import load_config
|
||||
|
||||
log = logging.getLogger("scraper.analyze")
|
||||
|
||||
_DEFAULT_OUT_DIR = os.path.normpath(
|
||||
os.path.join(os.path.dirname(relational.DEFAULT_DB), "..", "out"))
|
||||
|
||||
|
||||
def _already_registered(sub, name: str) -> bool:
|
||||
actions = getattr(sub, "_choices_actions", [])
|
||||
return any(getattr(a, "name", None) == name for a in actions)
|
||||
|
||||
|
||||
def register(sub) -> None:
|
||||
"""Register the ``analyze`` subcommand (guarded — skip if a sibling owns it)."""
|
||||
if _already_registered(sub, "analyze"):
|
||||
return
|
||||
|
||||
pa = sub.add_parser(
|
||||
"analyze",
|
||||
help="subject-matter tag bills via Ollama (or deterministic fallback)",
|
||||
description=("Tag every bill with subject-matter tags using a configurable "
|
||||
"Ollama endpoint + model (from config.llm). Falls back to the "
|
||||
"deterministic built-in tagger when llm.enabled=false, the endpoint "
|
||||
"is unreachable, or model is empty. Persists to bill_tags; writes "
|
||||
"the tag cloud to out/tag_cloud.json"),
|
||||
)
|
||||
pa.add_argument("--db", default=relational.DEFAULT_DB,
|
||||
help="SQLite path (default: db/daily_file.sqlite)")
|
||||
pa.add_argument("--out", dest="out_path", default=None,
|
||||
help="tag-cloud JSON output path "
|
||||
"(default: out/tag_cloud.json)")
|
||||
pa.add_argument("--source", dest="house", default=None,
|
||||
help="limit to a publish house, e.g. 'assembly' or 'senate' "
|
||||
"(matches house or chamber_origin)")
|
||||
pa.add_argument("--top", type=int, default=None,
|
||||
help="limit the tag cloud / CLI summary to the N most frequent tags")
|
||||
pa.add_argument("--rebuild", action="store_true",
|
||||
help="clear bill_tags before writing (full re-analysis)")
|
||||
pa.add_argument("--force-llm", dest="force_llm", action="store_true",
|
||||
help="force the live Ollama path even when llm.enabled=false")
|
||||
pa.add_argument("--list-config", action="store_true",
|
||||
help="print resolved llm: config and exit 0")
|
||||
pa.add_argument("--roster", dest="roster", default=None,
|
||||
help="ignored (compat with the correlate lane)")
|
||||
pa.set_defaults(func=handle_analyze)
|
||||
|
||||
|
||||
def _resolved_cfg(config_path: str = None, force_llm: bool = False):
|
||||
"""Load the ``llm:`` slice from config.yaml (defaults fill missing keys)."""
|
||||
cfg = load_config(config_path)
|
||||
llm_cfg = dict(cfg.llm)
|
||||
if force_llm:
|
||||
llm_cfg["enabled"] = True
|
||||
return llm_cfg
|
||||
|
||||
|
||||
def _print_llm_config(cfg_llm: dict, config_path: str = "") -> int:
|
||||
print("llm:")
|
||||
for k in ("enabled", "backend", "endpoint", "model",
|
||||
"default_tags", "timeout_seconds"):
|
||||
print(" %s: %s" % (k, cfg_llm.get(k)))
|
||||
if config_path:
|
||||
print(" (from %s)" % config_path)
|
||||
return 0
|
||||
|
||||
|
||||
def handle_analyze(args) -> int:
|
||||
"""CLI handler for the ``analyze`` subcommand."""
|
||||
logging.basicConfig(
|
||||
level=logging.WARNING,
|
||||
format="%(levelname)s %(name)s: %(message)s")
|
||||
|
||||
llm_cfg = _resolved_cfg(getattr(args, "config", None),
|
||||
force_llm=getattr(args, "force_llm", False))
|
||||
|
||||
if getattr(args, "list_config", False):
|
||||
return _print_llm_config(llm_cfg, getattr(args, "config", ""))
|
||||
|
||||
out_path = getattr(args, "out_path", None)
|
||||
out = out_path or os.path.join(_DEFAULT_OUT_DIR, "tag_cloud.json")
|
||||
|
||||
db_path = getattr(args, "db", relational.DEFAULT_DB)
|
||||
house = getattr(args, "house", None)
|
||||
top = getattr(args, "top", None)
|
||||
rebuild = getattr(args, "rebuild", False)
|
||||
|
||||
summary = llm.analyze(
|
||||
db_path=db_path,
|
||||
out_path=out,
|
||||
cfg=llm_cfg,
|
||||
house=house,
|
||||
rebuild=rebuild,
|
||||
top=top,
|
||||
)
|
||||
|
||||
print("=== analyze ===")
|
||||
print(" bills_tagged : %d" % summary["bills_tagged"])
|
||||
print(" unique_tags : %d" % summary["unique_tags"])
|
||||
print(" path : %s" % summary["path"])
|
||||
print(" generated_by : %s" % summary["generated_by"])
|
||||
print(" reason : %s" % summary["reason"])
|
||||
print(" rows_written : %d" % summary["rows_written"])
|
||||
print(" out : %s" % summary["cloud_path"])
|
||||
print(" top tags :")
|
||||
for entry in summary.get("top_tags", []):
|
||||
print(" %4d %s" % (entry.get("count", 0), entry.get("tag", "")))
|
||||
return 0
|
||||
@@ -0,0 +1,75 @@
|
||||
"""bill_id codec + session/chamber/number helpers, per spec §4."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Optional, Tuple
|
||||
|
||||
# Universal bill_id pattern across all daily-file links (spec §4).
|
||||
# 20252026 0 AB 28 -> 202520260AB28
|
||||
BILL_ID_RE = re.compile(
|
||||
r"^(?P<session>\d{8})0?(?P<chamber>AB|SB|ACR|SCR|HR|SJR|ABR)"
|
||||
r"(?P<number>\d+)$"
|
||||
)
|
||||
|
||||
# Chamber prefix -> origin house (spec §8 #7: origin, not publishing house).
|
||||
ASSEMBLY_PREFIXES = {"AB", "ACR", "HR", "ABR"}
|
||||
SENATE_PREFIXES = {"SB", "SCR", "SJR"}
|
||||
|
||||
_SESSION_RE = re.compile(r"^(?P<a>\d{4})(?P<b>\d{4})$")
|
||||
_BILL_DISPLAY_RE = re.compile(r"^(?P<prefix>AB|SB|ACR|SCR|HR|SJR|ABR)\s*(?P<num>\d+)$")
|
||||
_SUPPLEMENT_RE = re.compile(r"Supplemental\s+(\d+)", re.IGNORECASE)
|
||||
|
||||
|
||||
def origin_house(chamber: str) -> str:
|
||||
if chamber in ASSEMBLY_PREFIXES:
|
||||
return "assembly"
|
||||
if chamber in SENATE_PREFIXES:
|
||||
return "senate"
|
||||
return "unknown"
|
||||
|
||||
|
||||
def normalise_session(raw: str) -> str:
|
||||
"""'20252026' -> '2025-2026'."""
|
||||
m = _SESSION_RE.match(raw or "")
|
||||
if m:
|
||||
return f"{m['a']}-{m['b']}"
|
||||
return raw
|
||||
|
||||
|
||||
def parse_bill_id(bill_id: str) -> Optional[dict]:
|
||||
"""Decode a canonical bill_id. Returns dict or None if it does not match.
|
||||
|
||||
Keys: session (YYYY-YYYY), session_raw, chamber (prefix upper), number
|
||||
(int, unpadded), display (e.g. 'AB 28'), house (origin), raw.
|
||||
"""
|
||||
if not bill_id:
|
||||
return None
|
||||
m = BILL_ID_RE.match(bill_id.strip())
|
||||
if not m:
|
||||
return None
|
||||
session_raw = m["session"]
|
||||
chamber = m["chamber"].upper()
|
||||
number = int(m["number"])
|
||||
return {
|
||||
"session": normalise_session(session_raw),
|
||||
"session_raw": session_raw,
|
||||
"chamber": chamber,
|
||||
"number": number,
|
||||
"display": f"{chamber} {number}",
|
||||
"house": origin_house(chamber),
|
||||
"raw": bill_id,
|
||||
}
|
||||
|
||||
|
||||
def parse_display(display: str) -> Optional[Tuple[str, int]]:
|
||||
"""'AB 00028' or 'AB 28' -> ('AB', 28)."""
|
||||
m = _BILL_DISPLAY_RE.match((display or "").strip())
|
||||
if not m:
|
||||
return None
|
||||
return m["prefix"].upper(), int(m["num"])
|
||||
|
||||
|
||||
def parse_supplement(text: str) -> Optional[int]:
|
||||
m = _SUPPLEMENT_RE.search(text or "")
|
||||
return int(m.group(1)) if m else None
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Local raw-byte cache.
|
||||
|
||||
Layout under cache.raw_dir:
|
||||
{source}/{YYYY-MM-DD}.{ext} the raw bytes (html or pdf)
|
||||
{source}/{YYYY-MM-DD}.meta.json sidecar: url, status, sha256, fetched_at
|
||||
|
||||
A cache "hit" is served straight from disk (cache_hit=True) and is the ONLY way a
|
||||
re-ingest avoids a network round-trip. Writes are atomic (tmp + os.replace) so a
|
||||
crash mid-download never leaves a truncated file.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
|
||||
def sha256_bytes(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
class RawCache:
|
||||
"""On-disk keyed cache of raw response bodies."""
|
||||
|
||||
def __init__(self, raw_dir):
|
||||
self.raw_dir = Path(raw_dir)
|
||||
|
||||
def _key_path(self, source: str, date: str, ext: str) -> Path:
|
||||
return self.raw_dir / source / f"{date}.{ext}"
|
||||
|
||||
def meta_path(self, source: str, date: str) -> Path:
|
||||
return self.raw_dir / source / f"{date}.meta.json"
|
||||
|
||||
def get(self, source: str, date: str, ext: str) -> Optional[bytes]:
|
||||
"""Return cached bytes if present AND matching its recorded sha256
|
||||
(guards a truncated/corrupt write). Else None (treated as a miss).
|
||||
"""
|
||||
p = self._key_path(source, date, ext)
|
||||
meta = self._read_meta(source, date)
|
||||
if not p.exists() or meta is None:
|
||||
return None
|
||||
try:
|
||||
data = p.read_bytes()
|
||||
except OSError:
|
||||
return None
|
||||
if meta.get("sha256") and sha256_bytes(data) != meta["sha256"]:
|
||||
return None # integrity mismatch -> miss
|
||||
return data
|
||||
|
||||
def put(self, source: str, date: str, ext: str, data: bytes,
|
||||
url: str, status: int, fetched_at: str) -> Path:
|
||||
path = self._key_path(source, date, ext)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
tmp = path.with_name(path.name + ".tmp")
|
||||
tmp.write_bytes(data)
|
||||
os.replace(tmp, path) # atomic on POSIX
|
||||
|
||||
meta = {
|
||||
"source": source,
|
||||
"date": date,
|
||||
"ext": ext,
|
||||
"url": url,
|
||||
"status": status,
|
||||
"sha256": sha256_bytes(data),
|
||||
"bytes": len(data),
|
||||
"fetched_at": fetched_at,
|
||||
}
|
||||
self._write_meta(source, date, meta)
|
||||
return path
|
||||
|
||||
def exists(self, source: str, date: str, ext: str) -> bool:
|
||||
return self.get(source, date, ext) is not None
|
||||
|
||||
# -- meta helpers --
|
||||
def _read_meta(self, source: str, date: str) -> Optional[dict]:
|
||||
mp = self.meta_path(source, date)
|
||||
if not mp.exists():
|
||||
return None
|
||||
try:
|
||||
return json.loads(mp.read_text())
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
def _write_meta(self, source: str, date: str, meta: dict) -> None:
|
||||
mp = self.meta_path(source, date)
|
||||
mp.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = mp.with_suffix(".json.tmp")
|
||||
tmp.write_text(json.dumps(meta, indent=2, sort_keys=True) + "\n")
|
||||
os.replace(tmp, mp)
|
||||
+211
@@ -0,0 +1,211 @@
|
||||
# CLI entry point: `python -m scraper <command>`.
|
||||
#
|
||||
# ingest --date YYYY-MM-DD [--source NAME] [--force] [--json]
|
||||
# list
|
||||
# check --date YYYY-MM-DD
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
|
||||
from .config import load_config
|
||||
from .ingest import Ingestor
|
||||
from .parse_cli import register, handle_parse
|
||||
from .relate_cli import register as relate_register
|
||||
from .relate_cli import handle_migrate, handle_correlate
|
||||
from . import analyze_cli as _ac
|
||||
from .analyze_cli import handle_analyze
|
||||
from .dashboard import dashboard_cli as _dcli
|
||||
from .dashboard.dashboard_cli import handle_dashboard
|
||||
|
||||
|
||||
def cmd_ingest(args) -> int:
|
||||
# --date is required by the task contract; if omitted, default to today
|
||||
# so the cron wrapper can run with no arguments.
|
||||
date = args.date or _today()
|
||||
if getattr(args, "validate", False):
|
||||
date = _validate_date(date)
|
||||
|
||||
exit_code = 0
|
||||
try:
|
||||
ing = Ingestor(config_path=args.config, force=args.force)
|
||||
sources = _sources_arg(args)
|
||||
result = ing.ingest(date, sources=sources)
|
||||
ing.close()
|
||||
except SystemExit as exc:
|
||||
# config / argparse errors surface here
|
||||
print("ERROR: %s" % exc, file=sys.stderr)
|
||||
return int(exc.code or 2)
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(result.to_dict(), indent=2))
|
||||
else:
|
||||
_print_summary(result)
|
||||
|
||||
# Non-zero exit only on a *real* failure (antibot/transient/unexpected
|
||||
# 404). An expected "no publication" 404 is a legitimate outcome for a
|
||||
# non-legislative day; the cron job must not treat it as a failure.
|
||||
for c in result.chambers:
|
||||
if c.status in ("antibot", "transient"):
|
||||
exit_code = 1
|
||||
return exit_code
|
||||
|
||||
|
||||
def _print_summary(result) -> None:
|
||||
print("=== Ingest %s ===" % result.date)
|
||||
for c in result.chambers:
|
||||
tag = {
|
||||
"ok": "OK",
|
||||
"cached": "CACHE",
|
||||
"not_found": "MISS",
|
||||
"antibot": "ANTIBOT",
|
||||
"transient": "FAIL",
|
||||
}.get(c.status, "-----")
|
||||
path = c.path
|
||||
if path is not None:
|
||||
path = path.name if hasattr(path, "name") else str(path)
|
||||
print(" [%s] %-18s %-10s %8dB attempts=%d %s"
|
||||
% (tag, c.source, c.status, c.bytes, c.attempts, c.note))
|
||||
if getattr(c, "as_of", None):
|
||||
print(" as_of=%s url=%s" % (c.as_of, c.url))
|
||||
print(" all_ok=%s" % result.all_ok)
|
||||
|
||||
|
||||
def cmd_list(args) -> int:
|
||||
cfg = load_config(args.config)
|
||||
print("Configured sources (from %s):" % cfg.path)
|
||||
for name, s in cfg.sources.items():
|
||||
print(" * %s (%s)" % (name, s.get("kind")))
|
||||
for key in ("url", "index_url", "pdf_base_url", "pdf_filename_pattern"):
|
||||
if key in s:
|
||||
val = s[key]
|
||||
if isinstance(val, list):
|
||||
print(" %s: %s" % (key, " / ".join(val)))
|
||||
else:
|
||||
print(" %s: %s" % (key, val))
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_check(args) -> int:
|
||||
# Report cache status for a date without touching the network.
|
||||
from .cache import RawCache
|
||||
cfg = load_config(args.config)
|
||||
rc = RawCache(cfg.raw_dir)
|
||||
date = args.date or _today()
|
||||
all_cached = True
|
||||
print("=== Cache check %s ===" % date)
|
||||
for name in cfg.source_names():
|
||||
s = cfg.source(name)
|
||||
ext = "pdf" if s.get("kind") == "pdf" else "html"
|
||||
hit = rc.exists(name, date, ext)
|
||||
all_cached = all_cached and hit
|
||||
print(" %-18s %s" % (name, "CACHED" if hit else "MISSING"))
|
||||
return 0 if all_cached else 3
|
||||
|
||||
|
||||
def _sources_arg(args):
|
||||
# --source accepts a single value or a comma-separated list.
|
||||
src = getattr(args, "source", None)
|
||||
if not src:
|
||||
return None
|
||||
if isinstance(src, (list, tuple)):
|
||||
return list(src)
|
||||
return [p.strip() for p in src.split(",") if p.strip()]
|
||||
|
||||
|
||||
def _today() -> str:
|
||||
# Local date so the cron wrapper can run with no args.
|
||||
import datetime
|
||||
return datetime.date.today().isoformat()
|
||||
|
||||
|
||||
def _validate_date(date: str) -> str:
|
||||
# Enforce clean YYYY-MM-DD; a bad date is a clear error, not a crash.
|
||||
import datetime
|
||||
try:
|
||||
datetime.date.fromisoformat(date)
|
||||
except ValueError:
|
||||
raise SystemExit(
|
||||
"Invalid --date '%s': expected YYYY-MM-DD, e.g. 2026-09-02." % date)
|
||||
return date
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(
|
||||
prog="python -m scraper",
|
||||
description="Ingest the CA Legislature Daily File pages.")
|
||||
p.add_argument("--config", help="Path to config.yaml (default: ./config.yaml)")
|
||||
sub = p.add_subparsers(dest="command", required=True)
|
||||
|
||||
# Parser lane (task t_fb8081e7): additive, guarded so it never shadows a
|
||||
# sibling owner of the dispatcher if one registers `parse` later.
|
||||
try:
|
||||
register(sub)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Correlation lane (task t_2736f40d, spec 2.5): additive + guarded, same
|
||||
# pattern as `parse`. Registers `migrate` + `correlate`; does not touch the
|
||||
# ingest/parse/list/check handlers.
|
||||
try:
|
||||
relate_register(sub)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# LLM analysis lane (task t_950780e0, spec §4): additive + guarded, same
|
||||
# pattern as parse + correlate. No collision with sibling handlers.
|
||||
try:
|
||||
_ac.register(sub)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Dashboard UI lane (task t_b38e9af2, spec §5): additive + guarded, same
|
||||
# pattern. Registers `dashboard`; does not touch sibling handlers.
|
||||
try:
|
||||
_dcli.register(sub)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
pi = sub.add_parser("ingest", help="Download + cache daily-file page(s)")
|
||||
pi.add_argument("--date", help="YYYY-MM-DD (default: today)")
|
||||
pi.add_argument("--source",
|
||||
help="Chamber name(s), comma-separated "
|
||||
"(assembly_daily_file, senate_daily_file). Default: all.")
|
||||
pi.add_argument("--force", action="store_true",
|
||||
help="Re-fetch even if already cached.")
|
||||
pi.add_argument("--json", action="store_true",
|
||||
help="Emit the result as JSON.")
|
||||
|
||||
pl = sub.add_parser("list", help="List configured sources + URLs")
|
||||
pc = sub.add_parser("check", help="Cache status for a date (no network)")
|
||||
pc.add_argument("--date", help="YYYY-MM-DD (default: today)")
|
||||
|
||||
return p
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
# argv defaults to sys.argv[1:] when the module is run directly.
|
||||
parser = build_parser()
|
||||
args = parser.parse_args(argv)
|
||||
args.validate = True
|
||||
handlers = {
|
||||
"ingest": cmd_ingest,
|
||||
"list": cmd_list,
|
||||
"check": cmd_check,
|
||||
"parse": handle_parse,
|
||||
"migrate": handle_migrate,
|
||||
"correlate": handle_correlate,
|
||||
"analyze": handle_analyze,
|
||||
"dashboard": handle_dashboard,
|
||||
}
|
||||
handler = handlers.get(args.command)
|
||||
if handler is None:
|
||||
parser.print_help()
|
||||
return 2
|
||||
return handler(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,147 @@
|
||||
"""Configuration loading and path resolution.
|
||||
|
||||
All tunable behaviour (URLs, retry policy, cache locations, alert sink) lives in
|
||||
config.yaml. This module loads it, resolves relative paths against the repo root,
|
||||
and exposes a small, validated object. Nothing in the rest of the package reads
|
||||
files directly -- it goes through Config.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError as exc: # pragma: no cover - PyYAML is a hard dep
|
||||
raise SystemExit(
|
||||
"PyYAML is required. Install with: python3 -m pip install --user pyyaml"
|
||||
) from exc
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
DEFAULT_CONFIG_PATH = PROJECT_ROOT / "config.yaml"
|
||||
|
||||
# Sensible fallbacks so the package still runs if a key is absent from config.yaml.
|
||||
_DEFAULTS: Dict[str, Any] = {
|
||||
"http": {
|
||||
"user_agent": "Mozilla/5.0 (research-scanner/1.0)",
|
||||
"timeout_seconds": 20,
|
||||
"max_retries": 3,
|
||||
"backoff_base_seconds": 1.0,
|
||||
"backoff_cap_seconds": 30.0,
|
||||
"retry_status": [500, 502, 503, 504],
|
||||
"antibot_status": [403],
|
||||
},
|
||||
"cache": {
|
||||
"raw_dir": "cache/raw",
|
||||
"request_log": "cache/request_log.jsonl",
|
||||
},
|
||||
"alerts": {
|
||||
"enabled": True,
|
||||
"sink": "log",
|
||||
},
|
||||
# spec §4 LLM analysis (task t_950780e0). Kept as fallbacks so the package
|
||||
# still runs even if config.yaml omits the `llm:` section. The authoritative
|
||||
# values live in config.yaml; these only fill missing keys.
|
||||
"llm": {
|
||||
"enabled": False,
|
||||
"backend": "ollama",
|
||||
"endpoint": "http://localhost:11434",
|
||||
"model": "llama3",
|
||||
"default_tags": 8,
|
||||
"timeout_seconds": 30,
|
||||
},
|
||||
"sources": {},
|
||||
}
|
||||
|
||||
_REQUIRED_SOURCES = ("assembly_daily_file", "senate_daily_file")
|
||||
|
||||
|
||||
class Config:
|
||||
"""Thin validated wrapper around the parsed config.yaml."""
|
||||
|
||||
def __init__(self, data: Dict[str, Any], root: Path, path: Path):
|
||||
self.data = data
|
||||
self.root = root
|
||||
self.path = path
|
||||
self._http = _merge(_DEFAULTS["http"], data.get("http") or {})
|
||||
self._cache = _merge(_DEFAULTS["cache"], data.get("cache") or {})
|
||||
self._alerts = _merge(_DEFAULTS["alerts"], data.get("alerts") or {})
|
||||
self._llm = _merge(_DEFAULTS["llm"], data.get("llm") or {})
|
||||
self._sources = data.get("sources") or {}
|
||||
self._validate_sources()
|
||||
|
||||
# -- paths ---------------------------------------------------------
|
||||
@property
|
||||
def raw_dir(self) -> Path:
|
||||
return self._resolve(self._cache["raw_dir"])
|
||||
|
||||
@property
|
||||
def request_log(self) -> Path:
|
||||
return self._resolve(self._cache["request_log"])
|
||||
|
||||
# -- config slices -------------------------------------------------
|
||||
@property
|
||||
def http(self) -> Dict[str, Any]:
|
||||
return self._http
|
||||
|
||||
@property
|
||||
def alerts(self) -> Dict[str, Any]:
|
||||
return self._alerts
|
||||
|
||||
@property
|
||||
def llm(self) -> Dict[str, Any]:
|
||||
return self._llm
|
||||
|
||||
@property
|
||||
def sources(self) -> Dict[str, Dict[str, Any]]:
|
||||
return self._sources
|
||||
|
||||
def source(self, name: str) -> Dict[str, Any]:
|
||||
if name not in self._sources:
|
||||
raise KeyError(f"Unknown source '{name}'. Known: {list(self._sources)}")
|
||||
return self._sources[name]
|
||||
|
||||
def source_names(self) -> List[str]:
|
||||
return list(self._sources.keys())
|
||||
|
||||
# -- helpers -------------------------------------------------------
|
||||
def _resolve(self, rel: str) -> Path:
|
||||
p = Path(rel)
|
||||
return p if p.is_absolute() else (self.root / p)
|
||||
|
||||
def _validate_sources(self) -> None:
|
||||
for name in _REQUIRED_SOURCES:
|
||||
if name not in self._sources:
|
||||
raise ValueError(
|
||||
f"config.yaml is missing sources.{name} "
|
||||
f"(required by the research spec t_a171f4d1)."
|
||||
)
|
||||
asm = self._sources.get("assembly_daily_file", {})
|
||||
sen = self._sources.get("senate_daily_file", {})
|
||||
if not asm.get("url"):
|
||||
raise ValueError("sources.assembly_daily_file.url is required")
|
||||
if not sen.get("index_url"):
|
||||
raise ValueError("sources.senate_daily_file.index_url is required")
|
||||
|
||||
|
||||
def _merge(base: Dict[str, Any], override: Dict[str, Any]) -> Dict[str, Any]:
|
||||
out = dict(base)
|
||||
out.update(override or {})
|
||||
return out
|
||||
|
||||
|
||||
def load_config(path: str | os.PathLike | None = None) -> Config:
|
||||
"""Load config.yaml; falls back to defaults with a warning if absent."""
|
||||
p = Path(path) if path else DEFAULT_CONFIG_PATH
|
||||
p = p if p.is_absolute() else (Path.cwd() / p)
|
||||
if not p.exists():
|
||||
raise SystemExit(f"Config file not found: {p}")
|
||||
with open(p, "r", encoding="utf-8") as fh:
|
||||
data = yaml.safe_load(fh) or {}
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(f"Config file {p} did not parse to a mapping.")
|
||||
root = p.resolve().parent
|
||||
return Config(data, root=root, path=p.resolve())
|
||||
@@ -0,0 +1,112 @@
|
||||
# Legislative Dashboard
|
||||
|
||||
This is the anonymous + logged-in dashboard for `cali_scraper`, implementing
|
||||
spec section 5 (design documents: `design/API_CONTRACT.md`,
|
||||
`design/DESIGN_DASHBOARD.md`). It is served by a stdlib `http.server`,
|
||||
backed by the SQLite relational DB and the tag-cloud JSON, and produces the
|
||||
frozen API-contract response shapes.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
scraper/dashboard/
|
||||
__init__.py # package docstring + version
|
||||
ranking.py # pure I/O-free float-to-top scoring (tested in isolation)
|
||||
context.py # RelevanceContext from user prefs
|
||||
data.py # read-only SQLite access, tag enrichment, committee lookup
|
||||
prefs.py # JSON prefs store (watch/topic toggles, dev auth)
|
||||
resolve.py # ZIP -> district two-tier resolution
|
||||
api.py # API_CONTRACT handlers (no sockets) + 6 empty states
|
||||
server.py # http.server ThreadingHTTPServer + static file routing
|
||||
dashboard_cli.py # guarded CLI entry via cli.py subcommand registration
|
||||
static/
|
||||
index.html # frontend shell
|
||||
style.css # visual system
|
||||
app.js # fetch client, anon/personal rendering, ZIP resolve, watch/opt-in
|
||||
```
|
||||
|
||||
### How it fits the CLI
|
||||
|
||||
`scraper/cli.py` wires in the dashboard with the same guarded pattern as
|
||||
`analyze_cli` and `relate_cli`:
|
||||
|
||||
```python
|
||||
try:
|
||||
from .dashboard import dashboard_cli as _dcli
|
||||
_dcli.register(sub)
|
||||
except Exception:
|
||||
pass
|
||||
```
|
||||
|
||||
A broken or missing dashboard module never breaks `parse`, `correlate`, or
|
||||
`analyze`.
|
||||
|
||||
### Run it
|
||||
|
||||
```bash
|
||||
python3 -m scraper dashboard --port 8733
|
||||
# browse http://127.0.0.1:8733/
|
||||
```
|
||||
|
||||
## API
|
||||
|
||||
| Method | Path | Auth | Purpose |
|
||||
|--------|------------------------|-------|--------------------------------------|
|
||||
| GET | `/api/v1/bills` | any | Bill list, personal / anonymous |
|
||||
| GET | `/api/v1/tags` | any | Tag cloud |
|
||||
| GET | `/api/v1/legislators` | any | Legislators |
|
||||
| GET | `/api/v1/districts` | any | Districts by chamber/zip |
|
||||
| GET | `/api/v1/resolve` | any | ZIP -> district resolution |
|
||||
| POST | `/api/v1/prefs` | auth | Update user prefs |
|
||||
| POST | `/api/v1/auth/register`| none | Dev auth placeholder |
|
||||
| POST | `/api/v1/prefs/watch` | auth | Add/remove watched legislator |
|
||||
| POST | `/api/v1/prefs/topics` | auth | Add/remove opted-in topic |
|
||||
|
||||
### Float-to-top ranking
|
||||
|
||||
Personal view applies a pure scoring function (`ranking.py`) over the bill
|
||||
list:
|
||||
|
||||
- **Watched-legislator match (author)** = +3
|
||||
- **Watched-legislator's committee match** = +2
|
||||
- **Opted-in tag match** = +2 × (number of matched tags)
|
||||
- **Home-chamber match** = +1 (optional)
|
||||
- Bills with score > 0 are moved to the front with a "Relevant to you"
|
||||
marker and a "Floated: ..." reason line
|
||||
- Zero-scored bills remain in their date-grouped order below
|
||||
|
||||
## Tests
|
||||
|
||||
```bash
|
||||
python3 -m pytest tests/test_dashboard_ranking.py tests/test_dashboard_api.py
|
||||
```
|
||||
|
||||
32 tests (12 ranking + 20 API), all passing. Pure ranking logic is tested in
|
||||
isolation; API handlers are tested with no sockets via the
|
||||
`handler(req) -> (status, body)` pattern; a live `ThreadingHTTPServer` smoke test
|
||||
confirms static + API endpoints serve.
|
||||
|
||||
## DASHBOARD.md
|
||||
|
||||
This document. The full UX specification is `design/DESIGN_DASHBOARD.md`
|
||||
and the frozen API contract is `design/API_CONTRACT.md`.
|
||||
|
||||
## Renders
|
||||
|
||||
- `out/dashboard_render_anon.png` — anonymous dashboard
|
||||
- `out/dashboard_render_personal.png` — personalized dashboard (ZIP 90210,
|
||||
watch Schiavo, opt-in budget topic)
|
||||
|
||||
Both are data-faithful snapshots from the live API, rendered via macOS
|
||||
`qlmanage` (no headless browser available in this environment; the browser
|
||||
remote-debugging permission popup is a user-level security dialog that
|
||||
cannot be auto-approved).
|
||||
|
||||
## Known limitations / next steps
|
||||
|
||||
- `out/tag_cloud.json` must exist for tags endpoint (118 tags as of
|
||||
2026-09-06).
|
||||
- `0 committee_members` roster — committee lookup degrades gracefully when
|
||||
no roster data is supplied.
|
||||
- Dev auth is a JSON-file local account, not production auth.
|
||||
- ZIP -> district uses a documented demo dataset until roster ETL lands.
|
||||
@@ -0,0 +1,45 @@
|
||||
"""``scraper.dashboard`` — spec §5 (UI build, task t_b38e9af2).
|
||||
|
||||
An anonymous + logged-in legislative dashboard served over the stdlib HTTP
|
||||
server (no heavy framework), backed by the correlation lane's relational DB
|
||||
(``bills`` / ``committees`` / ``members`` / ``committee_members`` /
|
||||
``bill_committees`` + the flat ``daily_file_records`` view) and the LLM lane's
|
||||
``bill_tags`` table.
|
||||
|
||||
Submodules
|
||||
----------
|
||||
* :mod:`scraper.dashboard.ranking` — pure float-to-top ranking + ``hearing_sort``
|
||||
+ ``matched[]`` transparency. No I/O.
|
||||
* :mod:`scraper.dashboard.context` — :class:`RelevanceContext` builder (prefs ->
|
||||
watched-author / watched-committee / opt-in
|
||||
tag sets).
|
||||
* :mod:`scraper.dashboard.data` — read-only DB access (bill events, tag cloud,
|
||||
legislators, districts).
|
||||
* :mod:`scraper.dashboard.prefs` — ``user_prefs`` + watch/topic persistence +
|
||||
a *local/minimal* auth placeholder (D1).
|
||||
* :mod:`scraper.dashboard.resolve` — ZIP/address -> district -> legislators
|
||||
(two-tier, D2), with confidence.
|
||||
* :mod:`scraper.dashboard.api` — request handlers producing the frozen
|
||||
API_CONTRACT.md shapes / errors / empty states.
|
||||
* :mod:`scraper.dashboard.server` — stdlib ``http.server`` + static frontend.
|
||||
* :mod:`scraper.dashboard.dashboard_cli` — guardedly-wired ``dashboard`` CLI
|
||||
subcommand.
|
||||
|
||||
FROZEN CONTRACTS honoured (task t_b38e9af2 / parent t_21cd58a9):
|
||||
|
||||
* Python 3.9.6, stdlib only — no new third-party dependency (the server uses
|
||||
``http.server`` / ``urllib``, mirroring the LLM lane's degradation posture).
|
||||
* Field names match ``design/API_CONTRACT.md`` verbatim.
|
||||
* The ranking model is DESIGN_DASHBOARD §4: a bill floats to the top iff
|
||||
``author ∩ watched_legislators ≢ ∅`` OR ``bill.tags ∩ opted_in_tags ≢ ∅``.
|
||||
* Never fabricate: a committee with no roster produces no committee membership
|
||||
(inherited from the correlation lane); the committee-of-watched path simply
|
||||
does not fire for such bills (degrades to author + tag only).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = ["ranking", "context", "data", "prefs", "resolve", "api", "server"]
|
||||
|
||||
#: Default personalization band size (DESIGN_DASHBOARD §4: "top N = 10").
|
||||
PERSONAL_BAND_N = 10
|
||||
@@ -0,0 +1,438 @@
|
||||
# noqa: E501
|
||||
"""API handlers producing the frozen API_CONTRACT.md shapes (spec §5).
|
||||
|
||||
Handler(req) -> (status, body). Driven by the stdlib HTTP server and by unit
|
||||
tests with no sockets. A Services bundle carries collaborators so handlers stay
|
||||
pure of instantiation. Errors per §6; empty results are 200 with empty lists.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from . import data as _data
|
||||
from . import ranking
|
||||
from . import resolve as _resolve
|
||||
from .data import (load_tag_cloud, list_events,
|
||||
load_legislators, committee_roster_for)
|
||||
from .ranking import hearing_sort, full_queue, personal_band
|
||||
from .context import from_prefs
|
||||
from .prefs import PrefsStore, LocalAuth
|
||||
|
||||
# error model (§6)
|
||||
|
||||
def api_error(code, message, status=400, field=None):
|
||||
err = {"error": {"code": code, "message": message}}
|
||||
if field:
|
||||
err["error"]["field"] = field
|
||||
return status, err
|
||||
|
||||
|
||||
# service bundle
|
||||
|
||||
class Services:
|
||||
|
||||
def __init__(self, db_path, prefs, auth):
|
||||
self.db_path = db_path
|
||||
self.prefs = prefs
|
||||
self.auth = auth
|
||||
|
||||
@classmethod
|
||||
def default(cls, db_path=None,
|
||||
prefs_path="out/prefs.json",
|
||||
auth_path="out/auth.json"):
|
||||
from .. import relational
|
||||
db = db_path or relational.DEFAULT_DB
|
||||
return cls(db_path=db,
|
||||
prefs=PrefsStore(prefs_path),
|
||||
auth=LocalAuth(auth_path))
|
||||
|
||||
def prefs_for_user(self, user_id):
|
||||
if not user_id:
|
||||
return {}
|
||||
return self.prefs.get(user_id)
|
||||
|
||||
def user_from_token(self, token):
|
||||
if not token:
|
||||
return None
|
||||
return self.auth.verify_token(token)
|
||||
|
||||
|
||||
# request
|
||||
|
||||
class Request:
|
||||
|
||||
def __init__(self, method, path, query=None, body=None,
|
||||
token=None, services=None, user_id=None):
|
||||
self.method = str(method).upper()
|
||||
self.path = str(path).lstrip("/")
|
||||
self.query = query or {}
|
||||
self.body = body or {}
|
||||
self.token = token
|
||||
self.services = services
|
||||
self.user_id = user_id
|
||||
|
||||
@property
|
||||
def svc(self):
|
||||
if self.services is None:
|
||||
raise RuntimeError("Request built without a Services bundle.")
|
||||
return self.services
|
||||
|
||||
def require_auth(self):
|
||||
uid = self.user_id or self.svc.user_from_token(self.token)
|
||||
if not uid:
|
||||
return None, api_error("not_authenticated",
|
||||
"A bearer token is required for this endpoint.",
|
||||
401)
|
||||
self.user_id = uid
|
||||
return uid, None
|
||||
|
||||
|
||||
# query helpers
|
||||
|
||||
def q_list(key, query):
|
||||
v = query.get(key)
|
||||
if v is None:
|
||||
return []
|
||||
if isinstance(v, list):
|
||||
return [str(x).strip() for x in v if str(x).strip()]
|
||||
return [s.strip() for s in str(v).split(",") if s.strip()]
|
||||
|
||||
|
||||
def _now_iso():
|
||||
import datetime
|
||||
return datetime.datetime.now(datetime.timezone.utc).strftime(
|
||||
"%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def _truthy(v):
|
||||
if v is None:
|
||||
return False
|
||||
if isinstance(v, str):
|
||||
return v.strip().lower() in ("1", "true", "yes", "on")
|
||||
return bool(v)
|
||||
|
||||
|
||||
# GET /api/v1/bills (§1/§2)
|
||||
|
||||
def handle_bills(req):
|
||||
q = req.query
|
||||
events = list_events(
|
||||
db_path=req.svc.db_path,
|
||||
upcoming=_truthy(q.get("upcoming")),
|
||||
from_date=q.get("from") or q.get("date_from"),
|
||||
to_date=q.get("to") or q.get("date_to"),
|
||||
chamber=q.get("chamber"),
|
||||
tags=q_list("tags", q),
|
||||
q=q.get("q"),
|
||||
author=q.get("author"),
|
||||
as_of=q.get("as_of"),
|
||||
)
|
||||
if _truthy(q.get("personal")):
|
||||
uid, err = req.require_auth()
|
||||
if err:
|
||||
return err
|
||||
prefs = req.svc.prefs_for_user(uid)
|
||||
roster = committee_roster_for(
|
||||
req.svc.db_path,
|
||||
set(prefs.get("watched_legislators") or []))
|
||||
ctx = from_prefs(prefs, committee_roster=roster)
|
||||
for ev in events:
|
||||
ev["relevance"] = ctx.score(ev)
|
||||
if _truthy(q.get("full")):
|
||||
events = full_queue(events)
|
||||
else:
|
||||
events = personal_band(
|
||||
events, int(q.get("band_n") or ranking.PERSONAL_BAND_N))
|
||||
session = _detect_session(events, req.svc.db_path)
|
||||
items = [_client_bill(e) for e in events]
|
||||
groups = {}
|
||||
for e in items:
|
||||
key = e.get("date") or "TBD"
|
||||
groups[key] = groups.get(key, 0) + 1
|
||||
chamber_avail = _chamber_availability(items, req.svc.db_path)
|
||||
body = {
|
||||
"session": session,
|
||||
"as_of": q.get("as_of") or _now_iso(),
|
||||
"chamber_availability": chamber_avail,
|
||||
"items": items,
|
||||
"groups": groups,
|
||||
}
|
||||
return 200, body
|
||||
|
||||
|
||||
def _client_bill(ev):
|
||||
out = {k: ev.get(k) for k in _data._EVENT_FIELDS}
|
||||
out["hearing_sort"] = hearing_sort(ev.get("hearing_time"))
|
||||
out["tags"] = ev.get("tags")
|
||||
if "relevance" in ev:
|
||||
out["relevance"] = ev["relevance"]
|
||||
return out
|
||||
|
||||
|
||||
def _detect_session(events, db_path):
|
||||
for e in events:
|
||||
if e.get("session"):
|
||||
return str(e["session"])
|
||||
return "2025-2026"
|
||||
|
||||
|
||||
def _chamber_availability(items, db_path):
|
||||
have = {str(e.get("chamber_origin") or "ASSEMBLY").upper() for e in items}
|
||||
if not have:
|
||||
for e in list_events(db_path=db_path):
|
||||
have.add(str(e.get("chamber_origin") or "ASSEMBLY").upper())
|
||||
out = {}
|
||||
for ch in ("ASSEMBLY", "SENATE"):
|
||||
present = ch in have or any(h == ch for h in have)
|
||||
out[ch] = "available" if present else "pending"
|
||||
return out
|
||||
|
||||
|
||||
# GET /api/v1/tags (§4.1)
|
||||
|
||||
def handle_tags(req):
|
||||
top = req.query.get("top")
|
||||
top = int(top) if top and str(top).isdigit() else None
|
||||
return 200, load_tag_cloud(req.svc.db_path, top=top)
|
||||
|
||||
|
||||
# GET /api/v1/legislators (§3.2 / §4.2)
|
||||
|
||||
def handle_legislators(req):
|
||||
q = req.query.get("q")
|
||||
if isinstance(q, list):
|
||||
q = q[0]
|
||||
return 200, {"items": load_legislators(req.svc.db_path, q=q)}
|
||||
|
||||
|
||||
# GET /api/v1/districts (§4.2)
|
||||
|
||||
def handle_districts(req):
|
||||
seen = {}
|
||||
for entry in _resolve.DEMO_ZIPS.values():
|
||||
for d in entry:
|
||||
seen[d["district_id"]] = d
|
||||
items = [
|
||||
{"district_id": d["district_id"], "type": d["type"],
|
||||
"number": d.get("number"),
|
||||
"assembly_member": d.get("assembly_member"),
|
||||
"senate_member": d.get("senate_member")}
|
||||
for d in seen.values()
|
||||
]
|
||||
return 200, {"items": sorted(items, key=lambda x: x["district_id"])}
|
||||
|
||||
|
||||
# POST /api/v1/districts/resolve (§3.1 — proposes, never persists)
|
||||
|
||||
def handle_resolve(req):
|
||||
b = req.body or {}
|
||||
zip_ = b.get("zip")
|
||||
address = b.get("address")
|
||||
district = b.get("district")
|
||||
if not zip_ and not address and not district:
|
||||
return api_error("invalid_zip",
|
||||
"Provide one of: zip, address, or district.",
|
||||
400, field="zip")
|
||||
result = _resolve.resolve(zip=zip_, address=address, district=district)
|
||||
conf = result["confidence"]
|
||||
if conf == "out-of-state":
|
||||
return api_error("out_of_state",
|
||||
"That input is not in California.",
|
||||
400, field="zip")
|
||||
if conf == "zero":
|
||||
return api_error("district_not_found",
|
||||
"Could not match the input to a California district.",
|
||||
404, field="zip" if zip_ else "address")
|
||||
if conf == "ambiguous":
|
||||
out = dict(result)
|
||||
out["note"] = ("Multiple districts match; confirm one before "
|
||||
"watching (never auto-picked).")
|
||||
return 200, out
|
||||
return 200, result
|
||||
|
||||
|
||||
# /api/v1/prefs (§3.3)
|
||||
|
||||
def handle_get_prefs(req):
|
||||
uid, err = req.require_auth()
|
||||
if err:
|
||||
return err
|
||||
prefs = req.svc.prefs_for_user(uid)
|
||||
if not prefs:
|
||||
return api_error("prefs_not_set",
|
||||
"No preferences stored; set a ZIP / watch / topics.",
|
||||
404)
|
||||
prefs = dict(prefs)
|
||||
prefs["user_id"] = uid
|
||||
prefs["updated_at"] = prefs.get("updated_at") or _now_iso()
|
||||
return 200, prefs
|
||||
|
||||
|
||||
def handle_put_prefs(req):
|
||||
uid, err = req.require_auth()
|
||||
if err:
|
||||
return err
|
||||
b = dict(req.body or {})
|
||||
allowed = {"zip", "address", "home_chamber", "districts",
|
||||
"watched_legislators", "opted_in_tags"}
|
||||
valid_tags = _valid_tag_set(req)
|
||||
known_legs = {x["legislator_id"] for x in
|
||||
load_legislators(req.svc.db_path)}
|
||||
for k, v in b.items():
|
||||
if k not in allowed:
|
||||
return api_error("invalid_config",
|
||||
"Unknown prefs field: %s" % k, 400, field=k)
|
||||
if k == "opted_in_tags":
|
||||
for t in v:
|
||||
if str(t).lower() not in valid_tags:
|
||||
return api_error("tag_unknown",
|
||||
"Unknown topic: %s" % t, 400,
|
||||
field="opted_in_tags")
|
||||
if k == "watched_legislators":
|
||||
for w in v:
|
||||
if known_legs and w not in known_legs:
|
||||
return api_error("legislator_unknown",
|
||||
"Unknown legislator: %s" % w, 400,
|
||||
field="watched_legislators")
|
||||
clean = {k: b[k] for k in b if k in allowed}
|
||||
clean["updated_at"] = _now_iso()
|
||||
updated = req.svc.prefs.put(uid, clean)
|
||||
updated["user_id"] = uid
|
||||
return 200, updated
|
||||
|
||||
|
||||
def _valid_tag_set(req):
|
||||
tags = load_tag_cloud(req.svc.db_path)["tags"]
|
||||
return {t["tag_id"].split(":", 1)[-1].lower() for t in tags}
|
||||
|
||||
|
||||
# watch / topic toggles (§3.4 / §3.5 — idempotent)
|
||||
|
||||
def handle_add_watch(req):
|
||||
uid, err = req.require_auth()
|
||||
if err:
|
||||
return err
|
||||
b = req.body or {}
|
||||
lid = b.get("legislator_id") or b.get("id")
|
||||
if not lid:
|
||||
return api_error("invalid_legislator",
|
||||
"legislator_id is required.", 400,
|
||||
field="legislator_id")
|
||||
if not _known_legislator(req, lid):
|
||||
return api_error("legislator_unknown",
|
||||
"Unknown legislator: %s" % lid, 400, field="id")
|
||||
prefs = req.svc.prefs.add_watch(uid, lid)
|
||||
prefs["user_id"] = uid
|
||||
return 201, prefs
|
||||
|
||||
|
||||
def handle_remove_watch(req, legislator_id):
|
||||
uid, err = req.require_auth()
|
||||
if err:
|
||||
return err
|
||||
prefs = req.svc.prefs.remove_watch(uid, legislator_id)
|
||||
prefs["user_id"] = uid
|
||||
return 200, prefs
|
||||
|
||||
|
||||
def handle_add_topic(req):
|
||||
uid, err = req.require_auth()
|
||||
if err:
|
||||
return err
|
||||
b = req.body or {}
|
||||
tag = b.get("tag") or b.get("tag_id") or b.get("id")
|
||||
if not tag:
|
||||
return api_error("invalid_tag", "tag is required.", 400, field="tag")
|
||||
if str(tag).lower() not in _valid_tag_set(req):
|
||||
return api_error("tag_unknown",
|
||||
"Unknown topic: %s" % tag, 400, field="tag")
|
||||
prefs = req.svc.prefs.add_topic(uid, str(tag).lower())
|
||||
prefs["user_id"] = uid
|
||||
return 201, prefs
|
||||
|
||||
|
||||
def handle_remove_topic(req, tag):
|
||||
uid, err = req.require_auth()
|
||||
if err:
|
||||
return err
|
||||
prefs = req.svc.prefs.remove_topic(uid, tag)
|
||||
prefs["user_id"] = uid
|
||||
return 200, prefs
|
||||
|
||||
|
||||
def _known_legislator(req, lid):
|
||||
return any(x.get("legislator_id") == lid
|
||||
for x in load_legislators(req.svc.db_path))
|
||||
|
||||
|
||||
# Minimal local-account auth (D1a default — dev placeholder)
|
||||
|
||||
def handle_register(req):
|
||||
email = (req.body or {}).get("email", "").strip().lower()
|
||||
if not email or "@" not in email:
|
||||
return api_error("invalid_email",
|
||||
"A valid email is required.", 400, field="email")
|
||||
rec = req.svc.auth.register(email)
|
||||
req.svc.prefs.put(rec["user_id"], {"email": rec["email"]})
|
||||
return 201, rec
|
||||
|
||||
|
||||
def handle_whoami(req):
|
||||
uid = req.user_id or req.svc.user_from_token(req.token)
|
||||
if not uid:
|
||||
return {"user_id": None}
|
||||
return {"user_id": uid, "prefs": req.svc.prefs_for_user(uid)}
|
||||
|
||||
|
||||
# router
|
||||
|
||||
def _strip(req):
|
||||
p = req.path
|
||||
base = "api/v1/"
|
||||
return p[len(base):] if p.startswith(base) else p
|
||||
|
||||
|
||||
def run(req):
|
||||
m = req.method
|
||||
p2 = _strip(req)
|
||||
if m == "POST" and p2 == "auth/register":
|
||||
return handle_register(req)
|
||||
if m == "GET" and p2 == "whoami":
|
||||
return 200, handle_whoami(req)
|
||||
if m == "GET" and p2 == "bills":
|
||||
return handle_bills(req)
|
||||
if m == "GET" and p2 == "tags":
|
||||
return handle_tags(req)
|
||||
if m == "GET" and p2 == "legislators":
|
||||
return handle_legislators(req)
|
||||
if m == "GET" and p2 == "districts":
|
||||
return handle_districts(req)
|
||||
if m == "POST" and p2 == "districts/resolve":
|
||||
return handle_resolve(req)
|
||||
if m == "GET" and p2 == "prefs":
|
||||
return handle_get_prefs(req)
|
||||
if m == "PUT" and p2 == "prefs":
|
||||
return handle_put_prefs(req)
|
||||
if m == "POST" and p2 == "prefs/watch":
|
||||
return handle_add_watch(req)
|
||||
if m == "DELETE" and p2.startswith("prefs/watch/"):
|
||||
return handle_remove_watch(req, p2.split("/", 3)[2])
|
||||
if m == "POST" and p2 == "prefs/topics":
|
||||
return handle_add_topic(req)
|
||||
if m == "DELETE" and p2.startswith("prefs/topics/"):
|
||||
return handle_remove_topic(req, p2.split("/", 3)[2])
|
||||
if m == "GET" and p2 == "":
|
||||
return 200, {"endpoints": [
|
||||
"GET /api/v1/bills?upcoming=1&personal=1",
|
||||
"GET /api/v1/tags",
|
||||
"GET /api/v1/legislators?q=",
|
||||
"GET /api/v1/districts",
|
||||
"POST /api/v1/districts/resolve",
|
||||
"GET/PUT /api/v1/prefs",
|
||||
"POST/DELETE /api/v1/prefs/watch[/:id]",
|
||||
"POST/DELETE /api/v1/prefs/topics[/:id]",
|
||||
"POST /api/v1/auth/register",
|
||||
"GET /api/v1/whoami",
|
||||
]}
|
||||
return api_error("not_found",
|
||||
"No such endpoint: %s %s" % (m, req.path), 404)
|
||||
@@ -0,0 +1,115 @@
|
||||
"""Build a relevance context from a user's preferences (spec §5, DESIGN §4/§10).
|
||||
|
||||
The :class:`RelevanceContext` is the pure input to :func:`ranking.score_bill`.
|
||||
It is built from a user's stored preferences plus (optionally) the correlation
|
||||
lane's committee<->member join, so the "committee of a watched legislator"
|
||||
float path fires *only* when a real roster-backed committee exists for the
|
||||
bill (never fabricated).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Optional, Set
|
||||
|
||||
from .ranking import Weights, DEFAULT_WEIGHTS
|
||||
|
||||
|
||||
def _slug(name):
|
||||
if not name:
|
||||
return ""
|
||||
return re.sub(r"[^a-z0-9]+", "-", name.strip().lower()).strip("-") or "unknown"
|
||||
|
||||
|
||||
@dataclass
|
||||
class RelevanceContext:
|
||||
"""Pure input to the ranking for one logged-in user.
|
||||
|
||||
Attributes
|
||||
----------
|
||||
watched_authors: lower-cased slugs of the primary author of each watched
|
||||
legislator (``author ∩ watched_legislators``).
|
||||
watched_committees: slugs of committees served by a watched legislator
|
||||
(committee-of-watched-legislator path). Built by :func:`from_prefs`
|
||||
when a committee-roster lookup is supplied; otherwise empty.
|
||||
opted_in_tags: lower-cased tag strings (``tags ∩ opted_in_tags``).
|
||||
home_chamber: optional chamber hint for the +1 score term.
|
||||
weights: scoring weights (tunable — Open Decision D3).
|
||||
"""
|
||||
|
||||
watched_authors: Set[str] = field(default_factory=set)
|
||||
watched_committees: Set[str] = field(default_factory=set)
|
||||
opted_in_tags: Set[str] = field(default_factory=set)
|
||||
home_chamber: Optional[str] = None
|
||||
weights: Weights = field(default_factory=lambda: DEFAULT_WEIGHTS)
|
||||
|
||||
def score(self, bill: Dict) -> Dict:
|
||||
"""Score one bill against this context."""
|
||||
from .ranking import score_bill
|
||||
return score_bill(
|
||||
bill,
|
||||
self.watched_authors,
|
||||
self.watched_committees,
|
||||
self.opted_in_tags,
|
||||
home_chamber=self.home_chamber,
|
||||
weights=self.weights,
|
||||
)
|
||||
|
||||
@property
|
||||
def is_personalised(self) -> bool:
|
||||
"""True when the user has set a watch or opt-in (DESIGN §6.3 gate)."""
|
||||
return bool(self.watched_authors or self.watched_committees
|
||||
or self.opted_in_tags)
|
||||
|
||||
|
||||
def from_prefs(prefs,
|
||||
committee_roster=None,
|
||||
weights=None) -> RelevanceContext:
|
||||
"""Build a :class:`RelevanceContext` from a stored preferences blob.
|
||||
|
||||
``prefs`` matches API_CONTRACT §3.3::
|
||||
|
||||
{ "watched_legislators": ["L-asm-14", ...],
|
||||
"opted_in_tags": ["waste", "health"],
|
||||
"home_chamber": "ASSEMBLY",
|
||||
"districts": [...] }
|
||||
|
||||
``committee_roster`` optionally maps a ``legislator_id`` -> list of
|
||||
committee slugs that legislator chairs/serves in, so the
|
||||
committee-of-watched path can be derived. It is looked up against the
|
||||
correlation lane by the caller (:mod:`scraper.dashboard.data`); when
|
||||
absent the committee path is empty (honest degradation — the correlation
|
||||
lane found no roster, and we never fabricate membership).
|
||||
"""
|
||||
prefs = prefs or {}
|
||||
ctx = RelevanceContext(weights=weights or DEFAULT_WEIGHTS)
|
||||
|
||||
# Key the author-matching set on both the id-as-given and its slug so a
|
||||
# bill whose `author` column is the *name* still matches a name-based watch.
|
||||
for lid in prefs.get("watched_legislators") or []:
|
||||
s = _slug(lid)
|
||||
if s and s != "unknown":
|
||||
ctx.watched_authors.add(s)
|
||||
if isinstance(lid, str) and lid.strip():
|
||||
ctx.watched_authors.add(lid.strip().lower())
|
||||
|
||||
for tag in prefs.get("opted_in_tags") or []:
|
||||
t = str(tag).strip().lower()
|
||||
if t:
|
||||
ctx.opted_in_tags.add(t)
|
||||
|
||||
hc = prefs.get("home_chamber")
|
||||
if hc:
|
||||
ctx.home_chamber = str(hc).lower()
|
||||
|
||||
# Resolve committee-of-watched legs from the roster map (never fabricated).
|
||||
if committee_roster:
|
||||
for lid in prefs.get("watched_legislators") or []:
|
||||
committees = committee_roster.get(lid)
|
||||
if committees:
|
||||
for c in committees:
|
||||
cs = _slug(c)
|
||||
if cs and cs != "unknown":
|
||||
ctx.watched_committees.add(cs)
|
||||
return ctx
|
||||
@@ -0,0 +1,84 @@
|
||||
"""`dashboard` CLI subcommand (spec §5, task t_b38e9af2).
|
||||
|
||||
Wires the dashboard server into the shared `scraper/cli.py` ADDITIVELY +
|
||||
DEFENSIVELY, mirroring parser/correlate/analyze (the guarded `register()`
|
||||
pattern): `cli.py` calls `register(sub)` inside a try/except and adds its
|
||||
handler without touching any sibling handler. The `dashboard` name does not
|
||||
collide.
|
||||
|
||||
This is the CLI/app entry point per the task contract. The server itself lives
|
||||
in `scraper/dashboard/server.py`; this module just parses flags and calls
|
||||
`server.serve(...)`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from .. import relational
|
||||
from . import server as _server
|
||||
|
||||
log = logging.getLogger("scraper.dashboard_cli")
|
||||
|
||||
|
||||
def _already_registered(sub, name: str) -> bool:
|
||||
actions = getattr(sub, "_choices_actions", [])
|
||||
return any(getattr(a, "name", None) == name for a in actions)
|
||||
|
||||
|
||||
def register(sub) -> None:
|
||||
"""Register the `dashboard` subcommand (guarded — skip if a sibling owns it)."""
|
||||
if _already_registered(sub, "dashboard"):
|
||||
return
|
||||
pd = sub.add_parser(
|
||||
"dashboard",
|
||||
help="serve the legislative dashboard (API + static UI)",
|
||||
description=("Serve the legislative dashboard. Reads the relational DB "
|
||||
"(bills/committees/members + bill_tags) and serves the "
|
||||
"API_CONTRACT.md endpoints + the static frontend."),
|
||||
)
|
||||
pd.add_argument("--host", default="127.0.0.1",
|
||||
help="bind host (default 127.0.0.1)")
|
||||
pd.add_argument("--port", type=int, default=8000,
|
||||
help="bind port (default 8000)")
|
||||
pd.add_argument("--db", default=relational.DEFAULT_DB,
|
||||
help="SQLite path (default db/daily_file.sqlite)")
|
||||
pd.add_argument("--prefs", default="out/prefs.json",
|
||||
help="preferences JSON file (default out/prefs.json)")
|
||||
pd.add_argument("--auth", default="out/auth.json",
|
||||
help="local-auth JSON file (default out/auth.json)")
|
||||
pd.add_argument("--no-static", dest="static", action="store_false",
|
||||
help="API only, no static frontend")
|
||||
pd.add_argument("--log", action="store_true",
|
||||
help="log HTTP requests to stderr")
|
||||
pd.set_defaults(func=handle_dashboard)
|
||||
|
||||
|
||||
def handle_dashboard(args) -> int:
|
||||
"""CLI handler for the `dashboard` subcommand."""
|
||||
return _server.serve(
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
db_path=args.db,
|
||||
prefs_path=args.prefs,
|
||||
auth_path=args.auth,
|
||||
serve_static=getattr(args, "static", True),
|
||||
log_on=getattr(args, "log", False),
|
||||
)
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
"""Direct invocation: `python -m scraper.dashboard_cli`."""
|
||||
import argparse
|
||||
p = argparse.ArgumentParser(
|
||||
prog="scraper.dashboard_cli",
|
||||
description="Serve the legislative dashboard.")
|
||||
sub = p.add_subparsers(dest="command", required=True)
|
||||
register(sub)
|
||||
args = p.parse_args(argv)
|
||||
return args.func(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,335 @@
|
||||
"""Read-only access to the dashboard's data sources (spec §5).
|
||||
|
||||
The dashboard never mutates the relational DB. It reads:
|
||||
|
||||
* the flat ``v_daily_file_records`` view (or ``daily_file_records``) for the
|
||||
upcoming-events queue — one bill event per ``(date, bill_id)``, enriched
|
||||
with per-bill ``tags`` from the LLM lane's ``bill_tags`` table;
|
||||
* ``bill_tags`` for the tag cloud (§4.1);
|
||||
* ``members`` / ``committees`` / ``bill_committees`` / ``committee_members`` for
|
||||
the legislators reference data (§3.2 / §4.2) and the
|
||||
committee-of-watched-legislator float path.
|
||||
|
||||
Everything here returns plain JSON-serialisable dicts so the API layer can
|
||||
ship them verbatim into the API_CONTRACT shapes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
from typing import Any, Dict, List, Optional, Sequence
|
||||
|
||||
from .. import relational
|
||||
|
||||
FLAT_TABLE = relational.FLAT_TABLE
|
||||
VIEW_NAME = "v_" + FLAT_TABLE
|
||||
|
||||
# Fields a bill event carries into the API_CONTRACT §1 Bill shape.
|
||||
# `tags` is injected separately (from bill_tags), not a stored column.
|
||||
_EVENT_FIELDS = (
|
||||
"bill_id", "bill_number", "chamber_origin", "author", "short_title",
|
||||
"description", "committee", "hearing_time", "location", "date",
|
||||
"session", "item_number", "supplement", "source",
|
||||
)
|
||||
|
||||
|
||||
def _slug(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
return re.sub(r"[^a-z0-9]+", "-", name.strip().lower()).strip("-") or "unknown"
|
||||
|
||||
|
||||
def _title_case(s: str) -> str:
|
||||
return " ".join(w[:1].upper() + w[1:] for w in s.split() if w)
|
||||
|
||||
|
||||
def _table_exists(conn: sqlite3.Connection, name: str) -> bool:
|
||||
cur = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (name,))
|
||||
return cur.fetchone() is not None
|
||||
|
||||
|
||||
def _view_exists(conn: sqlite3.Connection, name: str) -> bool:
|
||||
cur = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='view' AND name=?", (name,))
|
||||
return cur.fetchone() is not None
|
||||
|
||||
|
||||
def _flat_source(conn: sqlite3.Connection) -> str:
|
||||
"""Prefer the flat-preserving view (the QA contract); fall back to the table."""
|
||||
return VIEW_NAME if _view_exists(conn, VIEW_NAME) else FLAT_TABLE
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tags (LLM lane: bill_tags table, §4.1)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def load_bill_tags(db_path: str = relational.DEFAULT_DB) -> Dict[str, List[str]]:
|
||||
"""Map ``bill_id`` -> [tag, ...] from the ``bill_tags`` table.
|
||||
|
||||
Absent/empty table -> ``{}`` so a bill with no analysis degrades to
|
||||
``tags: []`` (DESIGN §6.6 — no chips, ranking by legislator only).
|
||||
"""
|
||||
tags: Dict[str, List[str]] = {}
|
||||
if not os.path.exists(db_path):
|
||||
return tags
|
||||
conn = relational.connect(db_path, foreign_keys=False)
|
||||
try:
|
||||
if not _table_exists(conn, "bill_tags"):
|
||||
return tags
|
||||
rows = conn.execute(
|
||||
"SELECT bill_id, tag FROM bill_tags "
|
||||
"WHERE bill_id IS NOT NULL AND tag IS NOT NULL AND tag != '' "
|
||||
"ORDER BY bill_id, tag").fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
for r in rows:
|
||||
bid = r["bill_id"]
|
||||
tag = r["tag"]
|
||||
bucket = tags.setdefault(bid, [])
|
||||
if tag not in bucket:
|
||||
bucket.append(tag)
|
||||
return tags
|
||||
|
||||
|
||||
def load_tag_cloud(db_path: str = relational.DEFAULT_DB,
|
||||
as_of: Optional[str] = None,
|
||||
top: Optional[int] = None) -> Dict[str, Any]:
|
||||
"""The /tags cloud (§4.1): ``[{tag_id, label, count}, ...]`` sorted desc.
|
||||
|
||||
Count = number of bills carrying the tag. May be sparse early (LLM lane
|
||||
warming) — the UI shows a "filling in" hint, never a 0-count chip.
|
||||
"""
|
||||
import datetime
|
||||
conn = relational.connect(db_path, foreign_keys=False)
|
||||
try:
|
||||
if not _table_exists(conn, "bill_tags"):
|
||||
data = []
|
||||
else:
|
||||
data = conn.execute(
|
||||
"SELECT tag, COUNT(DISTINCT bill_id) AS n FROM bill_tags "
|
||||
"WHERE tag IS NOT NULL AND tag != '' GROUP BY tag "
|
||||
"ORDER BY n DESC, tag").fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
out = []
|
||||
for r in data:
|
||||
out.append({
|
||||
"tag_id": "tag:" + str(r["tag"]),
|
||||
"label": _title_case(str(r["tag"])),
|
||||
"count": int(r["n"]),
|
||||
})
|
||||
if top:
|
||||
out = out[:top]
|
||||
return {
|
||||
"as_of": as_of or _now_iso(),
|
||||
"tags": out,
|
||||
}
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
import datetime
|
||||
return datetime.datetime.now(datetime.timezone.utc).strftime(
|
||||
"%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Upcoming-events queue (the daily-file calendar, §2 / §5 "upcoming")
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _in_window(date: Optional[str], frm: Optional[str], to: Optional[str],
|
||||
as_of: Optional[str]) -> bool:
|
||||
"""API_CONTRACT §5 'upcoming' gate on event_date (= date here).
|
||||
|
||||
* a T-1 day (date < as_of) is EXCLUDED when an as_of floor is set;
|
||||
* 'this week'/'this month' are from/to windows over event_date.
|
||||
Dates are ISO ``YYYY-MM-DD`` so string compare == chronological.
|
||||
"""
|
||||
if not date:
|
||||
return not frm and not to
|
||||
if frm and date < frm:
|
||||
return False
|
||||
if to and date > to:
|
||||
return False
|
||||
if as_of and date < as_of:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def list_events(db_path: str = relational.DEFAULT_DB,
|
||||
upcoming: bool = False,
|
||||
from_date: Optional[str] = None,
|
||||
to_date: Optional[str] = None,
|
||||
chamber: Optional[str] = None,
|
||||
tags: Optional[Sequence[str]] = None,
|
||||
q: Optional[str] = None,
|
||||
author: Optional[str] = None,
|
||||
as_of: Optional[str] = None) -> List[Dict[str, Any]]:
|
||||
"""Build the bill-event queue, one event per ``(date, bill_id)``.
|
||||
|
||||
* ``upcoming=True`` + ``as_of`` -> keep only future-dated events
|
||||
(API_CONTRACT §5). When the caller omits ``as_of`` the default is the
|
||||
latest date present, so a demo on a single historical day still shows its
|
||||
queue; pin ``as_of`` for a true "future-of-now" view.
|
||||
* dedupe *within* a date by ``bill_id`` (multiple agenda items the same day
|
||||
collapse to one); across dates they are distinct events.
|
||||
* enriched with per-bill ``tags`` from the LLM lane.
|
||||
* optional filters: chamber, tags (any-match), free-text ``q``, author.
|
||||
|
||||
Returns list of dicts matching API_CONTRACT §1 **without** the ``relevance``
|
||||
block (the API layer adds that for logged-in views).
|
||||
"""
|
||||
conn = relational.connect(db_path, foreign_keys=False)
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM " + _flat_source(conn) +
|
||||
" WHERE bill_id IS NOT NULL AND bill_id != '' "
|
||||
"ORDER BY date, house, bill_id, committee, id").fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
flat = [dict(r) for r in rows]
|
||||
|
||||
bill_tags = load_bill_tags(db_path)
|
||||
tag_filter = {t.strip().lower() for t in (tags or []) if str(t).strip()}
|
||||
ql = (q or "").strip().lower()
|
||||
al = (author or "").strip().lower()
|
||||
|
||||
if as_of is None:
|
||||
dates = sorted({r["date"] for r in flat if r.get("date")})
|
||||
as_of = dates[-1] if dates else None
|
||||
|
||||
# Collapse to one event per (date, bill_id); prefer a row carrying a hearing.
|
||||
seen: Dict[tuple, Dict[str, Any]] = {}
|
||||
order: List[tuple] = []
|
||||
for r in flat:
|
||||
key = (r.get("date"), r.get("bill_id"))
|
||||
if key in seen:
|
||||
if not seen[key].get("_has_time") and r.get("hearing_time"):
|
||||
seen[key] = r
|
||||
continue
|
||||
r["_has_time"] = bool(r.get("hearing_time"))
|
||||
seen[key] = r
|
||||
order.append(key)
|
||||
|
||||
events = []
|
||||
for key in order:
|
||||
r = seen[key]
|
||||
origin = str(r.get("chamber_origin") or r.get("house") or "").lower()
|
||||
if chamber and origin != chamber.lower():
|
||||
continue
|
||||
if not _in_window(r.get("date"), from_date, to_date, as_of):
|
||||
continue
|
||||
btags = bill_tags.get(r["bill_id"], [])
|
||||
if tag_filter and not (tag_filter & {t.lower() for t in btags}):
|
||||
continue
|
||||
if ql:
|
||||
hay = " ".join(str(r.get(f) or "") for f in
|
||||
("short_title", "description", "bill_number",
|
||||
"author", "committee")).lower()
|
||||
if ql not in hay:
|
||||
continue
|
||||
if al and al not in str(r.get("author") or "").lower():
|
||||
continue
|
||||
ev = {f: r.get(f) for f in _EVENT_FIELDS}
|
||||
for f in ("hearing_time", "location", "committee", "short_title",
|
||||
"author", "item_number", "supplement", "date", "session"):
|
||||
if ev.get(f) == "":
|
||||
ev[f] = None
|
||||
ev["tags"] = btags if btags else None
|
||||
ev["source"] = r.get("source") or "ASSEMBLY_DAILY_FILE"
|
||||
events.append(ev)
|
||||
return events
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Legislators + districts reference data (§3.2 / §4.2)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def load_legislators(db_path: str = relational.DEFAULT_DB,
|
||||
q: Optional[str] = None) -> List[Dict[str, Any]]:
|
||||
"""Browse legislators (members table) with their committees + domains.
|
||||
|
||||
``domains`` = the domain tags of the committees a member sits on
|
||||
(committee -> its bills' tags). Degrades to an empty list when the
|
||||
correlation lane has no roster (no committee_members).
|
||||
"""
|
||||
conn = relational.connect(db_path, foreign_keys=False)
|
||||
try:
|
||||
members = [dict(r) for r in conn.execute(
|
||||
"SELECT name_norm, full_name, chamber, kind "
|
||||
"FROM members ORDER BY full_name, chamber").fetchall()]
|
||||
rows = conn.execute(
|
||||
"SELECT cm.member, c.name_raw AS committee "
|
||||
"FROM committee_members cm "
|
||||
"JOIN committees c ON c.name_key=cm.committee "
|
||||
"AND c.chamber=cm.chamber AND c.session=cm.session "
|
||||
"ORDER BY cm.member, c.name_raw").fetchall()
|
||||
served: Dict[str, List[str]] = {}
|
||||
for r in rows:
|
||||
served.setdefault(r["member"], []).append(r["committee"])
|
||||
|
||||
if _table_exists(conn, "bill_tags"):
|
||||
cmt_rows = conn.execute(
|
||||
"SELECT c.name_raw AS cn, t.tag FROM bill_committees bc "
|
||||
"JOIN committees c ON c.name_key=bc.committee "
|
||||
"AND c.chamber=bc.chamber AND c.session=bc.session "
|
||||
"JOIN bill_tags t ON t.bill_id=bc.bill_id").fetchall()
|
||||
cmt_tags: Dict[str, set] = {}
|
||||
for r in cmt_rows:
|
||||
cmt_tags.setdefault(r["cn"], set()).add(r["tag"])
|
||||
else:
|
||||
cmt_tags = {}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
ql = (q or "").strip().lower()
|
||||
out = []
|
||||
for m in members:
|
||||
slug = m["name_norm"]
|
||||
committees = served.get(slug, [])
|
||||
domains = sorted({t for cn in committees for t in cmt_tags.get(cn, set())})
|
||||
if ql and ql not in str(m.get("full_name") or slug).lower():
|
||||
continue
|
||||
out.append({
|
||||
"legislator_id": slug,
|
||||
"name": m.get("full_name") or slug.title(),
|
||||
"party": None,
|
||||
"chamber": (m.get("chamber") or "").upper(),
|
||||
"district": None,
|
||||
"committees": committees,
|
||||
"domains": domains,
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def committee_roster_for(db_path: str = relational.DEFAULT_DB,
|
||||
legislator_ids=None) -> Dict[str, List[str]]:
|
||||
"""Map a watched legislator (by its ``member`` slug) to the committee
|
||||
name_keys that legislator is known to serve on, via the correlation lane's
|
||||
``committee_members`` join.
|
||||
|
||||
Used by the ranking to fire the ``committee-of-watched-legislator`` float
|
||||
path. When the correlation lane found no roster (``committee_members``
|
||||
empty — the honest default), this returns ``{}`` and that path does not
|
||||
fire, so nothing is fabricated.
|
||||
"""
|
||||
conn = relational.connect(db_path, foreign_keys=False)
|
||||
try:
|
||||
if not _table_exists(conn, "committee_members"):
|
||||
return {}
|
||||
rows = conn.execute(
|
||||
"SELECT DISTINCT cm.member AS m, cm.committee AS ck "
|
||||
"FROM committee_members cm").fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
roster: Dict[str, List[str]] = {}
|
||||
for r in rows:
|
||||
m = r["m"]
|
||||
if legislator_ids is None or m in legislator_ids:
|
||||
bucket = roster.setdefault(m, [])
|
||||
if r["ck"] not in bucket:
|
||||
bucket.append(r["ck"])
|
||||
return roster
|
||||
@@ -0,0 +1,181 @@
|
||||
"""Personalization store + watch/topic toggles + a minimal dev auth (spec §5).
|
||||
|
||||
The persistence is a small JSON file (documented in DASHBOARD.md) holding one
|
||||
preferences blob per user_id, matching API_CONTRACT section 3.3. It is
|
||||
intentionally simple and swappable: replacing PrefsStore with a DB-backed store
|
||||
is a local change; the preferences *shape* (and therefore the whole contract)
|
||||
is auth-method-independent, which is the point of Open Decision D1.
|
||||
|
||||
A minimal LOCAL account (LocalAuth) is provided as the D1a default so the
|
||||
dashboard is runnable end-to-end without an IdP. It is explicitly a development
|
||||
placeholder (no password hashing, single-host JSON): swap it for a real auth
|
||||
service in production. Auth is NOT modelled in the data contract.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Preferences store (API_CONTRACT section 3.3)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class PrefsStore:
|
||||
"""File-backed per-user preferences. One blob per user_id.
|
||||
|
||||
On-disk shape is `{"u_001": {...}, ...}` so the store is human-readable and
|
||||
diff-stable. All mutating ops are idempotent: re-adding an existing
|
||||
watch/topic is a no-op, and removing an absent one does not error.
|
||||
"""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
self._data = self._load()
|
||||
|
||||
# -- persistence --------------------------------------------------
|
||||
def _load(self):
|
||||
if not os.path.exists(self.path):
|
||||
return {}
|
||||
try:
|
||||
with open(self.path, encoding="utf-8") as fh:
|
||||
return json.load(fh) or {}
|
||||
except (ValueError, OSError):
|
||||
return {}
|
||||
|
||||
def _save(self):
|
||||
os.makedirs(os.path.dirname(os.path.abspath(self.path)) or ".",
|
||||
exist_ok=True)
|
||||
tmp = self.path + ".tmp"
|
||||
with open(tmp, "w", encoding="utf-8") as fh:
|
||||
json.dump(self._data, fh, indent=2, sort_keys=True)
|
||||
os.replace(tmp, self.path)
|
||||
|
||||
# -- reads --------------------------------------------------------
|
||||
def get(self, user_id):
|
||||
return self._data.get(user_id) or {}
|
||||
|
||||
def has(self, user_id):
|
||||
return user_id in self._data and bool(self._data[user_id])
|
||||
|
||||
def list_users(self):
|
||||
return sorted(k for k in self._data.keys() if self._data[k])
|
||||
|
||||
# -- writes -------------------------------------------------------
|
||||
def put(self, user_id, prefs):
|
||||
merged = self.get(user_id)
|
||||
merged.update(prefs or {})
|
||||
self._clean(merged)
|
||||
self._data[user_id] = merged
|
||||
self._save()
|
||||
return merged
|
||||
|
||||
def add_watch(self, user_id, legislator_id):
|
||||
prefs = self.get(user_id)
|
||||
watch = list(prefs.get("watched_legislators") or [])
|
||||
if legislator_id not in watch:
|
||||
watch.append(legislator_id)
|
||||
return self._set(user_id, "watched_legislators", watch)
|
||||
|
||||
def remove_watch(self, user_id, legislator_id):
|
||||
prefs = self.get(user_id)
|
||||
watch = [x for x in (prefs.get("watched_legislators") or [])
|
||||
if x != legislator_id]
|
||||
return self._set(user_id, "watched_legislators", watch)
|
||||
|
||||
def add_topic(self, user_id, tag):
|
||||
prefs = self.get(user_id)
|
||||
topics = list(prefs.get("opted_in_tags") or [])
|
||||
if tag not in topics:
|
||||
topics.append(tag)
|
||||
return self._set(user_id, "opted_in_tags", topics)
|
||||
|
||||
def remove_topic(self, user_id, tag):
|
||||
prefs = self.get(user_id)
|
||||
topics = [t for t in (prefs.get("opted_in_tags") or []) if t != tag]
|
||||
return self._set(user_id, "opted_in_tags", topics)
|
||||
|
||||
def _set(self, user_id, key, value):
|
||||
prefs = self.get(user_id)
|
||||
prefs[key] = value
|
||||
self._clean(prefs)
|
||||
self._data[user_id] = prefs
|
||||
self._save()
|
||||
return prefs
|
||||
|
||||
@staticmethod
|
||||
def _clean(prefs):
|
||||
for key in ("watched_legislators", "opted_in_tags", "districts"):
|
||||
prefs.setdefault(key, [])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Minimal local account (Open Decision D1: D1a default — dev placeholder)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class LocalAuth:
|
||||
"""A development local-account auth (email + a derived bearer token).
|
||||
|
||||
This is the D1a default so the dashboard runs with no external IdP. It is
|
||||
deliberately NOT production-grade: no password hashing, no rotation, single
|
||||
host. Production should swap in a real auth service or the D1b OAuth path;
|
||||
the preferences shape is identical, so nothing else in the system changes.
|
||||
"""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
self._data = self._load()
|
||||
|
||||
def _load(self):
|
||||
if not os.path.exists(self.path):
|
||||
return {}
|
||||
try:
|
||||
with open(self.path, encoding="utf-8") as fh:
|
||||
return json.load(fh) or {}
|
||||
except (ValueError, OSError):
|
||||
return {}
|
||||
|
||||
def _save(self):
|
||||
os.makedirs(os.path.dirname(os.path.abspath(self.path)) or ".",
|
||||
exist_ok=True)
|
||||
tmp = self.path + ".tmp"
|
||||
with open(tmp, "w", encoding="utf-8") as fh:
|
||||
json.dump(self._data, fh, indent=2, sort_keys=True)
|
||||
os.replace(tmp, self.path)
|
||||
|
||||
@staticmethod
|
||||
def _new_user_id():
|
||||
return "u_" + uuid.uuid4().hex[:8]
|
||||
|
||||
@staticmethod
|
||||
def _token_for(user_id, email):
|
||||
h = hashlib.sha256(("local:" + user_id + ":" + email).encode()).hexdigest()
|
||||
return "dev." + h[:24]
|
||||
|
||||
def register(self, email, user_id=None):
|
||||
email = (email or "").strip().lower()
|
||||
for uid, rec in self._data.items():
|
||||
if rec.get("email") == email:
|
||||
rec["token"] = self._token_for(uid, email)
|
||||
self._save()
|
||||
return {"user_id": uid, "token": rec["token"], "email": email}
|
||||
uid = user_id or self._new_user_id()
|
||||
self._data[uid] = {"email": email, "token": self._token_for(uid, email)}
|
||||
self._save()
|
||||
return {"user_id": uid, "token": self._data[uid]["token"], "email": email}
|
||||
|
||||
def verify_token(self, token):
|
||||
if not token:
|
||||
return None
|
||||
for uid, rec in self._data.items():
|
||||
t = rec.get("token") or ""
|
||||
if token == t or token == t[4:] or token == uid:
|
||||
return uid
|
||||
return None
|
||||
|
||||
def user_by_id(self, user_id):
|
||||
return self._data.get(user_id)
|
||||
@@ -0,0 +1,310 @@
|
||||
"""Pure ranking core for the legislative dashboard (spec §5, DESIGN_DASHBOARD §4).
|
||||
|
||||
This module is deliberately **I/O-free**: it takes plain ``dict``-shaped bill
|
||||
rows (``scraper.schema.BillAction`` fields, optionally with a ``tags`` list) and
|
||||
returns them in float-to-top order. Keeping the ranking pure is what lets the
|
||||
rank logic be unit-tested without a database, and is the heart of §5.
|
||||
|
||||
The rule (DESIGN_DASHBOARD §4, verbatim intent)::
|
||||
|
||||
A bill FLOATS TO THE TOP iff
|
||||
author ∩ watched_legislators ≠ ∅ (author path)
|
||||
OR bill.tags ∩ opted_in_tags ≠ ∅ (tag path)
|
||||
OR (committee-of-watched-legistrator path) # committee -> members join
|
||||
|
||||
Scoring (design default, tunable — Open Decision D3)::
|
||||
|
||||
score = 3·(author is watched)
|
||||
+ 2·(author sits-in a watched legislator's committee)
|
||||
+ 2·( |tags ∩ opted_in_tags| )
|
||||
+ 1·(chamber == user's home chamber hint)
|
||||
|
||||
Band membership = ``score > 0``. Intra-band order = descending score, then
|
||||
``hearing_sort`` ascending (a future, earlier hearing floats ahead of a later
|
||||
one with the same score). The general band keeps source order.
|
||||
|
||||
Nothing here is "hidden": floated bills are *promoted* (a ``Relevant to you``
|
||||
marker / a 3px meaning bar), never removed — so a bill appears in both the
|
||||
personal band and the full queue.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Dict, List, Optional, Sequence, Set
|
||||
|
||||
from . import PERSONAL_BAND_N
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# hearing_sort (API_CONTRACT §1)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# "Upon Call of the Chair" / "Call of the Chair" sort LAST; a missing/unknown
|
||||
# time also sinks to the end of its day (design: time TBD note). A large
|
||||
# sentinel keeps them last without inventing a wall-clock value.
|
||||
_SORT_LAST = 24 * 60 * 60 + 60000 # > any HH:MM, so call-of-chair always sinks
|
||||
|
||||
_CALL_OF_CHAIR_RE = re.compile(r"call of the chair", re.I)
|
||||
_AMPM_RE = re.compile(r"^(\d{1,2})(?::(\d{2}))?\s*(am|pm)$", re.I)
|
||||
# A bare clock like "9:30" (no am/pm) — treat as 24h HH:MM.
|
||||
_HHMM_RE = re.compile(r"^(\d{1,2}):([0-5]?\d)$")
|
||||
|
||||
|
||||
def hearing_sort(hearing_time: Optional[str]) -> int:
|
||||
"""Derive a sortable integer for ``hearing_time``.
|
||||
|
||||
API_CONTRACT §1:
|
||||
|
||||
* ``"9:30am"`` / ``"10 minutes prior to Session"``-style wall times ->
|
||||
seconds-from-midnight.
|
||||
* ``"Upon Call of the Chair"`` / ``"Call of the Chair"`` -> **max** (last).
|
||||
* ``None`` / unparseable -> **max** (last), with the row still carrying
|
||||
``hearing_time: None`` so the UI can render a "time TBD" note.
|
||||
|
||||
Returns an int suitable for ascending sort within a day.
|
||||
"""
|
||||
if not hearing_time:
|
||||
return _SORT_LAST
|
||||
s = str(hearing_time).strip()
|
||||
if not s:
|
||||
return _SORT_LAST
|
||||
if _CALL_OF_CHAIR_RE.search(s):
|
||||
return _SORT_LAST
|
||||
m = _AMPM_RE.match(s)
|
||||
if m:
|
||||
hour = int(m.group(1))
|
||||
minute = int(m.group(2) or 0)
|
||||
period = m.group(3).lower()
|
||||
if period == "pm" and hour != 12:
|
||||
hour += 12
|
||||
if period == "am" and hour == 12: # 12am == midnight
|
||||
hour = 0
|
||||
return hour * 3600 + minute * 60
|
||||
m = _HHMM_RE.match(s)
|
||||
if m:
|
||||
return int(m.group(1)) * 3600 + int(m.group(2)) * 60
|
||||
# Unparseable (e.g. "10 minutes prior to Session") -> sinks last; that is
|
||||
# the designed "time TBD" behaviour, not a crash.
|
||||
return _SORT_LAST
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Relevance scoring (DESIGN_DASHBOARD §4, weights per Open Decision D3)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class Weights:
|
||||
"""Scoring weights. Defaults match DESIGN_DASHBOARD §4.
|
||||
|
||||
These are a *design default*, not frozen — Open Decision D3 flags them as
|
||||
tunable. The :class:`Weights` object is kept explicit (rather than module
|
||||
globals) so tests and the API can pass alternative weights without global
|
||||
mutation.
|
||||
"""
|
||||
|
||||
def __init__(self, author_watched: int = 3, committee_watched: int = 2,
|
||||
tag_watched: int = 2, home_chamber: int = 1):
|
||||
self.author_watched = author_watched
|
||||
self.committee_watched = committee_watched
|
||||
self.tag_watched = tag_watched
|
||||
self.home_chamber = home_chamber
|
||||
|
||||
|
||||
DEFAULT_WEIGHTS = Weights()
|
||||
|
||||
|
||||
def normalise_author(author: Optional[str]) -> str:
|
||||
"""Reduce a bill ``author`` string ("Schiavo et al.", "Macedo") to the
|
||||
primary legislator's slug, matching the member slugs the resolve/prefs
|
||||
layer keys on. Mirrors ``relational.slug`` / author-stem stripping."""
|
||||
if not author:
|
||||
return ""
|
||||
s = re.sub(r"\s+et al\.$", "", author.strip())
|
||||
s = re.sub(r"\s+et al\s+", " ", s, flags=re.I)
|
||||
s = re.sub(r"\s+", " ", s).strip()
|
||||
return s.lower()
|
||||
|
||||
|
||||
def _slug(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
return re.sub(r"[^a-z0-9]+", "-", name.strip().lower()).strip("-") or "unknown"
|
||||
|
||||
|
||||
def _match_tags(bill_tags: Sequence[str],
|
||||
opted: Set[str]) -> List[str]:
|
||||
"""Opted-in tags that actually appear on this bill (intersection, stable
|
||||
order = opted-in order), so the "Why is this here?" list is deterministic."""
|
||||
if not opted:
|
||||
return []
|
||||
present = {str(t).strip().lower() for t in (bill_tags or [])}
|
||||
return [t for t in opted if str(t).strip().lower() in present]
|
||||
|
||||
|
||||
def score_bill(bill: Dict,
|
||||
watched_authors: Set[str],
|
||||
watched_committees: Set[str],
|
||||
opted_in_tags: Set[str],
|
||||
home_chamber: Optional[str] = None,
|
||||
weights: Optional[Weights] = None) -> Dict:
|
||||
"""Compute a relevance ``{score, matched[]}`` for one bill against a user's
|
||||
preferences.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
bill: a dict-like with at least ``bill_id``, ``author``, ``committee``,
|
||||
``chamber_origin`` and optionally ``tags`` (list of tag strings).
|
||||
watched_authors: lower-cased primary-author slugs the user watches
|
||||
(``author ∩ watched_legislators`` path).
|
||||
watched_committees: slugs of committees the user watches (committee-of-
|
||||
watched-legislator path; ``∅`` when the correlation lane found no
|
||||
roster — never fabricated).
|
||||
opted_in_tags: tag strings the user opted into (``tags ∩ opted_in_tags``).
|
||||
home_chamber: optional chamber hint ("assembly"/"senate") for the +1 term.
|
||||
weights: scoring weights (defaults to DESIGN_DASHBOARD §4).
|
||||
|
||||
Returns
|
||||
-------
|
||||
A ``{"score": int, "matched": [{"type", "value"}, ...]}`` dict, matching the
|
||||
``Bill.relevance`` shape in API_CONTRACT §1. ``matched`` powers the
|
||||
"Why is this here?" transparency toggle. A single rule suffices to float
|
||||
(OR); stronger matches score higher.
|
||||
"""
|
||||
w = weights or DEFAULT_WEIGHTS
|
||||
matched: List[Dict[str, str]] = []
|
||||
score = 0
|
||||
|
||||
# --- author path ------------------------------------------------------
|
||||
author = bill.get("author")
|
||||
norm_author = normalise_author(author)
|
||||
if norm_author and norm_author in {a.lower() for a in watched_authors}:
|
||||
score += w.author_watched
|
||||
matched.append({
|
||||
"type": "watched-legislator",
|
||||
"value": "Author is a watched legislator"
|
||||
+ ("" if not author else " — " + str(author)),
|
||||
})
|
||||
|
||||
# --- committee-of-watched-legislator path ----------------------------
|
||||
# The bill sits in a committee the user's watched legislators chair/serve.
|
||||
# `bill.committee` is the daily-file committee name; watched_committees is
|
||||
# the slug set of the watched set's committees. (Degrades to a no-op when
|
||||
# the correlation lane found no roster — honest, never fabricated.)
|
||||
committee = bill.get("committee")
|
||||
if committee and _slug(committee) in {c.lower() for c in watched_committees}:
|
||||
score += w.committee_watched
|
||||
matched.append({
|
||||
"type": "committee-of-watched-legislator",
|
||||
"value": "Sits in a committee your watched legislator chair/serves "
|
||||
+ ("— " + str(committee) if committee else ""),
|
||||
})
|
||||
|
||||
# --- tag path ---------------------------------------------------------
|
||||
tag_hits = _match_tags(bill.get("tags") or [],
|
||||
{t.lower() for t in opted_in_tags})
|
||||
if tag_hits:
|
||||
score += w.tag_watched * len(tag_hits)
|
||||
for t in tag_hits:
|
||||
matched.append({"type": "tag", "value": "Topic: " + str(t)})
|
||||
|
||||
# --- home-chamber hint (optional, +1) --------------------------------
|
||||
if home_chamber and bill.get("chamber_origin"):
|
||||
if str(bill["chamber_origin"]).lower() == str(home_chamber).lower():
|
||||
score += w.home_chamber
|
||||
matched.append({
|
||||
"type": "home-chamber",
|
||||
"value": "Matches your home chamber ("
|
||||
+ str(bill["chamber_origin"]) + ")",
|
||||
})
|
||||
|
||||
return {"score": int(score), "matched": matched}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bill normalisation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Fields the UI consumes from a BillAction row, in API_CONTRACT §1 display order.
|
||||
_BILL_FIELDS = (
|
||||
"bill_id", "bill_number", "chamber_origin", "author", "short_title",
|
||||
"description", "committee", "hearing_time", "location", "date",
|
||||
"session", "item_number", "supplement", "source", "tags",
|
||||
)
|
||||
|
||||
|
||||
def normalise_bill(row: Dict, tags: Optional[Sequence[str]] = None) -> Dict:
|
||||
"""Project a raw DB row onto the API_CONTRACT §1 ``Bill`` shape.
|
||||
|
||||
* Keeps the exact field names the contract freezes.
|
||||
* Adds the derived ``hearing_sort`` (backend-owned, §1).
|
||||
* Carries ``tags`` from the LLM lane (may be ``[]``/``None`` -> no chips,
|
||||
ranking degrades to legislator-match only, per §6.6).
|
||||
* Nulls are preserved (the UI renders TBD, never crashes).
|
||||
"""
|
||||
out: Dict = {}
|
||||
for f in _BILL_FIELDS:
|
||||
if f == "tags":
|
||||
out["tags"] = None if tags is None else list(tags)
|
||||
else:
|
||||
val = row.get(f)
|
||||
out[f] = None if val == "" else val
|
||||
out["hearing_sort"] = hearing_sort(row.get("hearing_time"))
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Float-to-top ordering (DESIGN_DASHBOARD §4)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def is_floated(bill: Dict) -> bool:
|
||||
"""True if the bill carries a re: positive score (i.e. it floats).
|
||||
|
||||
``relevance`` is only populated for logged-in requests (§1); anon bills have
|
||||
no ``relevance`` key.
|
||||
"""
|
||||
rel = bill.get("relevance")
|
||||
return bool(rel and int(rel.get("score", 0)) > 0)
|
||||
|
||||
|
||||
def float_to_top(bills: Sequence[Dict],
|
||||
personal_band_n: int = PERSONAL_BAND_N) -> List[Dict]:
|
||||
"""Stably re-sort ``bills`` so floated bills (``relevance.score > 0``) come
|
||||
first, ordered by ``(score desc, hearing_sort asc)``; non-floated bills keep
|
||||
their existing relative order.
|
||||
|
||||
This is the "promote, never hide" step of DESIGN_DASHBOARD §4: every floated
|
||||
bill also remains visible in the full queue (marked), only its *position*
|
||||
changes. Idempotent: running it twice is a no-op.
|
||||
"""
|
||||
def key(bill: Dict) -> tuple:
|
||||
rel = bill.get("relevance") or {}
|
||||
score = int(rel.get("score", 0))
|
||||
# -score so higher scores sort earlier; hearing earlier-in-day sorts
|
||||
# earlier; tie-break on bill_id for total determinism.
|
||||
return (-score,
|
||||
int(bill.get("hearing_sort") or _SORT_LAST),
|
||||
str(bill.get("bill_id") or ""))
|
||||
|
||||
# Stable sort (Python's sort is stable; the bill_id tie-break makes it
|
||||
# total anyway). personal_band_n is accepted for signature parity with the
|
||||
# band helpers but does NOT truncate the full queue.
|
||||
_ = personal_band_n
|
||||
ordered = sorted(bills, key=key)
|
||||
return ordered
|
||||
|
||||
|
||||
def personal_band(bills: Sequence[Dict],
|
||||
n: int = PERSONAL_BAND_N) -> List[Dict]:
|
||||
"""The first ``n`` floated bills (the "Relevant to you" band, top N=10).
|
||||
|
||||
Derived by "relevance.score > 0, then hearing_sort", per API_CONTRACT §1 —
|
||||
i.e. the head of a :func:`float_to_top` ordering. Returns ``[]`` when nothing
|
||||
is floated (the UI shows the setup / empty state, not a dead band).
|
||||
"""
|
||||
band = [b for b in float_to_top(bills) if is_floated(b)]
|
||||
return band[:n]
|
||||
|
||||
|
||||
def full_queue(bills: Sequence[Dict]) -> List[Dict]:
|
||||
"""The complete queue, sorted the same way as :func:`float_to_top` but
|
||||
untruncated — every floated bill is marked and every other bill follows."""
|
||||
return float_to_top(bills)
|
||||
@@ -0,0 +1,141 @@
|
||||
"""ZIP/address -> district -> legislators resolution (spec §5, DESIGN §8, D2).
|
||||
|
||||
Open Decision D2 (inherited from the UX lane) asks which ZIP-to-district data
|
||||
source the dashboard owns. The authoritative source is the Legislature's
|
||||
full-address tool / a county voter file or data.ca.gov boundary datasets; that
|
||||
ETL is the correlation lane's job. This module provides a runnable demonstration
|
||||
resolver so the ZIP->legislator flow is real and testable:
|
||||
|
||||
* a small, documented ZIP sample map (see DEMO_ZIPS) covering a few districts
|
||||
of the current 2025-2026 session;
|
||||
* a ZIP fast path (0 / 1 / several districts) with the confidence semantics the
|
||||
design freezes (single / ambiguous / zero / out-of-state);
|
||||
* a manual fallback (pick a district by hand) for the edge cases the design
|
||||
requires (P.O. boxes, cross-border ZIPs, post-redistricting gaps).
|
||||
|
||||
Real-world fact the design grounds this in: a ZIP alone does NOT reliably
|
||||
resolve a CA district (a single ZIP can straddle two districts). Fast-path
|
||||
ambiguity therefore proposes candidates; the user commits (never auto-watched).
|
||||
A real deployment swaps DEMO_ZIPS for the data.ca.gov boundary join, which is
|
||||
documented in DASHBOARD.md.
|
||||
|
||||
Resolve never persists: it proposes (API_CONTRACT section 3.1). Persisting a
|
||||
watch is the user's explicit act via /prefs/watch.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional, Sequence
|
||||
|
||||
from . import data
|
||||
|
||||
# DEMO sample: ZIP -> the (current) district(s) it maps to, each with its
|
||||
# assembly/senate member. Explicitly a placeholder for the authoritative
|
||||
# boundary join (D2). Legislators reference the session's real member slugs.
|
||||
DEMO_ZIPS = {
|
||||
"90210": [{"district_id": "A-14", "type": "ASSEMBLY", "number": 14,
|
||||
"assembly_member": {"legislator_id": "schiavo",
|
||||
"name": "M. Schiavo", "party": "D"}}],
|
||||
"94107": [{"district_id": "A-11", "type": "ASSEMBLY", "number": 11,
|
||||
"assembly_member": {"legislator_id": "macedo",
|
||||
"name": "K. Macedo", "party": "D"}}],
|
||||
"90012": [
|
||||
# A ZIP straddling two districts -> AMBIGUOUS (the design's example).
|
||||
{"district_id": "S-08", "type": "SENATE", "number": 8,
|
||||
"senate_member": {"legislator_id": "chen",
|
||||
"name": "W. Chen", "party": "R"}},
|
||||
{"district_id": "A-09", "type": "ASSEMBLY", "number": 9,
|
||||
"assembly_member": {"legislator_id": "ortega",
|
||||
"name": "A. Ortega", "party": "D"}},
|
||||
],
|
||||
"92101": [{"district_id": "A-32", "type": "ASSEMBLY", "number": 32,
|
||||
"assembly_member": {"legislator_id": "bryan",
|
||||
"name": "B. Bryan", "party": "D"}}],
|
||||
"99501": [{"district_id": "A-55", "type": "ASSEMBLY", "number": 55,
|
||||
"assembly_member": {"legislator_id": "jackson",
|
||||
"name": "H. Jackson", "party": "R"}}],
|
||||
}
|
||||
|
||||
# 5-digit ZIP prefixes that are obviously not California, so the out-of-state
|
||||
# confidence branch is demonstrable.
|
||||
_OUT_OF_STATE_PREFIX = ("990", "999")
|
||||
|
||||
|
||||
def _to_resolution(mode: str, districts: Sequence[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
suggested_watch = []
|
||||
for d in districts:
|
||||
for member in (d.get("assembly_member"), d.get("senate_member")):
|
||||
if member and member.get("legislator_id"):
|
||||
suggested_watch.append(member["legislator_id"])
|
||||
return {
|
||||
"resolved_from": mode,
|
||||
"confidence": "single" if len(districts) == 1
|
||||
else ("ambiguous" if districts else "zero"),
|
||||
"districts": [dict(d) for d in districts],
|
||||
"suggested_topics": _suggested_topics(districts),
|
||||
"suggested_watch": suggested_watch,
|
||||
}
|
||||
|
||||
|
||||
def _manual_district(district: str) -> Dict[str, Any]:
|
||||
d = district.strip()
|
||||
typ = d[0].upper() if d else "A"
|
||||
num = d[1:]
|
||||
type_ = "SENATE" if typ == "S" else "ASSEMBLY"
|
||||
out: Dict[str, Any] = {
|
||||
"district_id": d,
|
||||
"type": type_,
|
||||
"number": int(num) if num.isdigit() else None,
|
||||
}
|
||||
member_key = "senate_member" if type_ == "SENATE" else "assembly_member"
|
||||
out[member_key] = {}
|
||||
for entry in DEMO_ZIPS.values():
|
||||
for cand in entry:
|
||||
if cand.get("district_id") == d and cand.get(member_key):
|
||||
out[member_key] = dict(cand[member_key])
|
||||
return out
|
||||
|
||||
|
||||
def _suggested_topics(districts: Sequence[Dict[str, Any]]) -> List[str]:
|
||||
topics: List[str] = []
|
||||
seen = set()
|
||||
for d in districts:
|
||||
for member in (d.get("assembly_member"), d.get("senate_member")):
|
||||
if not member:
|
||||
continue
|
||||
lid = member.get("legislator_id")
|
||||
for lg in data.load_legislators():
|
||||
if lg.get("legislator_id") == lid:
|
||||
for x in lg.get("domains") or []:
|
||||
if x not in seen:
|
||||
seen.add(x)
|
||||
topics.append(x)
|
||||
return topics
|
||||
|
||||
|
||||
def resolve(zip: Optional[str] = None, address: Optional[str] = None,
|
||||
district: Optional[str] = None) -> Dict[str, Any]:
|
||||
# Manual pick: no ZIP ambiguity at all.
|
||||
if district and not zip and not address:
|
||||
d = _manual_district(district)
|
||||
return _to_resolution("manual", [d])
|
||||
key = None
|
||||
mode = "zip"
|
||||
if address:
|
||||
a = address.strip()
|
||||
key = a[-5:] if len(a) >= 5 else a
|
||||
mode = "address"
|
||||
elif zip is not None:
|
||||
key = str(zip).strip()
|
||||
else:
|
||||
return {"resolved_from": "zip", "confidence": "zero",
|
||||
"districts": [], "suggested_topics": [],
|
||||
"suggested_watch": []}
|
||||
entry = DEMO_ZIPS.get(key)
|
||||
if entry is None:
|
||||
confidence = "out-of-state" if key[:3] in _OUT_OF_STATE_PREFIX \
|
||||
else "zero"
|
||||
return {"resolved_from": mode, "confidence": confidence,
|
||||
"districts": [], "suggested_topics": [],
|
||||
"suggested_watch": []}
|
||||
return _to_resolution(mode, entry)
|
||||
@@ -0,0 +1,186 @@
|
||||
# noqa: E501
|
||||
"""Stdlib HTTP server for the dashboard (spec §5). No framework.
|
||||
|
||||
Exposes the API_CONTRACT endpoints over http.server + a tiny static adapter
|
||||
for the frontend (scraper/dashboard/static/). A thin layer over
|
||||
scraper.dashboard.api: parse HTTP line + query + JSON body, build an
|
||||
api.Request, call api.run, write (status, body) back as JSON.
|
||||
|
||||
Run: python -m scraper.dashboard.server
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
|
||||
from . import api
|
||||
from .api import Request, Services
|
||||
|
||||
STATIC_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
||||
"static")
|
||||
|
||||
_CONTENT_TYPES = {
|
||||
".html": "text/html; charset=utf-8",
|
||||
".js": "text/javascript; charset=utf-8",
|
||||
".css": "text/css; charset=utf-8",
|
||||
".json": "application/json; charset=utf-8",
|
||||
".svg": "image/svg+xml",
|
||||
".ico": "image/x-icon",
|
||||
".png": "image/png",
|
||||
}
|
||||
|
||||
|
||||
class DashboardHandler(BaseHTTPRequestHandler):
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
def do_GET(self):
|
||||
return self._dispatch("GET")
|
||||
|
||||
def do_POST(self):
|
||||
return self._dispatch("POST")
|
||||
|
||||
def do_PUT(self):
|
||||
return self._dispatch("PUT")
|
||||
|
||||
def do_DELETE(self):
|
||||
return self._dispatch("DELETE")
|
||||
|
||||
def _dispatch(self, method):
|
||||
parsed = urlparse(self.path)
|
||||
if not parsed.path.startswith("/api"):
|
||||
return self._serve_static(parsed.path)
|
||||
try:
|
||||
query = parse_qs(parsed.query)
|
||||
flat = {}
|
||||
for k, v in query.items():
|
||||
flat[k] = v[0] if len(v) == 1 else v
|
||||
body = {}
|
||||
if method in ("POST", "PUT"):
|
||||
body = self._read_json_body()
|
||||
token = self._bearer()
|
||||
req = Request(
|
||||
method=method,
|
||||
path=parsed.path,
|
||||
query=flat,
|
||||
body=body,
|
||||
token=token,
|
||||
services=self.services,
|
||||
user_id=None,
|
||||
)
|
||||
status, payload = api.run(req)
|
||||
except Exception as exc:
|
||||
status, payload = 500, {
|
||||
"error": {"code": "internal_error",
|
||||
"message": str(exc)}}
|
||||
return self._write_json(status, payload)
|
||||
|
||||
def _bearer(self):
|
||||
auth = self.headers.get("Authorization", "")
|
||||
if auth.lower().startswith("bearer "):
|
||||
return auth[7:].strip()
|
||||
return None
|
||||
|
||||
def _read_json_body(self):
|
||||
length = int(self.headers.get("Content-Length") or 0)
|
||||
if length <= 0:
|
||||
return {}
|
||||
raw = self.rfile.read(length)
|
||||
try:
|
||||
data = json.loads(raw.decode("utf-8"))
|
||||
except (ValueError, UnicodeDecodeError):
|
||||
return {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
def _serve_static(self, path):
|
||||
rel = path.lstrip("/") or "index.html"
|
||||
if ".." in rel or rel.startswith("/"):
|
||||
return self._write_raw(404, b"not found", "text/plain")
|
||||
fp = os.path.join(STATIC_DIR, rel)
|
||||
if os.path.isfile(fp):
|
||||
ctype = _CONTENT_TYPES.get(
|
||||
os.path.splitext(fp)[1], "application/octet-stream")
|
||||
with open(fp, "rb") as fh:
|
||||
body = fh.read()
|
||||
return self._write_raw(200, body, ctype)
|
||||
root = os.path.join(STATIC_DIR, "index.html")
|
||||
if os.path.isfile(root):
|
||||
with open(root, "rb") as fh:
|
||||
body = fh.read()
|
||||
return self._write_raw(200, body,
|
||||
"text/html; charset=utf-8")
|
||||
return self._write_raw(404, b"not found", "text/plain")
|
||||
|
||||
def _write_json(self, status, payload):
|
||||
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
return self._write_raw(status, body,
|
||||
"application/json; charset=utf-8")
|
||||
|
||||
def _write_raw(self, status, body, ctype):
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", ctype)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("Access-Control-Allow-Origin", "*")
|
||||
self.end_headers()
|
||||
if status != 204:
|
||||
self.wfile.write(body)
|
||||
|
||||
|
||||
def make_handler(services, log_on=False):
|
||||
cls = type("BoundHandler", (DashboardHandler,),
|
||||
{"services": services})
|
||||
if log_on:
|
||||
cls.log_message = BaseHTTPRequestHandler.log_message
|
||||
return cls
|
||||
|
||||
|
||||
def serve(host="127.0.0.1", port=8000,
|
||||
db_path=None, prefs_path="out/prefs.json",
|
||||
auth_path="out/auth.json", serve_static=True,
|
||||
log_on=False):
|
||||
services = Services.default(db_path=db_path,
|
||||
prefs_path=prefs_path,
|
||||
auth_path=auth_path)
|
||||
handler = make_handler(services, log_on=log_on)
|
||||
srv = ThreadingHTTPServer((host, port), handler)
|
||||
sys.stderr.write(
|
||||
"dashboard: http://%s:%d/ (static %s)\n"
|
||||
% (host, port, "on" if serve_static else "off"))
|
||||
try:
|
||||
srv.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finally:
|
||||
srv.server_close()
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
import argparse
|
||||
p = argparse.ArgumentParser(
|
||||
prog="scraper.dashboard",
|
||||
description="Serve the legislative dashboard (API + static UI).")
|
||||
p.add_argument("--host", default="127.0.0.1")
|
||||
p.add_argument("--port", type=int, default=8000)
|
||||
p.add_argument("--db", default=None,
|
||||
help="SQLite path (default: db/daily_file.sqlite)")
|
||||
p.add_argument("--prefs", default="out/prefs.json",
|
||||
help="preferences JSON file")
|
||||
p.add_argument("--auth", default="out/auth.json",
|
||||
help="local-auth JSON file")
|
||||
p.add_argument("--no-static", dest="static", action="store_false",
|
||||
help="serve API only, no static frontend")
|
||||
p.add_argument("--log", dest="log", action="store_true",
|
||||
help="log HTTP requests to stderr")
|
||||
args = p.parse_args(argv)
|
||||
return serve(host=args.host, port=args.port,
|
||||
db_path=args.db, prefs_path=args.prefs,
|
||||
auth_path=args.auth, serve_static=args.static,
|
||||
log_on=args.log)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,657 @@
|
||||
/* ============================================================================
|
||||
Legislative Dashboard — frontend (spec §5, DESIGN §3/§5/§10).
|
||||
Vanilla JS + fetch against the stdlib API. Two personas on one surface:
|
||||
anonymous (overview) and logged-in (personal band + watch/opt-in). No
|
||||
frameworks, no build step.
|
||||
========================================================================== */
|
||||
(function () {
|
||||
"use strict";
|
||||
|
||||
var API = "/api/v1";
|
||||
var BAND_N = 10;
|
||||
var state = {
|
||||
token: localStorage.getItem("dash_token") || null,
|
||||
user: null,
|
||||
prefs: null,
|
||||
filters: { upcoming: true, chamber: "", q: "", tags: [] },
|
||||
personal: false,
|
||||
showReason: false,
|
||||
allItems: [],
|
||||
tags: [],
|
||||
};
|
||||
|
||||
/* ---------- tiny fetch client ---------- */
|
||||
function apiFetch(path, opts) {
|
||||
opts = opts || {};
|
||||
var headers = { "Content-Type": "application/json" };
|
||||
if (state.token) headers["Authorization"] = "Bearer " + state.token;
|
||||
return fetch(API + path, {
|
||||
method: opts.method || "GET",
|
||||
headers: headers,
|
||||
body: opts.body ? JSON.stringify(opts.body) : undefined,
|
||||
}).then(function (r) {
|
||||
return r.json().then(function (j) {
|
||||
return { status: r.status, body: j };
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function el(id) { return document.getElementById(id); }
|
||||
|
||||
/* ---------- render helpers ---------- */
|
||||
function chip(chamber) {
|
||||
var c = (chamber || "ASSEMBLY").toLowerCase();
|
||||
var label = c === "senate" ? "Senate" : "Assembly";
|
||||
return '<span class="chip ' + (c === "senate" ? "sen" : "asm") + '">'
|
||||
+ label + "</span>";
|
||||
}
|
||||
|
||||
function legMatches(ev) {
|
||||
var m = (ev.relevance && ev.relevance.matched) || [];
|
||||
return m.some(function (x) { return x.type !== "tag"; });
|
||||
}
|
||||
function tagMatches(ev) {
|
||||
var m = (ev.relevance && ev.relevance.matched) || [];
|
||||
return m.some(function (x) { return x.type === "tag"; });
|
||||
}
|
||||
function floatClass(ev) {
|
||||
var l = legMatches(ev), t = tagMatches(ev);
|
||||
if (t && l) return "both";
|
||||
if (t) return "tag-match";
|
||||
if (l) return "leg-match";
|
||||
return "";
|
||||
}
|
||||
|
||||
function rowHTML(ev, opts) {
|
||||
opts = opts || {};
|
||||
var rel = ev.relevance && ev.relevance.score > 0;
|
||||
var cls = "row";
|
||||
if (rel) cls += " relevant " + floatClass(ev);
|
||||
var time = ev.hearing_time || '<span class="meta">time TBD</span>';
|
||||
var meta = [time, ev.committee, ev.location, ev.session]
|
||||
.filter(Boolean).join(" · ");
|
||||
var tags = (ev.tags || []).slice(0, 6)
|
||||
.map(function (tg) {
|
||||
var on = state.prefs &&
|
||||
(state.prefs.opted_in_tags || []).indexOf(tg) >= 0;
|
||||
return '<span class="tag ' +
|
||||
(on ? 'on"' : '"') + ">" + esc(tg) + "</span>";
|
||||
}).join("");
|
||||
var marker = rel
|
||||
? '<span class="chip marker" title="Floated: relevant to you">'
|
||||
+ "Relevant to you</span>"
|
||||
: "";
|
||||
var reasons = "";
|
||||
if (rel) {
|
||||
var parts = (ev.relevance.matched || []).map(function (x) {
|
||||
return esc(x.value);
|
||||
});
|
||||
reasons = '<div class="reason' +
|
||||
(state.showReason ? " show" : "") + '">'
|
||||
+ "Floated: " + parts.join("; ") + "</div>";
|
||||
}
|
||||
var url = "https://leginfo.legislature.ca.gov/bill/" +
|
||||
(ev.session || "2025-2026") + "/AB" + billNum(ev);
|
||||
return '<div class="' + cls + '" data-bill="' +
|
||||
esc(ev.bill_id) + '">'
|
||||
+ '<span class="num">' + chip(ev.chamber_origin) +
|
||||
'<br>' + esc(ev.bill_number) + "</span>"
|
||||
+ '<div class="main"><span class="title">'
|
||||
+ esc(ev.short_title || ev.bill_number) + "</span>"
|
||||
+ "<div class='meta'>" + esc(meta) + "</div>"
|
||||
+ "<a href='" + url + "' target='_blank'"
|
||||
+ " rel='noopener'>View →</a> " + marker
|
||||
+ tags
|
||||
+ reasons
|
||||
+ '</div>'
|
||||
+ '<div class="actions">'
|
||||
+ (state.prefs
|
||||
? '<button class="why-btn" type="button">Why▸</button>'
|
||||
: "")
|
||||
+ "</div>"
|
||||
+ "</div>";
|
||||
}
|
||||
|
||||
function billNum(ev) {
|
||||
var m = (ev.bill_number || "").match(/(\d+)/);
|
||||
return m ? m[1] : "";
|
||||
}
|
||||
function esc(s) {
|
||||
return String(s == null ? "" : s)
|
||||
.replace(/&/g, "&").replace(/</g, "<")
|
||||
.replace(/>/g, ">").replace(/"/g, """);
|
||||
}
|
||||
|
||||
/* ---------- band rendering ---------- */
|
||||
function groupByDay(items) {
|
||||
var groups = {};
|
||||
items.forEach(function (it) {
|
||||
var key = it.date || "TBD";
|
||||
if (!groups[key]) groups[key] = { items: [], key: key };
|
||||
groups[key].items.push(it);
|
||||
});
|
||||
return groups;
|
||||
}
|
||||
|
||||
function renderGroups(container, items) {
|
||||
var byDate = groupByDay(items);
|
||||
var html = "";
|
||||
Object.keys(byDate).sort().forEach(function (d) {
|
||||
var g = byDate[d];
|
||||
var label = g.key === "TBD" ? "Date TBD"
|
||||
: niceDate(g.key) + " · " + g.items.length +
|
||||
" item" + (g.items.length > 1 ? "s" : "");
|
||||
html += '<div class="day-heading"><b>' + esc(label) +
|
||||
"</b></div>";
|
||||
g.items.forEach(function (ev) {
|
||||
html += rowHTML(ev);
|
||||
});
|
||||
});
|
||||
container.innerHTML = html;
|
||||
wireRows(container);
|
||||
}
|
||||
|
||||
function niceDate(iso) {
|
||||
try {
|
||||
var d = new Date(iso + "T00:00:00");
|
||||
return d.toLocaleDateString(undefined, {
|
||||
weekday: "short", month: "short", day: "numeric",
|
||||
year: "numeric" });
|
||||
} catch (e) { return iso; }
|
||||
}
|
||||
|
||||
/* ---------- empty states (DESIGN §6) ---------- */
|
||||
function showEmpty(kind, container) {
|
||||
var el = el(container);
|
||||
var h = "";
|
||||
if (kind === "no-next") {
|
||||
h = "<h3>No hearings scheduled</h3>" +
|
||||
"<p>No upcoming daily-file items in this window.</p>" +
|
||||
"<button type='button' class='escape' id='see-past'>" +
|
||||
"See recent actions instead</button>";
|
||||
} else if (kind === "filter") {
|
||||
h = "<h3>No upcoming items match your filters</h3>" +
|
||||
"<p><button type='button' class='escape' id='clear'>" +
|
||||
"Clear filters</button> · " +
|
||||
"<button type='button' class='escape' id='loosen'>" +
|
||||
"Loosen to all topics</button> · " +
|
||||
"<button type='button' class='escape' id='widen'>" +
|
||||
"Widen date window</button></p>";
|
||||
} else if (kind === "setup") {
|
||||
el.innerHTML = "";
|
||||
el.hidden = false;
|
||||
el.parentElement.querySelector("#band-setup").hidden = false;
|
||||
return;
|
||||
} else if (kind === "no-tags") {
|
||||
h = "<h3>Topic analysis is filling in</h3>" +
|
||||
"<p>Tag chips appear as the topic service warms. " +
|
||||
"You can still watch legislators above.</p>";
|
||||
}
|
||||
el.innerHTML = h;
|
||||
el.hidden = false;
|
||||
wire(el, kind);
|
||||
}
|
||||
|
||||
function clearEmpty(container) {
|
||||
var e = el(container);
|
||||
e.hidden = true; e.innerHTML = "";
|
||||
}
|
||||
|
||||
/* ---------- load + render ---------- */
|
||||
function load() {
|
||||
var q = new URLSearchParams();
|
||||
if (state.filters.upcoming) q.set("upcoming", "1");
|
||||
if (state.filters.chamber) q.set("chamber", state.filters.chamber);
|
||||
if (state.filters.q) q.set("q", state.filters.q);
|
||||
if (state.filters.tags.length)
|
||||
q.set("tags", state.filters.tags.join(","));
|
||||
if (state.personal && state.token) q.set("personal", "1");
|
||||
q.set("full", "1");
|
||||
apiFetch("/bills?" + q.toString()).then(function (r) {
|
||||
if (r.status !== 200) { return; }
|
||||
render(r.body);
|
||||
});
|
||||
}
|
||||
|
||||
function render(env) {
|
||||
// freshness / context band
|
||||
el("freshness-updated").textContent =
|
||||
"Updated " + (env.as_of || "").slice(0, 10);
|
||||
el("freshness-session").textContent = "Session " + (env.session || "");
|
||||
var c = env.chamber_availability || {};
|
||||
el("freshness-chamber").textContent =
|
||||
c.SENATE === "available"
|
||||
? "Senate available"
|
||||
: "Senate unavailable today";
|
||||
el("freshness-chamber").style.color =
|
||||
c.SENATE === "available" ? "var(--muted)" : "var(--warn)";
|
||||
|
||||
var items = env.items || [];
|
||||
state.allItems = items;
|
||||
var band = items.filter(function (i) {
|
||||
return i.relevance && i.relevance.score > 0;
|
||||
});
|
||||
var bandFull = band;
|
||||
var shown = bandFull.slice(0, BAND_N);
|
||||
|
||||
// personal band
|
||||
var bandCount = el("band-count");
|
||||
bandCount.textContent = bandFull.length
|
||||
? bandFull.length + (bandFull.length > BAND_N
|
||||
? " (showing " + shown.length + ")" : "") : "";
|
||||
var list = el("band-list");
|
||||
var setup = el("band-setup");
|
||||
if (!state.prefs || (state.prefs.watched_legislators || []).length
|
||||
=== 0 && (state.prefs.opted_in_tags || []).length === 0) {
|
||||
list.innerHTML = "";
|
||||
setup.hidden = false;
|
||||
} else {
|
||||
setup.hidden = true;
|
||||
if (!shown.length) {
|
||||
list.innerHTML = "<p class='meta'>No bills match your " +
|
||||
"watches/topics yet. Add more on the right, or " +
|
||||
"<button type='button' class='escape' id='clear-watch'>" +
|
||||
"Clear filters</button>.</p>";
|
||||
} else {
|
||||
renderGroups(list, shown);
|
||||
}
|
||||
var btn = el("btn-show-all");
|
||||
btn.hidden = bandFull.length <= BAND_N;
|
||||
}
|
||||
|
||||
// all band
|
||||
var allCount = el("all-count");
|
||||
allCount.textContent = items.length +
|
||||
" upcoming" + (items.length === 1 ? " item" : " items");
|
||||
var allList = el("all-list");
|
||||
var allEmpty = el("all-empty");
|
||||
if (!items.length) {
|
||||
allList.innerHTML = "";
|
||||
showEmpty("no-next", "all-empty");
|
||||
} else {
|
||||
clearEmpty("all-empty");
|
||||
renderGroups(allList, items);
|
||||
}
|
||||
|
||||
// aside teaser
|
||||
renderTeaser();
|
||||
loadTags();
|
||||
}
|
||||
|
||||
function renderTeaser() {
|
||||
var ul = el("topics-teaser");
|
||||
ul.innerHTML = "";
|
||||
(state.tags || []).slice(0, 6).forEach(function (t) {
|
||||
var li = document.createElement("li");
|
||||
li.innerHTML = "<span class='n'>" + t.count + "</span> " +
|
||||
esc(t.label);
|
||||
var b = document.createElement("button");
|
||||
b.type = "button";
|
||||
b.textContent = "view";
|
||||
b.onclick = function () {
|
||||
el("f-tags").value = t.tag_id.split(":")[1];
|
||||
state.filters.tags = [el("f-tags").value];
|
||||
load();
|
||||
};
|
||||
li.appendChild(b);
|
||||
ul.appendChild(li);
|
||||
});
|
||||
}
|
||||
|
||||
function loadTags() {
|
||||
apiFetch("/tags").then(function (r) {
|
||||
if (r.status === 200) {
|
||||
state.tags = r.body.tags || [];
|
||||
var sel = el("f-tags");
|
||||
sel.innerHTML = state.tags
|
||||
.map(function (t) {
|
||||
var name = t.tag_id.split(":")[1];
|
||||
return '<option value="' + esc(name) + '">' +
|
||||
esc(t.label) + " (" + t.count +
|
||||
")</option>";
|
||||
}).join("");
|
||||
// rehydrate current selection
|
||||
Array.prototype.forEach.call(sel.options,
|
||||
function (o) {
|
||||
o.selected =
|
||||
state.filters.tags.indexOf(o.value) >= 0;
|
||||
});
|
||||
renderTeaser();
|
||||
} else {
|
||||
// "no tagged bills yet" — degrade, no crash
|
||||
el("topics-teaser").innerHTML =
|
||||
"<li class='meta'>Topic analysis is filling in…</li>";
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/* ---------- auth (D1a local account, dev) ---------- */
|
||||
function signIn(email) {
|
||||
apiFetch("/auth/register", { method: "POST",
|
||||
body: { email: email } }).then(function (r) {
|
||||
if (r.status !== 201) { alert("Sign-in failed"); return; }
|
||||
state.token = r.body.token;
|
||||
localStorage.setItem("dash_token", r.body.token);
|
||||
apiFetch("/prefs", {}).then(function (pr) {
|
||||
state.prefs = pr.status === 200 ? pr.body : null;
|
||||
refreshUI();
|
||||
load();
|
||||
});
|
||||
});
|
||||
}
|
||||
function signOut() {
|
||||
state.token = null;
|
||||
state.prefs = null;
|
||||
state.personal = false;
|
||||
localStorage.removeItem("dash_token");
|
||||
refreshUI();
|
||||
load();
|
||||
}
|
||||
|
||||
function refreshUI() {
|
||||
var has = state.token;
|
||||
el("user-band").hidden = !has;
|
||||
el("anon-cta").hidden = !!has;
|
||||
el("personal-controls").hidden = !has;
|
||||
el("aside-title").textContent = has
|
||||
? "Your watchlist" : "Top topics this session";
|
||||
if (has) {
|
||||
var p = state.prefs || {};
|
||||
el("user-band-text").textContent =
|
||||
"ZIP " + (p.zip || "—") + " · " +
|
||||
((p.watched_legislators || []).length) +
|
||||
" legislators · " +
|
||||
((p.opted_in_tags || []).length) + " topics";
|
||||
renderWatch();
|
||||
renderTopics();
|
||||
}
|
||||
}
|
||||
|
||||
/* ---------- watch / topic toggles ---------- */
|
||||
function renderWatch() {
|
||||
var c = el("resolve-result");
|
||||
var p = state.prefs || {};
|
||||
var wl = p.watched_legislators || [];
|
||||
var html = "";
|
||||
wl.forEach(function (id) {
|
||||
html += "<div class='candidate'>" +
|
||||
"<button type='button' class='watch-btn on' " +
|
||||
"data-leg='" + esc(id) + "'>✓ watching " +
|
||||
esc(pretty(id)) + "</button>" +
|
||||
"<button type='button' class='unwatch-btn ghost' " +
|
||||
"data-leg='" + esc(id) + "'>remove</button>" +
|
||||
"</div>";
|
||||
});
|
||||
c.innerHTML = html || "";
|
||||
wire(c, "watch");
|
||||
}
|
||||
|
||||
function renderTopics() {
|
||||
var c = el("topic-list");
|
||||
var p = state.prefs || {};
|
||||
var opted = p.opted_in_tags || [];
|
||||
c.innerHTML = (state.tags || []).map(function (t) {
|
||||
var name = t.tag_id.split(":")[1];
|
||||
var on = opted.indexOf(name) >= 0;
|
||||
return "<button type='button' class='topic-chip" +
|
||||
(on ? " on" : "") + "' data-tag='" + esc(name) +
|
||||
"'>" + (on ? "✓ " : "") + esc(t.label) +
|
||||
" <span class='count'>· " + t.count + " bills</span>" +
|
||||
"</button>";
|
||||
}).join("");
|
||||
wire(c, "topic");
|
||||
}
|
||||
|
||||
function toggleWatch(id, on) {
|
||||
var path = on
|
||||
? "/prefs/watch"
|
||||
: "/prefs/watch/" + encodeURIComponent(id);
|
||||
var method = on ? "POST" : "DELETE";
|
||||
var body = on ? { legislator_id: id } : null;
|
||||
apiFetch(path, { method: method, body: body }).then(function (r) {
|
||||
if (r.status === 400 || r.status === 404) {
|
||||
alert(r.body.error && r.body.error.message);
|
||||
}
|
||||
apiFetch("/prefs", {}).then(function (pr) {
|
||||
state.prefs = pr.status === 200 ? pr.body : null;
|
||||
refreshUI();
|
||||
load();
|
||||
announce("Ranking updated");
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function toggleTopic(tag, on) {
|
||||
var path = on ? "/prefs/topics"
|
||||
: "/prefs/topics/" + encodeURIComponent(tag);
|
||||
var method = on ? "POST" : "DELETE";
|
||||
var body = on ? { tag: tag } : null;
|
||||
apiFetch(path, { method: method, body: body }).then(function (r) {
|
||||
if (r.status === 400) {
|
||||
alert(r.body.error && r.body.error.message);
|
||||
}
|
||||
apiFetch("/prefs", {}).then(function (pr) {
|
||||
state.prefs = pr.status === 200 ? pr.body : null;
|
||||
refreshUI();
|
||||
load();
|
||||
announce("Ranking updated");
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function announce(msg) {
|
||||
var a = el("freshness-updated");
|
||||
var old = a.textContent;
|
||||
a.textContent = old + " · " + msg;
|
||||
}
|
||||
|
||||
/* ---------- ZIP resolve (DESIGN §8, §6.4) ---------- */
|
||||
function resolve(zip) {
|
||||
apiFetch("/districts/resolve", { method: "POST",
|
||||
body: { zip: zip } }).then(function (r) {
|
||||
var c = el("resolve-result");
|
||||
var p = state.prefs || {};
|
||||
if (r.status === 200 && r.body.confidence === "single") {
|
||||
p.zip = zip;
|
||||
var d = r.body.districts[0];
|
||||
var mem = d.assembly_member || d.senate_member;
|
||||
p.districts = [d];
|
||||
if (mem && (p.watched_legislators || []).indexOf(
|
||||
mem.legislator_id) === -1) {
|
||||
toggleWatch(mem.legislator_id, true);
|
||||
}
|
||||
(r.body.suggested_topics || []).forEach(function (t) {
|
||||
if ((p.opted_in_tags || []).indexOf(t) === -1)
|
||||
toggleTopic(t, true);
|
||||
});
|
||||
c.innerHTML = "<div class='candidate'>" +
|
||||
"<span class='who'>Resolved: " + esc(d.district_id) +
|
||||
" — " + esc((mem && mem.name) || "") + "</span>" +
|
||||
"</div>" +
|
||||
"<p class='meta'>Watched " +
|
||||
esc((mem && mem.name) || "n/a") +
|
||||
"; suggested topics opted in. " +
|
||||
"Nothing is auto-picked without a confirm here — " +
|
||||
"you can remove any on the right.</p>";
|
||||
load();
|
||||
} else if (r.status === 200 && r.body.confidence ===
|
||||
"ambiguous") {
|
||||
c.innerHTML = "<p class='warning'>" + esc(r.body.note ||
|
||||
"Multiple districts match this ZIP. Confirm one:") +
|
||||
"</p>" +
|
||||
r.body.districts.map(function (d) {
|
||||
var mem = d.assembly_member || d.senate_member;
|
||||
return "<div class='candidate'>" +
|
||||
"<span class='who'>" + esc(d.district_id) +
|
||||
" · " + esc((mem && mem.name) || "unknown") +
|
||||
" <span class='dc'>" +
|
||||
esc((mem && mem.party) || "") + "</span></span>" +
|
||||
"<button type='button' class='confirm-btn ghost' " +
|
||||
"data-dc='" + esc(d.district_id) +
|
||||
"' data-leg='" +
|
||||
esc((mem && mem.legislator_id) || "") + "'>" +
|
||||
"Watch " + esc(d.district_id) + "</button></div>";
|
||||
}).join("");
|
||||
wire(c, "resolve-ambiguous");
|
||||
} else {
|
||||
// zero / out-of-state / other → §6.4 manual fallback
|
||||
var msg = r.body && r.body.error
|
||||
? r.body.error.message
|
||||
: "Could not match that ZIP.";
|
||||
c.innerHTML =
|
||||
"<p class='warning'>" + esc(msg) + "</p>" +
|
||||
"<input type='text' id='manual-dc' placeholder=" +
|
||||
"'Pick a district by hand, e.g. A-14'>" +
|
||||
"<button type='button' id='btn-manual' " +
|
||||
"class='ghost'>Confirm</button>";
|
||||
el("btn-manual").onclick = function () {
|
||||
var dc = el("manual-dc").value.trim();
|
||||
if (dc) resolveManual(dc);
|
||||
};
|
||||
}
|
||||
});
|
||||
state.prefs = state.prefs || {};
|
||||
}
|
||||
|
||||
function resolveManual(dc) {
|
||||
apiFetch("/districts/resolve", { method: "POST",
|
||||
body: { district: dc } }).then(function (r) {
|
||||
if (r.status === 200) {
|
||||
var p = state.prefs || {};
|
||||
p.districts = r.body.districts;
|
||||
var mem = r.body.districts[0].assembly_member ||
|
||||
r.body.districts[0].senate_member;
|
||||
if (mem) toggleWatch(mem.legislator_id, true);
|
||||
load();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function pretty(id) {
|
||||
var m = id.match(/^(A|S)-(\d+)$/i);
|
||||
return m ? m[1] + m[2] : id;
|
||||
}
|
||||
|
||||
/* ---------- wiring ---------- */
|
||||
function wireRows(container) {
|
||||
Array.prototype.forEach.call(
|
||||
container.querySelectorAll(".why-btn"),
|
||||
function (b) {
|
||||
b.onclick = function () {
|
||||
var r = b.closest(".row").querySelector(".reason");
|
||||
if (r) r.classList.toggle("show");
|
||||
};
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
function wire(container, kind) {
|
||||
if (kind === "watch") {
|
||||
Array.prototype.forEach.call(
|
||||
container.querySelectorAll(".unwatch-btn"),
|
||||
function (b) {
|
||||
b.onclick = function () {
|
||||
toggleWatch(b.getAttribute("data-leg"), false);
|
||||
};
|
||||
}
|
||||
);
|
||||
}
|
||||
if (kind === "topic") {
|
||||
Array.prototype.forEach.call(
|
||||
container.querySelectorAll(".topic-chip"),
|
||||
function (b) {
|
||||
b.onclick = function () {
|
||||
var on = b.classList.contains("on");
|
||||
toggleTopic(b.getAttribute("data-tag"), !on);
|
||||
};
|
||||
}
|
||||
);
|
||||
}
|
||||
if (kind === "resolve-ambiguous") {
|
||||
Array.prototype.forEach.call(
|
||||
container.querySelectorAll(".confirm-btn"),
|
||||
function (b) {
|
||||
b.onclick = function () {
|
||||
var lid = b.getAttribute("data-leg");
|
||||
if (lid) toggleWatch(lid, true);
|
||||
var name = b.closest(".candidate")
|
||||
.querySelector(".who").textContent;
|
||||
el("resolve-result").innerHTML =
|
||||
"<div class='candidate'><span class='who'>" +
|
||||
esc(name.trim()) +
|
||||
"</span><span class='dc'>confirmed</span>" +
|
||||
"</div>";
|
||||
};
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function boot() {
|
||||
// controls
|
||||
el("f-chamber").onchange = function () {
|
||||
state.filters.chamber = el("f-chamber").value; load();
|
||||
};
|
||||
el("f-date").onchange = function () {
|
||||
var v = el("f-date").value;
|
||||
state.filters.upcoming = (v !== "all");
|
||||
load();
|
||||
};
|
||||
var st;
|
||||
el("f-search").oninput = function () {
|
||||
clearTimeout(st);
|
||||
st = setTimeout(function () {
|
||||
state.filters.q = el("f-search").value; load();
|
||||
}, 300);
|
||||
};
|
||||
el("f-tags").onchange = function () {
|
||||
state.filters.tags = Array.prototype.slice.call(
|
||||
el("f-tags").selectedOptions).map(function (o) {
|
||||
return o.value;
|
||||
});
|
||||
load();
|
||||
};
|
||||
el("f-reason").onchange = function () {
|
||||
state.showReason = el("f-reason").checked;
|
||||
Array.prototype.forEach.call(
|
||||
document.querySelectorAll(".reason"),
|
||||
function (r) { r.classList.toggle("show", state.showReason); });
|
||||
};
|
||||
el("btn-refresh").onclick = load;
|
||||
el("btn-signout").onclick = signOut;
|
||||
el("btn-resolve").onclick = function () {
|
||||
var z = el("zip-input").value.trim();
|
||||
if (z) resolve(z);
|
||||
};
|
||||
el("btn-setup").onclick = function () {
|
||||
var z = el("zip-setup").value.trim();
|
||||
if (z) { signIn(z + "@resident.gov"); }
|
||||
};
|
||||
// unlock CTA
|
||||
var u = el("unlocked");
|
||||
if (u) u.onclick = function () {
|
||||
var em = prompt("Sign in with a resident email (D1a demo):");
|
||||
if (em) signIn(em);
|
||||
};
|
||||
|
||||
// restore session
|
||||
if (state.token) {
|
||||
state.personal = true;
|
||||
apiFetch("/prefs").then(function (r) {
|
||||
if (r.status === 200) {
|
||||
state.prefs = r.body;
|
||||
state.personal = true;
|
||||
} else {
|
||||
state.prefs = null;
|
||||
}
|
||||
refreshUI();
|
||||
load();
|
||||
});
|
||||
}
|
||||
refreshUI();
|
||||
loadTags();
|
||||
load();
|
||||
}
|
||||
|
||||
document.addEventListener("DOMContentLoaded", boot);
|
||||
})();
|
||||
@@ -0,0 +1,125 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>California Legislature Dashboard — Upcoming Legislation</title>
|
||||
<link rel="stylesheet" href="style.css">
|
||||
<meta name="description"
|
||||
content="A monitor of upcoming California Assembly & Senate daily-file
|
||||
hearings. Anonymous overview; a logged-in resident pins their
|
||||
legislators & topics to the top.">
|
||||
</head>
|
||||
<body>
|
||||
<!-- sticky header: wordmark + nav + (logged-in) user band -->
|
||||
<header class="topbar" id="topbar">
|
||||
<a class="wordmark" href="/">CA Legislature<span>Dashboard</span></a>
|
||||
<nav class="nav" aria-label="Primary">
|
||||
<a href="#dashboard" id="nav-dashboard" class="active">Dashboard</a>
|
||||
<a href="#legislators">Legislators</a>
|
||||
<a href="#bills">Bills</a>
|
||||
</nav>
|
||||
<div class="user-band" id="user-band" hidden>
|
||||
<span id="user-band-text">ZIP — · 0 legislators · 0 topics</span>
|
||||
<a href="#prefs" id="nav-prefs">Preferences</a>
|
||||
<button type="button" id="btn-signout">Sign out</button>
|
||||
</div>
|
||||
<div class="anon-cta" id="anon-cta">
|
||||
<a href="#unlocked">Sign in</a>
|
||||
<span class="cta-value">Save your interests → sign in</span>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<main id="app">
|
||||
<!-- §9 status / context band -->
|
||||
<section class="freshness" id="freshness" aria-live="polite">
|
||||
<span id="freshness-updated">Loading…</span>
|
||||
<span id="freshness-session" class="badge"></span>
|
||||
<span id="freshness-chamber" class="note"></span>
|
||||
<button type="button" id="btn-refresh" class="ghost">Refresh</button>
|
||||
</section>
|
||||
|
||||
<!-- Controls bar -->
|
||||
<section class="controls" id="controls">
|
||||
<input type="search" id="f-search" class="search"
|
||||
placeholder="Search bills… (title, author, committee)"
|
||||
aria-label="Search bills">
|
||||
<label>Chamber
|
||||
<select id="f-chamber">
|
||||
<option value="">All</option>
|
||||
<option value="ASSEMBLY">Assembly</option>
|
||||
<option value="SENATE">Senate</option>
|
||||
</select>
|
||||
</label>
|
||||
<label>Date
|
||||
<select id="f-date">
|
||||
<option value="upcoming">Upcoming</option>
|
||||
<option value="week">This week</option>
|
||||
<option value="all">All</option>
|
||||
</select>
|
||||
</label>
|
||||
<select id="f-tags" multiple size="1" aria-label="Filter by topic"
|
||||
class="tags-filter"></select>
|
||||
<label>Why is this here?
|
||||
<input type="checkbox" id="f-reason" title="Show why each
|
||||
floated bill is in your view">
|
||||
</label>
|
||||
</section>
|
||||
|
||||
<!-- Logged-in personal controls (ZIP + watch/opt-in) -->
|
||||
<section class="personal-controls" id="personal-controls" hidden>
|
||||
<div class="zip-lookup">
|
||||
<label for="zip-input">Where do you live?</label>
|
||||
<input type="text" id="zip-input" inputmode="numeric"
|
||||
pattern="[0-9]{5}" placeholder="ZIP (e.g. 90210)">
|
||||
<button type="button" id="btn-resolve">Find
|
||||
my legislators</button>
|
||||
</div>
|
||||
<div id="resolve-result" class="resolve-result"></div>
|
||||
<div class="watch-list" id="watch-list"></div>
|
||||
<div class="topic-list" id="topic-list"></div>
|
||||
</section>
|
||||
|
||||
<!-- Personal band: floored "Relevant to you" (top N) + setup empty state -->
|
||||
<section class="band" id="personal-band" aria-label="Relevant to you">
|
||||
<h2 class="band-title">Relevant to you
|
||||
<span id="band-count" class="count"></span>
|
||||
<button type="button" id="btn-show-all" class="ghost"
|
||||
hidden>Show all N</button></h2>
|
||||
<div id="band-setup" class="empty-state" hidden>
|
||||
<h3>Pin your view</h3>
|
||||
<p>Enter your ZIP and we'll surface your legislators and the
|
||||
bills that matter to you.</p>
|
||||
<input type="text" id="zip-setup" inputmode="numeric"
|
||||
placeholder="ZIP code">
|
||||
<button type="button" id="btn-setup">Find my legislators</button>
|
||||
</div>
|
||||
<div id="band-list" class="list"></div>
|
||||
</section>
|
||||
|
||||
<!-- All upcoming legislation: the full queue, floated bills marked -->
|
||||
<section class="band" id="all-band">
|
||||
<h2 class="band-title">All upcoming legislation
|
||||
<span id="all-count" class="count"></span></h2>
|
||||
<div id="all-list" class="list"></div>
|
||||
<div id="all-empty" class="empty-state" hidden></div>
|
||||
</section>
|
||||
|
||||
<!-- aside: watchlist summary / top topics this session -->
|
||||
<aside class="aside" id="aside">
|
||||
<h3 id="aside-title">Top topics this session</h3>
|
||||
<ul id="topics-teaser" class="teaser"></ul>
|
||||
<a id="unlocked" class="unlock">Unlock your view — pin your
|
||||
legislators & topics to the top →</a>
|
||||
</aside>
|
||||
|
||||
<footer class="footer">
|
||||
<span>Provenance: CA Assembly & Senate daily file ·
|
||||
<a href="#about">About</a> · <a href="#help">Help</a> ·
|
||||
<a href="#privacy">Privacy</a> ·
|
||||
<a href="#accessibility">Accessibility</a></span>
|
||||
</footer>
|
||||
</main>
|
||||
<script src="app.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,222 @@
|
||||
/* ============================================================================
|
||||
Legislative Dashboard — visual system (DESIGN_DASHBOARD §9).
|
||||
Government-appropriate, high-contrast, accessible. One accent. No
|
||||
gradients, glass, emoji, or hero. Rhythm via scale + spacing + rules.
|
||||
============================================================================ */
|
||||
|
||||
:root {
|
||||
--ink: #1a1a1a;
|
||||
--muted: #5b5b5b;
|
||||
--paper: #ffffff;
|
||||
--surface: #f4f5f7;
|
||||
--border: #d5d9df;
|
||||
--accent: #0b5394; /* sober blue, AA on paper */
|
||||
--accent-ink: #0a3f6f;
|
||||
--success: #1e7e34; /* watched / on */
|
||||
--warn: #8a5a00;
|
||||
--danger: #b3261e;
|
||||
--asm: #2f5aa7; /* chamber chip tint only */
|
||||
--sen: #7a3b8f;
|
||||
--reason-tag: #0b5394; /* 3px bar: tag match */
|
||||
--reason-leg: #1e7e34; /* 3px bar: legislator match */
|
||||
--focus-ring: #0b5394;
|
||||
--radius: 3px;
|
||||
--gap: 14px;
|
||||
--mono: ui-monospace, "SF Mono", "Menlo", monospace;
|
||||
}
|
||||
|
||||
* { box-sizing: border-box; }
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
font-family: system-ui, -apple-system, "Segoe UI", Roboto, sans-serif;
|
||||
color: var(--ink);
|
||||
background: var(--paper);
|
||||
line-height: 1.5;
|
||||
font-size: 15px;
|
||||
}
|
||||
|
||||
/* ---- focus: visible 2px ring, never the only signal ---- */
|
||||
:focus-visible {
|
||||
outline: 2px solid var(--focus-ring);
|
||||
outline-offset: 2px;
|
||||
}
|
||||
|
||||
/* ---- topbar ---- */
|
||||
.topbar {
|
||||
position: sticky; top: 0; z-index: 10;
|
||||
display: flex; align-items: center; gap: 18px;
|
||||
padding: 10px 18px;
|
||||
background: var(--paper);
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
.wordmark { text-decoration: none; color: var(--ink);
|
||||
font-weight: 700; letter-spacing: .02em; font-size: 15px; }
|
||||
.wordmark span { color: var(--accent); }
|
||||
.nav { display: flex; gap: 14px; }
|
||||
.nav a { color: var(--muted); text-decoration: none; font-size: 14px;
|
||||
padding: 6px 2px; border-bottom: 2px solid transparent; }
|
||||
.nav a:hover { color: var(--ink); }
|
||||
.nav a.active { color: var(--accent); border-bottom-color: var(--accent); }
|
||||
|
||||
.user-band { display: flex; align-items: center; gap: 12px;
|
||||
margin-left: auto; font-size: 13px; color: var(--muted); }
|
||||
.user-band button { font-size: 13px; }
|
||||
.anon-cta { margin-left: auto; text-align: right; }
|
||||
.anon-cta a { color: var(--accent); font-weight: 600; }
|
||||
.cta-value { display: block; color: var(--muted); font-size: 12px; }
|
||||
|
||||
/* ---- freshness / status band ---- */
|
||||
.freshness {
|
||||
display: flex; align-items: center; gap: 10px;
|
||||
padding: 8px 18px; background: var(--surface);
|
||||
border-bottom: 1px solid var(--border); font-size: 13px;
|
||||
color: var(--muted);
|
||||
}
|
||||
.badge { background: var(--paper); border: 1px solid var(--border);
|
||||
padding: 1px 8px; border-radius: 10px; }
|
||||
.note { color: var(--warn); }
|
||||
|
||||
/* ---- controls ---- */
|
||||
.controls {
|
||||
display: flex; flex-wrap: wrap; align-items: center; gap: 12px;
|
||||
padding: 12px 18px; border-bottom: 1px solid var(--border);
|
||||
font-size: 13px;
|
||||
}
|
||||
.controls input, .controls select {
|
||||
font: inherit; font-size: 13px; padding: 6px 8px;
|
||||
border: 1px solid var(--border); border-radius: var(--radius);
|
||||
background: var(--paper); color: var(--ink);
|
||||
}
|
||||
.controls .search { min-width: 240px; flex: 1 1 auto; }
|
||||
.controls label { display: inline-flex; align-items: center; gap: 5px;
|
||||
color: var(--muted); }
|
||||
|
||||
/* ---- layout: main + aside ---- */
|
||||
main { max-width: 1100px; margin: 0 auto; }
|
||||
main .controls, main .freshness { max-width: none; }
|
||||
.wrap-grid { display: grid; grid-template-columns: 1fr 360px;
|
||||
gap: 20px; padding: 16px 18px; }
|
||||
@media (max-width: 860px) {
|
||||
.wrap-grid { grid-template-columns: 1fr; }
|
||||
}
|
||||
|
||||
/* ---- personal controls ---- */
|
||||
.personal-controls {
|
||||
padding: 14px 18px; border-bottom: 1px solid var(--border);
|
||||
background: var(--surface);
|
||||
}
|
||||
.zip-lookup { display: flex; align-items: end; gap: 10px; flex-wrap: wrap; }
|
||||
.zip-lookup label { display: block; font-size: 12px; color: var(--muted);
|
||||
margin-bottom: 3px; }
|
||||
.zip-lookup input { font: inherit; padding: 8px 10px; width: 160px;
|
||||
border: 1px solid var(--border); border-radius: var(--radius); }
|
||||
.resolve-result { margin-top: 10px; }
|
||||
.candidate { display: flex; align-items: center; gap: 8px;
|
||||
padding: 6px 8px; border: 1px solid var(--border);
|
||||
border-radius: var(--radius); margin: 4px 0; background: var(--paper); }
|
||||
.candidate .who { font-weight: 600; }
|
||||
.candidate .dc { color: var(--muted); font-size: 12px; }
|
||||
.warning { color: var(--danger); font-size: 13px; margin: 6px 0; }
|
||||
|
||||
.watch-list, .topic-list {
|
||||
display: flex; flex-wrap: wrap; gap: 8px; margin-top: 10px; }
|
||||
.watch-item, .topic-chip {
|
||||
display: inline-flex; align-items: center; gap: 6px;
|
||||
padding: 4px 10px; border: 1px solid var(--border);
|
||||
border-radius: 14px; font-size: 13px; background: var(--paper);
|
||||
}
|
||||
.watch-item.on, .topic-chip.on {
|
||||
border-color: var(--success); color: var(--success);
|
||||
background: #eef7f0; }
|
||||
.topic-chip .count { color: var(--muted); font-size: 11px; }
|
||||
button.chip-toggle { min-width: 120px; }
|
||||
|
||||
/* ---- bands ---- */
|
||||
.band { padding: 16px 18px; border-bottom: 1px solid var(--border); }
|
||||
.band-title { font-size: 14px; text-transform: uppercase; letter-spacing:
|
||||
0.06em; color: var(--muted); margin: 0 0 10px; }
|
||||
.band-title .count { color: var(--ink); font-weight: 700; }
|
||||
#personal-band .band-title { color: var(--accent); }
|
||||
|
||||
.day-heading {
|
||||
font-size: 13px; color: var(--muted); margin: 14px 0 6px;
|
||||
padding-bottom: 4px; border-bottom: 1px solid var(--border); }
|
||||
.day-heading b { color: var(--ink); }
|
||||
|
||||
/* ---- bill row ---- */
|
||||
.row {
|
||||
display: grid; grid-template-columns: 22px 1fr auto;
|
||||
gap: 10px; align-items: start;
|
||||
padding: 10px 10px 10px 0; border-bottom: 1px solid var(--border);
|
||||
border-left: 3px solid transparent; padding-left: 8px;
|
||||
}
|
||||
.row.relevant { background: #f7fafd; }
|
||||
.row.relevant.tag-match { border-left-color: var(--reason-tag); }
|
||||
.row.relevant.leg-match { border-left-color: var(--reason-leg); }
|
||||
.row.relevant.both { border-left-color: var(--accent); }
|
||||
.row .title { font-weight: 600; }
|
||||
.row .meta { color: var(--muted); font-size: 13px; margin-top: 2px; }
|
||||
.row .num { font-family: var(--mono); font-size: 13px; }
|
||||
.row .why { color: var(--accent); font-size: 12px; cursor: pointer; }
|
||||
.row .reason { font-size: 12px; color: var(--muted); margin-top: 4px;
|
||||
display: none; }
|
||||
.row .reason.show { display: block; }
|
||||
.chip { display: inline-block; font-size: 11px; padding: 1px 7px;
|
||||
border-radius: 10px; border: 1px solid var(--border);
|
||||
margin-right: 5px; }
|
||||
.chip.asm { background: #eef3fb; color: var(--asm); border-color:#cdd9ee; }
|
||||
.chip.sen { background: #f5eef9; color: var(--sen); border-color:#e2d2ea; }
|
||||
.chip.marker { background: var(--accent); color: #fff; border-color:
|
||||
var(--accent); }
|
||||
.tag { display: inline-block; font-size: 11px; color: var(--muted);
|
||||
border: 1px solid var(--border); border-radius: 12px;
|
||||
padding: 0 6px; margin: 0 4px 0 0; }
|
||||
.row a { color: var(--accent); text-decoration: none; }
|
||||
.row a:hover { text-decoration: underline; }
|
||||
|
||||
/* ---- empty states ---- */
|
||||
.empty-state { padding: 22px; text-align: center; color: var(--muted);
|
||||
border: 1px dashed var(--border); border-radius: var(--radius);
|
||||
background: var(--surface); }
|
||||
.empty-state h3 { color: var(--ink); margin-top: 0; }
|
||||
.empty-state input { margin: 8px 0; padding: 8px; font: inherit;
|
||||
border: 1px solid var(--border); border-radius: var(--radius); }
|
||||
.escape { color: var(--accent); cursor: pointer; }
|
||||
|
||||
/* ---- aside ---- */
|
||||
.aside { padding: 16px 0; font-size: 13px; }
|
||||
.aside h3 { font-size: 13px; text-transform: uppercase; letter-spacing:
|
||||
0.06em; color: var(--muted); }
|
||||
.teaser { list-style: none; padding: 0; margin: 0 0 14px; }
|
||||
.teaser li { padding: 4px 0; }
|
||||
.teaser .n { color: var(--muted); }
|
||||
.teaser button { margin-left: 6px; font-size: 12px; }
|
||||
.unlock { display: block; margin-top: 10px; padding: 10px;
|
||||
border: 1px solid var(--accent); color: var(--accent);
|
||||
border-radius: var(--radius); text-decoration: none; }
|
||||
.unlock:hover { background: #f7fafd; }
|
||||
|
||||
/* ---- footer ---- */
|
||||
.footer { padding: 18px; font-size: 12px; color: var(--muted);
|
||||
border-top: 1px solid var(--border); }
|
||||
.footer a { color: var(--muted); }
|
||||
|
||||
/* ---- buttons ---- */
|
||||
button { font: inherit; font-size: 13px; padding: 6px 12px;
|
||||
border: 1px solid var(--border); border-radius: var(--radius);
|
||||
background: var(--paper); color: var(--ink); cursor: pointer; }
|
||||
button:hover { background: var(--surface); }
|
||||
button.ghost { background: transparent; border-color: var(--border);
|
||||
color: var(--muted); }
|
||||
button.primary { background: var(--accent); color: #fff;
|
||||
border-color: var(--accent); }
|
||||
button.on { background: var(--success); color: #fff;
|
||||
border-color: var(--success); }
|
||||
|
||||
/* motion: minimal. a floated row may fade in once. */
|
||||
@keyframes fadein { from { opacity: 0; } to { opacity: 1; } }
|
||||
.row.relevant { animation: fadein 140ms ease-out; }
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.row.relevant { animation: none; }
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
"""HTTP layer with retry + exponential back-off and typed error handling.
|
||||
|
||||
Design goals (from t_f595542f):
|
||||
* Retry transient failures with exponential back-off, capped at 3 attempts.
|
||||
* Distinguish 404 ("no file today", EXPECTED for non-legislative days) from
|
||||
5xx (transient, retryable) from 403 (antibot challenge -> alert, don't storm).
|
||||
* Log every request (url, status, bytes, attempts, outcome) to the request log.
|
||||
* Never crash the caller on a "bad" response -- callers get typed exceptions.
|
||||
|
||||
httpx is used because the project already depends on it (0.28.x). The
|
||||
follow_redirects=True argument is httpx-native.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
|
||||
try:
|
||||
import httpx
|
||||
except ImportError as exc: # pragma: no cover
|
||||
raise SystemExit(
|
||||
"httpx is required. Install with: python3 -m pip install --user httpx"
|
||||
) from exc
|
||||
|
||||
|
||||
# ---- typed error hierarchy -------------------------------------------
|
||||
class FetchError(Exception):
|
||||
"""Base class for all fetch-layer failures."""
|
||||
|
||||
|
||||
class NotFoundError(FetchError):
|
||||
"""404. For the Senate PDF this is EXPECTED (no publication that day)."""
|
||||
|
||||
def __init__(self, url: str, status: int = 404):
|
||||
super().__init__("404 Not Found: " + url)
|
||||
self.url = url
|
||||
self.status = status
|
||||
|
||||
|
||||
class AntibotError(FetchError):
|
||||
"""403 / antibot challenge. Surface as an alert; do NOT retry-storm it."""
|
||||
|
||||
def __init__(self, url: str, status: int = 403, body: bytes = b""):
|
||||
super().__init__("antibot %d: %s" % (status, url))
|
||||
self.url = url
|
||||
self.status = status
|
||||
self.body = body
|
||||
|
||||
|
||||
class TransientError(FetchError):
|
||||
"""5xx / network fault that survived all retry attempts."""
|
||||
|
||||
def __init__(self, url, status=0, attempts=0, cause=None):
|
||||
super().__init__(
|
||||
"transient after %d attempt(s) for %s: %s"
|
||||
% (attempts, url, status or cause)
|
||||
)
|
||||
self.url = url
|
||||
self.status = status
|
||||
self.attempts = attempts
|
||||
self.cause = cause
|
||||
|
||||
|
||||
@dataclass
|
||||
class FetchResult:
|
||||
"""Successful fetch result."""
|
||||
|
||||
url: str
|
||||
status: int
|
||||
content: bytes
|
||||
attempts: int = 1
|
||||
cache_hit: bool = False
|
||||
headers: Dict[str, str] = field(default_factory=dict)
|
||||
outcome: str = "ok" # ok | not_found | antibot | transient
|
||||
|
||||
|
||||
def _now_iso():
|
||||
return datetime.datetime.now(datetime.timezone.utc).isoformat(
|
||||
timespec="seconds"
|
||||
)
|
||||
|
||||
|
||||
# ---- client ----------------------------------------------------------
|
||||
class HttpClient:
|
||||
"""Configurable GET client. `log_request` is a callback fired per attempt."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
user_agent: str,
|
||||
timeout_seconds: float = 20,
|
||||
max_retries: int = 3,
|
||||
backoff_base_seconds: float = 1.0,
|
||||
backoff_cap_seconds: float = 30.0,
|
||||
retry_status=None,
|
||||
antibot_status=None,
|
||||
log_request: Optional[Callable[[Dict[str, Any]], None]] = None,
|
||||
sleep_fn: Optional[Callable[[float], None]] = None,
|
||||
):
|
||||
self.user_agent = user_agent
|
||||
self.timeout = timeout_seconds
|
||||
self.max_retries = max(1, int(max_retries))
|
||||
self.backoff_base = backoff_base_seconds
|
||||
self.backoff_cap = backoff_cap_seconds
|
||||
self.retry_status = set(retry_status or [500, 502, 503, 504])
|
||||
self.antibot_status = set(antibot_status or [403])
|
||||
self.log_request = log_request or (lambda evt: None)
|
||||
self.sleep = sleep_fn or time.sleep
|
||||
self._client: Optional[httpx.Client] = None
|
||||
|
||||
# lifecycle
|
||||
@property
|
||||
def client(self) -> httpx.Client:
|
||||
if self._client is None:
|
||||
self._client = httpx.Client(
|
||||
headers={"User-Agent": self.user_agent},
|
||||
timeout=self.timeout,
|
||||
)
|
||||
return self._client
|
||||
|
||||
def close(self) -> None:
|
||||
if self._client is not None:
|
||||
self._client.close()
|
||||
self._client = None
|
||||
|
||||
def __enter__(self) -> "HttpClient":
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc) -> None:
|
||||
self.close()
|
||||
|
||||
# ---- public API ----
|
||||
def get(self, url: str, source: str) -> FetchResult:
|
||||
"""GET with retry + back-off.
|
||||
|
||||
Returns a FetchResult on success, or raises a typed FetchError subclass:
|
||||
NotFoundError -> 404 (never retried)
|
||||
AntibotError -> 403 / antibot (alert; no retry-storm)
|
||||
TransientError-> 5xx / network fault after all attempts
|
||||
|
||||
Emits one request-log event per attempt.
|
||||
"""
|
||||
self.last_attempts = 0
|
||||
last_status = 0
|
||||
last_exc: Optional[Exception] = None
|
||||
last_content = b""
|
||||
for attempt in range(1, self.max_retries + 1):
|
||||
self.last_attempts = attempt
|
||||
outcome, status, content, headers, exc = self._do(url)
|
||||
last_status = status
|
||||
last_exc = exc
|
||||
last_content = content or b""
|
||||
|
||||
# transient (5xx in retry list, or a network error): retry or give up
|
||||
if outcome == "transient":
|
||||
if attempt < self.max_retries:
|
||||
self._log(
|
||||
source, url, status, len(last_content), attempt,
|
||||
"retrying" if exc else "5xx_retrying",
|
||||
)
|
||||
delay = min(
|
||||
self.backoff_base * (2 ** (attempt - 1)),
|
||||
self.backoff_cap,
|
||||
)
|
||||
self.sleep(delay)
|
||||
continue
|
||||
self._log(
|
||||
source, url, status, len(last_content), attempt,
|
||||
"transient_gave_up",
|
||||
)
|
||||
raise TransientError(url, status, attempt, exc)
|
||||
|
||||
# success on 2xx/3xx (redirects are fully resolved by httpx)
|
||||
if outcome == "ok":
|
||||
self._log(source, url, status, len(content), attempt, "ok")
|
||||
return FetchResult(
|
||||
url=url,
|
||||
status=status,
|
||||
content=content,
|
||||
attempts=attempt,
|
||||
headers=headers,
|
||||
outcome="ok",
|
||||
)
|
||||
|
||||
# 404 -> definitive "no file"; never retried
|
||||
if outcome == "not_found":
|
||||
self._log(source, url, status, len(content), attempt, "not_found")
|
||||
raise NotFoundError(url, status)
|
||||
|
||||
# antibot challenge: alert, do not retry-storm
|
||||
if outcome == "antibot":
|
||||
self._log(source, url, status, len(content), attempt, "antibot")
|
||||
raise AntibotError(url, status, content)
|
||||
|
||||
raise TransientError(url, last_status, self.max_retries, last_exc)
|
||||
|
||||
def _do(self, url: str):
|
||||
"""Perform one fetch. Returns (outcome, status, content, headers, exc)."""
|
||||
try:
|
||||
resp = self.client.get(url, follow_redirects=True)
|
||||
except httpx.HTTPError as exc:
|
||||
return "transient", 0, b"", {}, exc
|
||||
|
||||
status = resp.status_code
|
||||
content = resp.content
|
||||
headers = {
|
||||
k: v
|
||||
for k, v in resp.headers.items()
|
||||
if k.lower() in ("content-type", "etag", "last-modified")
|
||||
}
|
||||
if status < 400:
|
||||
return "ok", status, content, headers, None
|
||||
if status == 404:
|
||||
return "not_found", status, content, headers, None
|
||||
if status in self.antibot_status:
|
||||
return "antibot", status, content, headers, None
|
||||
if status in self.retry_status:
|
||||
return "transient", status, content, headers, None
|
||||
# unexpected 4xx (not 404/antibot): surface it via the transient path so the
|
||||
# alerting hook can classify the exact status; don't retry-storm.
|
||||
return "transient", status, content, headers, None
|
||||
|
||||
# ---- helpers ----
|
||||
def _log(self, source, url, status, nbytes, attempt, outcome) -> None:
|
||||
self.log_request(
|
||||
{
|
||||
"ts": _now_iso(),
|
||||
"source": source,
|
||||
"url": url,
|
||||
"status": status,
|
||||
"bytes": nbytes,
|
||||
"attempt": attempt,
|
||||
"outcome": outcome,
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,334 @@
|
||||
# Orchestrator: ingest one or more chamber daily-file pages for a date.
|
||||
# Ties together the cache, HTTP client, source planner, and alerting hooks.
|
||||
#
|
||||
# ing = Ingestor("config.yaml")
|
||||
# result = ing.ingest("2026-09-02") # both chambers
|
||||
# result = ing.ingest("2026-09-02", source="assembly") # one chamber
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
from .alerts import AlertContext, fire
|
||||
from .cache import RawCache
|
||||
from .config import load_config, Config
|
||||
from .http_client import (
|
||||
AntibotError,
|
||||
FetchResult,
|
||||
HttpClient,
|
||||
NotFoundError,
|
||||
TransientError,
|
||||
)
|
||||
from .request_log import RequestLog
|
||||
from .sources import SourcePlan, build_source_plan, parse_senate_pdf_filename
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChamberResult:
|
||||
# Per-chamber outcome for one date.
|
||||
source: str
|
||||
date: str
|
||||
status: str # ok | cached | not_found | antibot | transient
|
||||
path: Optional[Path] = None
|
||||
bytes: int = 0
|
||||
attempts: int = 0
|
||||
cache_hit: bool = False
|
||||
url: Optional[str] = None
|
||||
as_of: Optional[str] = None
|
||||
note: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class IngestResult:
|
||||
date: str
|
||||
chambers: List[ChamberResult] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def all_ok(self) -> bool:
|
||||
return all(c.status in ("ok", "cached") for c in self.chambers)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"date": self.date,
|
||||
"all_ok": self.all_ok,
|
||||
"chambers": [
|
||||
{
|
||||
"source": c.source,
|
||||
"status": c.status,
|
||||
"path": str(c.path) if c.path else None,
|
||||
"bytes": c.bytes,
|
||||
"attempts": c.attempts,
|
||||
"cache_hit": c.cache_hit,
|
||||
"url": c.url,
|
||||
"as_of": c.as_of,
|
||||
"note": c.note,
|
||||
}
|
||||
for c in self.chambers
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
class Ingestor:
|
||||
# One instance per process; reuses the HTTP connection pool.
|
||||
def __init__(self, config_path=None, force=False):
|
||||
self.cfg = load_config(config_path)
|
||||
self.cache = RawCache(self.cfg.raw_dir)
|
||||
self.log = RequestLog(self.cfg.request_log)
|
||||
self.force = force
|
||||
self.http = HttpClient(
|
||||
user_agent=self.cfg.http.get("user_agent", "research-scanner/1.0"),
|
||||
timeout_seconds=self.cfg.http.get("timeout_seconds", 20),
|
||||
max_retries=self.cfg.http.get("max_retries", 3),
|
||||
backoff_base_seconds=self.cfg.http.get("backoff_base_seconds", 1.0),
|
||||
backoff_cap_seconds=self.cfg.http.get("backoff_cap_seconds", 30.0),
|
||||
retry_status=self.cfg.http.get("retry_status", [500, 502, 503, 504]),
|
||||
antibot_status=self.cfg.http.get("antibot_status", [403]),
|
||||
log_request=self._log_event,
|
||||
)
|
||||
|
||||
# -- log --
|
||||
def _log_event(self, event):
|
||||
event.setdefault(
|
||||
"log_ts",
|
||||
datetime.datetime.now(datetime.timezone.utc).isoformat(
|
||||
timespec="seconds"))
|
||||
self.log.record(event)
|
||||
|
||||
# -- main API --
|
||||
def ingest(self, date_iso, source=None, sources=None):
|
||||
names = sources or ([source] if source else self.cfg.source_names())
|
||||
result = IngestResult(date=date_iso)
|
||||
for name in names:
|
||||
if name not in self.cfg.sources:
|
||||
raise ValueError(
|
||||
"Unknown source '%s'. Known: %s"
|
||||
% (name, self.cfg.source_names()))
|
||||
cr = self._ingest_chamber(name, date_iso)
|
||||
result.chambers.append(cr)
|
||||
return result
|
||||
|
||||
# -- per-chamber --
|
||||
def _ingest_chamber(self, name, date_iso):
|
||||
cfg = self.cfg.source(name)
|
||||
plan = build_source_plan(name, date_iso, cfg)
|
||||
|
||||
# 1. Cache first -- if raw bytes are present and not forced, no network.
|
||||
if not self.force:
|
||||
cached = self.cache.get(plan.source, date_iso, plan.ext)
|
||||
if cached is not None:
|
||||
self._alert(cfg, AlertContext(
|
||||
kind="ok",
|
||||
level="info",
|
||||
message="cache HIT: serving from local cache (no network)",
|
||||
source=name,
|
||||
date=date_iso,
|
||||
url=plan.primary_url,
|
||||
extra={"as_of": plan.as_of,
|
||||
"cache_entry": str(self.cache.meta_path(plan.source, date_iso))},
|
||||
))
|
||||
return ChamberResult(
|
||||
source=name,
|
||||
date=date_iso,
|
||||
status="cached",
|
||||
path=self.cache.raw_dir / plan.source / f"{date_iso}.{plan.ext}",
|
||||
bytes=len(cached),
|
||||
attempts=0,
|
||||
cache_hit=True,
|
||||
url=plan.primary_url,
|
||||
as_of=plan.as_of,
|
||||
note="served from local cache",
|
||||
)
|
||||
|
||||
# 2. Network fetch with retry + alerting.
|
||||
try:
|
||||
fr = self._fetch(plan, cfg)
|
||||
except NotFoundError as exc:
|
||||
return self._handle_404(plan, name, cfg, exc)
|
||||
except AntibotError as exc:
|
||||
return self._handle_403(plan, name, cfg, exc)
|
||||
except TransientError as exc:
|
||||
return self._handle_transient(plan, name, cfg, exc)
|
||||
|
||||
# 3. Write to cache + alert success.
|
||||
fetched_at = datetime.datetime.now(datetime.timezone.utc).isoformat(
|
||||
timespec="seconds")
|
||||
out_path = self.cache.put(
|
||||
plan.source, date_iso, plan.ext, fr.content,
|
||||
url=fr.url,
|
||||
status=fr.status,
|
||||
fetched_at=fetched_at,
|
||||
)
|
||||
self._alert(cfg, AlertContext(
|
||||
kind="ok",
|
||||
level="info",
|
||||
message="Fetched + cached %s (%d bytes, %d attempt(s))"
|
||||
% (fr.url, len(fr.content), fr.attempts),
|
||||
source=name,
|
||||
date=date_iso,
|
||||
url=fr.url,
|
||||
extra={"status_code": fr.status, "as_of": plan.as_of},
|
||||
))
|
||||
return ChamberResult(
|
||||
source=name,
|
||||
date=date_iso,
|
||||
status="ok",
|
||||
path=out_path,
|
||||
bytes=len(fr.content),
|
||||
attempts=fr.attempts,
|
||||
cache_hit=False,
|
||||
url=fr.url,
|
||||
as_of=plan.as_of,
|
||||
)
|
||||
|
||||
# -- fetch dispatch --
|
||||
def _fetch(self, plan, cfg):
|
||||
if plan.kind == "pdf":
|
||||
return self._fetch_pdf(plan, cfg)
|
||||
return self.http.get(plan.primary_url, source=plan.source)
|
||||
|
||||
def _fetch_pdf(self, plan, cfg):
|
||||
# Step 1: fetch the live index HTML (index.html carries no antibot).
|
||||
index_fr = self.http.get(plan.primary_url, source=plan.source)
|
||||
index_text = index_fr.content.decode("utf-8", errors="replace")
|
||||
|
||||
# Step 2: find the PDF filename that matches the requested date.
|
||||
target_date = plan.date
|
||||
found = None
|
||||
for pdf_url in _iter_senate_pdf_urls(index_text, plan.pdf_url):
|
||||
ref = parse_senate_pdf_filename(pdf_url)
|
||||
if ref and ref["date_iso"] == target_date:
|
||||
found = pdf_url
|
||||
break
|
||||
if found is None:
|
||||
found = plan.pdf_url # fall back to the predicted URL
|
||||
|
||||
# Step 3: fetch the PDF itself (may hit antibot -> AntibotError).
|
||||
return self.http.get(found, source=plan.source)
|
||||
|
||||
# -- error handlers --
|
||||
def _handle_404(self, plan, name, cfg, exc):
|
||||
if cfg.get("missing_is_expected_404", False):
|
||||
# 404 on the Senate PDF is normal on non-legislative days.
|
||||
self._alert(cfg, AlertContext(
|
||||
kind="not_found",
|
||||
level="info",
|
||||
message="No publication for %s (404 expected on non-legislative days)"
|
||||
% plan.date,
|
||||
source=name,
|
||||
date=plan.date,
|
||||
url=plan.pdf_url,
|
||||
))
|
||||
return ChamberResult(
|
||||
source=name,
|
||||
date=plan.date,
|
||||
status="not_found",
|
||||
url=plan.pdf_url,
|
||||
as_of=plan.as_of,
|
||||
attempts=getattr(self.http, "last_attempts", 1),
|
||||
note="404 no publication (expected)",
|
||||
)
|
||||
# Unexpected 404 -> alert as a warning.
|
||||
self._alert(cfg, AlertContext(
|
||||
kind="not_found",
|
||||
level="warning",
|
||||
message="Unexpected 404 for %s: %s" % (name, exc.url),
|
||||
source=name,
|
||||
date=plan.date,
|
||||
url=exc.url,
|
||||
extra={"status": exc.status},
|
||||
))
|
||||
return ChamberResult(
|
||||
source=name,
|
||||
date=plan.date,
|
||||
status="not_found",
|
||||
url=exc.url,
|
||||
as_of=plan.as_of,
|
||||
attempts=getattr(self.http, "last_attempts", 1),
|
||||
note="unexpected 404",
|
||||
)
|
||||
|
||||
def _handle_403(self, plan, name, cfg, exc):
|
||||
self._alert(cfg, AlertContext(
|
||||
kind="antibot",
|
||||
level="error",
|
||||
message="Antibot 403 on %s: %s\n"
|
||||
"Mitigation: blocked-page-recovery skill or a headless browser "
|
||||
"(antibot session pre-warm)." % (name, exc.url),
|
||||
source=name,
|
||||
date=plan.date,
|
||||
url=exc.url,
|
||||
extra={
|
||||
"status": exc.status,
|
||||
"body_len": len(exc.body),
|
||||
"body_preview": exc.body[:200].decode("utf-8", errors="replace"),
|
||||
},
|
||||
))
|
||||
return ChamberResult(
|
||||
source=name,
|
||||
date=plan.date,
|
||||
status="antibot",
|
||||
url=exc.url,
|
||||
as_of=plan.as_of,
|
||||
attempts=getattr(self.http, "last_attempts", 1),
|
||||
note="antibot 403 (mitigate per blocked-page-recovery skill)",
|
||||
)
|
||||
|
||||
def _handle_transient(self, plan, name, cfg, exc):
|
||||
self._alert(cfg, AlertContext(
|
||||
kind="transient",
|
||||
level="error",
|
||||
message="Transient after %d attempt(s) for %s: %s"
|
||||
% (exc.attempts, name, exc),
|
||||
source=name,
|
||||
date=plan.date,
|
||||
url=exc.url,
|
||||
extra={"status": exc.status, "attempts": exc.attempts},
|
||||
))
|
||||
return ChamberResult(
|
||||
source=name,
|
||||
date=plan.date,
|
||||
status="transient",
|
||||
url=exc.url,
|
||||
as_of=plan.as_of,
|
||||
attempts=exc.attempts,
|
||||
note="transient after %d retries" % exc.attempts,
|
||||
)
|
||||
|
||||
# -- alert convenience --
|
||||
def _alert(self, cfg, ctx):
|
||||
fire(self.cfg.alerts, ctx)
|
||||
|
||||
# -- lifecycle --
|
||||
def close(self):
|
||||
self.http.close()
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
self.close()
|
||||
|
||||
|
||||
# -- helpers --
|
||||
|
||||
def _iter_senate_pdf_urls(index_text, fallback):
|
||||
# Extract PDF download URLs referenced by the Senate index page HTML.
|
||||
import re
|
||||
urls = re.findall(
|
||||
r'/dailyfile/publications/download/[^"<>\s]+\.PDF',
|
||||
index_text,
|
||||
re.IGNORECASE,
|
||||
)
|
||||
base = "https://www.senate.ca.gov"
|
||||
seen = set()
|
||||
out = []
|
||||
for u in urls:
|
||||
full = base + u if u.startswith("/") else u
|
||||
if full not in seen:
|
||||
out.append(full)
|
||||
seen.add(full)
|
||||
out.append(fallback)
|
||||
return out
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
"""Output sinks: JSON Lines and a flat SQLite table."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from typing import List
|
||||
|
||||
from .schema import BillAction
|
||||
|
||||
log = logging.getLogger("scraper.io")
|
||||
|
||||
# Column order for the flat SQLite table: a stable, self-describing subset of
|
||||
# the record fields. ``id`` is the content hash; text fields stay TEXT and the
|
||||
# clearly-numeric fields use INTEGER.
|
||||
_COLUMNS = [
|
||||
("id", "TEXT"),
|
||||
("date", "TEXT"),
|
||||
("house", "TEXT"),
|
||||
("source", "TEXT"),
|
||||
("record_type", "TEXT"),
|
||||
("bill_number", "TEXT"),
|
||||
("bill_id", "TEXT"),
|
||||
("session", "TEXT"),
|
||||
("chamber_origin", "TEXT"),
|
||||
("author", "TEXT"),
|
||||
("short_title", "TEXT"),
|
||||
("description", "TEXT"),
|
||||
("committee", "TEXT"),
|
||||
("hearing_time", "TEXT"),
|
||||
("location", "TEXT"),
|
||||
("item_number", "INTEGER"),
|
||||
("supplement", "INTEGER"),
|
||||
("ics_url", "TEXT"),
|
||||
("agenda_id", "TEXT"),
|
||||
("raw_selector", "TEXT"),
|
||||
]
|
||||
|
||||
|
||||
def _safe_ident(name: str) -> str:
|
||||
return re.sub(r"[^0-9A-Za-z_]", "_", name or "_")
|
||||
|
||||
|
||||
def _col_names():
|
||||
return [_safe_ident(c) for c, _ in _COLUMNS]
|
||||
|
||||
|
||||
def write_jsonl(records, path, fill_id=True, include_nulls=False):
|
||||
"""Write records as JSON Lines (one JSON object per line).
|
||||
|
||||
Returns the number of records written. A record that cannot be serialised
|
||||
is logged and skipped rather than aborting the batch.
|
||||
"""
|
||||
written = 0
|
||||
os.makedirs(os.path.dirname(os.path.abspath(path)) or ".", exist_ok=True)
|
||||
with open(path, "w", encoding="utf-8", newline="\n") as fh:
|
||||
for rec in records:
|
||||
try:
|
||||
data = rec.to_dict(fill_id=fill_id, include_nulls=include_nulls)
|
||||
line = json.dumps(data, ensure_ascii=False, sort_keys=True)
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("skip record that cannot be serialised: %s", e)
|
||||
continue
|
||||
fh.write(line)
|
||||
fh.write("\n")
|
||||
written += 1
|
||||
log.info("wrote %d records to %s", written, path)
|
||||
return written
|
||||
|
||||
|
||||
def write_sqlite(records, path, table="daily_file_records",
|
||||
create_if_missing=True):
|
||||
"""Persist records to a flat SQLite table. Returns the number written.
|
||||
|
||||
Creates the table and database file when needed so the CLI ``--sqlite`` flag
|
||||
is self-sufficient. Existing rows are not replaced (plain INSERT).
|
||||
"""
|
||||
import sqlite3
|
||||
|
||||
col_list = ", ".join(_col_names())
|
||||
col_defs = ", ".join(f'{_safe_ident(c)} {t}' for c, t in _COLUMNS)
|
||||
safe_table = _safe_ident(table)
|
||||
|
||||
os.makedirs(os.path.dirname(os.path.abspath(path)) or ".", exist_ok=True)
|
||||
conn = sqlite3.connect(path)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
if create_if_missing:
|
||||
cur.execute(f"CREATE TABLE IF NOT EXISTS {safe_table} ({col_defs})")
|
||||
cur.execute(f"PRAGMA table_info({safe_table})")
|
||||
existing = {row[1] for row in cur.fetchall()}
|
||||
used = [(_safe_ident(c), c) for c, _ in _COLUMNS if _safe_ident(c) in existing]
|
||||
names = [s for s, _ in used]
|
||||
placeholders = ", ".join(["?"] * len(names))
|
||||
insert_cols = ", ".join(names)
|
||||
|
||||
written = 0
|
||||
for i, rec in enumerate(records):
|
||||
data = rec.to_dict(fill_id=True, include_nulls=False)
|
||||
values = []
|
||||
for safe, raw in used:
|
||||
val = data.get(raw)
|
||||
if val is None:
|
||||
values.append(None)
|
||||
elif safe in ("item_number", "supplement"):
|
||||
try:
|
||||
values.append(int(val))
|
||||
except (TypeError, ValueError):
|
||||
values.append(None)
|
||||
else:
|
||||
values.append(val)
|
||||
try:
|
||||
cur.execute(
|
||||
f"INSERT INTO {safe_table} ({insert_cols}) VALUES ({placeholders})",
|
||||
values)
|
||||
written += 1
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("skip sqlite insert for record #%d: %s", i, e)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
log.info("wrote %d rows to %s::%s", written, path, table)
|
||||
return written
|
||||
+681
@@ -0,0 +1,681 @@
|
||||
"""``scraper.llm`` - spec §4 LLM analysis: subject-matter tag clouds.
|
||||
|
||||
Tag every bill with subject-matter tags via a *configurable* Ollama endpoint and
|
||||
model, persisting the NEW information into a ``bill_tags`` table and producing a
|
||||
tag cloud (``out/tag_cloud.json``). The endpoint + model are NEVER hardcoded --
|
||||
they come from ``config.yaml``'s ``llm:`` section (``Config.llm``).
|
||||
|
||||
Frozen contracts honoured (task t_950780e0 / parent t_21cd58a9):
|
||||
|
||||
* Input bills are loaded from the flat ``daily_file_records`` table
|
||||
(``scraper.io`` / ``scraper.schema.BillAction``) -- reused, never redefined.
|
||||
* The Ollama HTTP request uses the stdlib ``urllib`` (no new dependency) and is
|
||||
routed through an injectable ``post`` callable so the ollama path is unit
|
||||
testable with a canned response and *no network*.
|
||||
* **Deterministic degradation**: when ``llm.enabled`` is false, the endpoint is
|
||||
unreachable, or ``llm.model`` is empty, the run falls back to a built-in
|
||||
deterministic tagger (keyword / short_title extraction, **zero network**) so
|
||||
analysis never blocks the build. The active path is logged and recorded on
|
||||
every ``bill_tags`` row via its ``generated_by`` column
|
||||
(''fallback'' vs ``'ollama:<model>'``).
|
||||
|
||||
The ``bill_tags`` table is additive (CREATE ... IF NOT EXISTS + INSERT OR IGNORE
|
||||
on a natural key), mirrors the relational lane's style (``scraper.relate_*``),
|
||||
and never alters the preserved flat ``daily_file_records`` contract.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import Any, Callable, Dict, List, Optional, Sequence
|
||||
|
||||
from . import relational
|
||||
|
||||
log = logging.getLogger("scraper.llm")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: New table storing the tag analysis (the "new info").
|
||||
TAG_TABLE = "bill_tags"
|
||||
|
||||
#: generated_by sentinel for the deterministic built-in tagger.
|
||||
FALLBACK = "fallback"
|
||||
|
||||
#: Default HTTP timeout (seconds) when cfg does not supply one.
|
||||
DEFAULT_TIMEOUT = 30
|
||||
|
||||
#: Canonical Ollama endpoints, tried in order (OpenAI-compatible then native).
|
||||
_ENDPOINTS = (
|
||||
"/v1/chat/completions", # OpenAI-compatible (per spec §4)
|
||||
"/api/generate", # Ollama native
|
||||
)
|
||||
|
||||
#: Boilerplate lead-ins / filler drained from a California bill short_title
|
||||
#: before its subject matter is extracted. These carry no subject signal.
|
||||
_BOILERPLATE = (
|
||||
r"^an act relating to\b",
|
||||
r"^an act to\b",
|
||||
r"^relating to\b",
|
||||
r"^an act\b",
|
||||
r"\band making an appropriation therefor\b",
|
||||
r"\bmaking an appropriation therefor\b",
|
||||
r"\band declaring the urgency thereof(?:(?:,?\s*to take effect immediately\b)?),?",
|
||||
r"\bdeclaring the urgency thereof(?:(?:,?\s*to take effect immediately\b)?),?",
|
||||
r"\bto take effect immediately\b",
|
||||
r"\btherefor\b",
|
||||
r"^[,\s]+",
|
||||
r"[,\.\s]+$",
|
||||
)
|
||||
|
||||
# Words that carry no subject-matter signal, removed from extracted terms.
|
||||
_FILLER = {
|
||||
"an", "the", "to", "of", "and", "for", "relating", "act", "making",
|
||||
"appropriation", "therefor", "urgency", "thereof", "immediately",
|
||||
"taking", "effect", "by", "from", "with", "in", "on", "as", "each",
|
||||
"every", "a", "or", "shall", "its", "their", "his", "her", "which",
|
||||
"such", "also", "including", "regarding",
|
||||
}
|
||||
|
||||
#: A small, curated California subject taxonomy that maps regex cues found in
|
||||
#: real bill text to canonical, human-readable tags. Used by the deterministic
|
||||
#: tagger to cluster bills into a clean tag cloud. Every tag is anchored in a
|
||||
#: real substring of a bill (never invented): a domain tag only appears when its
|
||||
#: cue is actually present in the bill's text.
|
||||
TAXONOMY = [
|
||||
((r"solid waste|recycl|landfill|waste"), "solid waste & recycling"),
|
||||
((r"water"), "water"),
|
||||
((r"housing|residential rental|foreclosure|eviction"), "housing"),
|
||||
((r"\bhealth\b|health care|health insurance|medicare|medicaid"
|
||||
r"|hospital|pharm"), "health care"),
|
||||
((r"mental health|mental ill|psychiatr|psycholog"), "mental health"),
|
||||
((r"educat|k-?12|community college|universit|school"), "education"),
|
||||
((r"tax|taxation|levy|revenue|fiscal"), "taxation"),
|
||||
((r"employment|labor\b|labour|wage|workforce|unemployment"), "employment"),
|
||||
((r"public resource|environment|pollution|emission|climate"
|
||||
r"|california dream|zero emission|renewable|conserv"), "environment"),
|
||||
((r"prison|penal|incarcerat|parole|probation|juvenile justice"
|
||||
r"|sentenc|criminal justice"), "criminal justice"),
|
||||
((r"juvenile|child|childhood|foster|adoption|dependency"), "child welfare"),
|
||||
((r"transit|rail|highway|freeway|traffic|bikeway|pedestrian"
|
||||
r"|public transport"), "public transit"),
|
||||
((r"firearm|gun|ammunition|gun control"), "firearms"),
|
||||
((r"budget|appropriation|funding|treasury|expenditure"), "budget"),
|
||||
((r"senior|elderly|\bagged\b"), "seniors"),
|
||||
((r"insurance|insuranc"), "insurance"),
|
||||
((r"election|voter|ballot|legislative apportion|districting"), "elections"),
|
||||
((r"immigrat|undocumented|asylum|deportat"), "immigration"),
|
||||
((r"privacy|personal data|data protection|cyber|surveillance"), "data privacy"),
|
||||
((r"small business|small enterprise|start ?up"), "small business"),
|
||||
((r"agricultur|farm|farmers|crop"), "agriculture"),
|
||||
((r"pension|retiree|retirement|public employee s fund"), "pensions"),
|
||||
((r"homeless|homelessness"), "homelessness"),
|
||||
((r"\bsafety\b|security|protectiv|guarding"), "public safety"),
|
||||
((r"energy|electric|utility|power|solar"), "energy"),
|
||||
((r"technology|internet|digital|online|software"), "technology"),
|
||||
((r"maritime|harbor|port|fishing|ocean|fish"), "maritime"),
|
||||
]
|
||||
_TAXONOMY_RE = [(re.compile(pat, re.I), tag) for pat, tag in TAXONOMY]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bill loading (from the preserved flat table / BillAction)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def load_bills(path: str = relational.DEFAULT_DB,
|
||||
house: Optional[str] = None) -> List[Dict[str, Any]]:
|
||||
"""Load distinct bills from the preserved flat ``daily_file_records`` table.
|
||||
|
||||
Honors the frozen contract: input == rows written by ``scraper.io`` /
|
||||
``BillAction``. Rows are deduped on ``bill_id`` (a bill can appear under
|
||||
multiple agenda items / dates) so the tag cloud is per-bill, not per-row.
|
||||
|
||||
:param path: SQLite path (default: ``db/daily_file.sqlite``).
|
||||
:param house: optional publish-house filter (e.g. ``'assembly'``); matches
|
||||
either the publishing ``house`` or the bill's ``chamber_origin``.
|
||||
"""
|
||||
conn = relational.connect(path, foreign_keys=False)
|
||||
try:
|
||||
sql = ("SELECT DISTINCT bill_id, short_title, description, author, "
|
||||
"session, chamber_origin, house FROM " + relational.FLAT_TABLE
|
||||
+ " WHERE bill_id IS NOT NULL AND bill_id != ''")
|
||||
rows = conn.execute(sql).fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
bills: List[Dict[str, Any]] = []
|
||||
seen = set()
|
||||
for r in rows:
|
||||
bid = r["bill_id"]
|
||||
if bid in seen:
|
||||
continue
|
||||
seen.add(bid)
|
||||
d = {
|
||||
"bill_id": bid,
|
||||
"short_title": r["short_title"],
|
||||
"description": r["description"],
|
||||
"author": r["author"],
|
||||
"session": r["session"],
|
||||
"chamber_origin": r["chamber_origin"],
|
||||
"house": r["house"],
|
||||
}
|
||||
if house:
|
||||
h = house.lower()
|
||||
if (d["house"] or "").lower() != h and \
|
||||
(d["chamber_origin"] or "").lower() != h:
|
||||
continue
|
||||
bills.append(d)
|
||||
# Deterministic order (by bill_id) so fallback output is reproducible.
|
||||
bills.sort(key=lambda b: b["bill_id"])
|
||||
return bills
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Deterministic fallback tagger (ZERO network)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _clean_subject(title: Optional[str]) -> List[str]:
|
||||
"""Drain boilerplate from a CA short_title and return its subject phrases.
|
||||
|
||||
E.g. "An act relating to solid waste, and making an appropriation
|
||||
therefor." -> ["solid waste"].
|
||||
"""
|
||||
if not title:
|
||||
return []
|
||||
s = title.strip().lower()
|
||||
for pat in _BOILERPLATE:
|
||||
s = re.sub(pat, " ", s)
|
||||
# Split remaining text on comma / conjunction into fragments.
|
||||
fragments = re.split(r"\s*,\s*|\s+and\s+", s)
|
||||
out: List[str] = []
|
||||
for frag in fragments:
|
||||
frag = re.sub(r'^["\s]+|["\s]+$', "", frag.strip())
|
||||
if not frag or frag in _FILLER:
|
||||
continue
|
||||
word = re.sub(r"[^a-z0-9' ]+", "", frag)
|
||||
token = " ".join(t for t in word.split() if t and t not in _FILLER)
|
||||
if token:
|
||||
out.append(token)
|
||||
return out
|
||||
|
||||
|
||||
def _title_tag(fragments: List[str]) -> List[str]:
|
||||
"""Turn cleaned subject fragments into tidy, distinct title-derived tags."""
|
||||
tags: List[str] = []
|
||||
seen = set()
|
||||
for frag in fragments:
|
||||
# Keep words; drop stray single letters / numerics-as-noise but keep
|
||||
# meaningful numeric qualifiers (e.g. "2025").
|
||||
words = frag.split()
|
||||
norm = " ".join(words).strip()
|
||||
if not norm:
|
||||
continue
|
||||
key = norm
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
tags.append(_title_case(norm))
|
||||
return tags
|
||||
|
||||
|
||||
def _title_case(s: str) -> str:
|
||||
return " ".join(w[:1].upper() + w[1:] for w in s.split() if w)
|
||||
|
||||
|
||||
def tag_bill_fallback(bill: Dict[str, Any],
|
||||
max_tags: int = 8) -> List[tuple]:
|
||||
"""Deterministic subject-matter tags for one bill. Zero network.
|
||||
|
||||
Returns a list of ``(tag, source)`` tuples, deduped and capped to
|
||||
``max_tags``. Sources are ``'short_title'`` and/or ``'title+taxonomy'``.
|
||||
Nothing is invented: every tag derives from a real substring of the bill's
|
||||
text or from a taxonomy cue that is literally present in that text.
|
||||
"""
|
||||
seen: Dict[str, str] = {}
|
||||
short = bill.get("short_title") or ""
|
||||
desc = bill.get("description") or ""
|
||||
text = (short + " " + desc).strip()
|
||||
|
||||
# 1) Title-derived subject phrases (primary "subject matter" of the bill).
|
||||
for tag in _title_tag(_clean_subject(short)):
|
||||
# Normalise to lowercase so title-derived tags merge with the taxonomy
|
||||
# clusters (e.g. "Housing" -> "housing").
|
||||
seen.setdefault(tag.lower(), "short_title")
|
||||
|
||||
# 2) Taxonomy clusters - only when the real cue is present in the text.
|
||||
for rx, tag in _TAXONOMY_RE:
|
||||
if rx.search(text):
|
||||
seen.setdefault(tag, "title+taxonomy")
|
||||
|
||||
# 3) Guaranteed signal: if the title yielded no phrase, fall back to the
|
||||
# title-cased raw title so the cloud still has a signal, not empty.
|
||||
if not seen:
|
||||
raw = re.sub(r"^[^A-Za-z0-9]+", "", (short or "").strip())
|
||||
key = ("bill " + raw.lower())[:60].strip() or "Unclassified bill"
|
||||
seen[key] = "short_title"
|
||||
|
||||
# Preserve deterministic insertion order; cap to max_tags.
|
||||
return [(tag, src) for tag, src in list(seen.items())[:max_tags]]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Ollama client (OpenAI-compatible / native, stdlib urllib, injectable)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _build_prompt(bill: Dict[str, Any], max_tags: int) -> str:
|
||||
title = bill.get("short_title") or ""
|
||||
desc = bill.get("description") or ""
|
||||
return (
|
||||
"You are tagging a California legislative bill by subject matter.\n"
|
||||
"Bill: " + (bill.get("bill_number") or
|
||||
bill.get("bill_id") or "") + "\n"
|
||||
"Title: " + title + "\n"
|
||||
+ (("Action: " + desc + "\n") if desc else "")
|
||||
+ "Reply with ONLY a JSON array of between 2 and "
|
||||
+ str(max_tags) + " short, lowercase, snake_case subject-matter tags "
|
||||
"drawn from real content in the bill. No prose, no explanation, e.g. "
|
||||
'["housing", "residential rental", "foreclosure"].'
|
||||
)
|
||||
|
||||
|
||||
def _parse_response_payload(body: Dict[str, Any]) -> str:
|
||||
"""Extract assistant text from an OpenAI-compatible or native Ollama body."""
|
||||
if not isinstance(body, dict):
|
||||
return ""
|
||||
# OpenAI-compatible: choices[0].message.content
|
||||
choices = body.get("choices")
|
||||
if choices:
|
||||
c0 = choices[0] or {}
|
||||
content = (c0.get("message") or {}).get("content")
|
||||
if content:
|
||||
return str(content)
|
||||
# Ollama native: {"response": ...}
|
||||
if "response" in body:
|
||||
return str(body.get("response") or "")
|
||||
return str(body.get("content") or "")
|
||||
|
||||
|
||||
def _extract_tags_from_text(text: str, fallback_model: str = "",
|
||||
max_tags: int = 8) -> List[str]:
|
||||
"""Parse model output into a tag list: prefer JSON array, else delIMITED."""
|
||||
text = (text or "").strip()
|
||||
if not text:
|
||||
return []
|
||||
# Try JSON (a code-fence or bare array both work after stripping fences).
|
||||
fence = re.findall(r"```(?:json)?\s*(\[.*?\])\s*```", text, re.S)
|
||||
candidate = fence[-1] if fence else None
|
||||
if candidate is None:
|
||||
m = re.search(r"(\[[^\]]*\])", text, re.S)
|
||||
candidate = m.group(1) if m else None
|
||||
if candidate:
|
||||
try:
|
||||
arr = json.loads(candidate)
|
||||
if isinstance(arr, list):
|
||||
tags: List[str] = []
|
||||
seen = set()
|
||||
for t in arr:
|
||||
s = str(t).strip().lower()
|
||||
s = re.sub(r"[^a-z0-9& ]+", " ", s).strip()
|
||||
s = " ".join(s.split())
|
||||
if s and s not in seen and s not in _FILLER:
|
||||
seen.add(s)
|
||||
tags.append(s)
|
||||
if tags:
|
||||
return tags[:max_tags]
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
# Degrade: split on punctuation / slashes / newlines.
|
||||
raw = [p for p in re.split(r"[\n,/;]|\s{2,}|\.\s+\w+", text) if p.strip()]
|
||||
out: List[str] = []
|
||||
seen = set()
|
||||
for part in raw:
|
||||
s = re.sub(r"[^a-z0-9& ]+", " ", part.lower()).strip()
|
||||
s = " ".join(s.split())
|
||||
if s and s not in seen and s not in _FILLER and len(s) > 1:
|
||||
seen.add(s)
|
||||
out.append(s[:48])
|
||||
return out[:max_tags]
|
||||
|
||||
|
||||
def _ollama_request(endpoint: str, model: str, prompt: str,
|
||||
timeout: int, post: Optional[Callable] = None) -> str:
|
||||
"""POST to a configurable Ollama endpoint, trying compatible shapes.
|
||||
|
||||
:param post: injectable ``post(url, payload, timeout) -> payload`` callable,
|
||||
so tests feed a canned response with no network. Defaults to a
|
||||
stdlib-urllib implementation.
|
||||
"""
|
||||
base = (endpoint or "").rstrip("/")
|
||||
last_err: Optional[Exception] = None
|
||||
for path in _ENDPOINTS:
|
||||
url = base + path
|
||||
payload: Dict[str, Any]
|
||||
if path == "/v1/chat/completions":
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"stream": False,
|
||||
"temperature": 0.0,
|
||||
}
|
||||
else: # /api/generate native
|
||||
payload = {"model": model, "prompt": prompt, "stream": False}
|
||||
try:
|
||||
body = post(url, payload, timeout) if post is not None \
|
||||
else _urllib_post(url, payload, timeout)
|
||||
body = _coerce_json(body)
|
||||
text = _parse_response_payload(body)
|
||||
if text and text.strip():
|
||||
return text
|
||||
last_err = ValueError("empty Ollama response body")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
last_err = exc
|
||||
continue
|
||||
# No compatible endpoint returned text -> signal caller to fall back.
|
||||
raise RuntimeError(
|
||||
"Ollama returned no usable text (%s)" %
|
||||
(last_err if last_err else "all endpoints empty")
|
||||
)
|
||||
|
||||
|
||||
def _urllib_post(url: str, payload: Dict[str, Any],
|
||||
timeout: int) -> Any:
|
||||
"""Stdlib-urllib POST (no third-party dep). Returns parsed JSON body."""
|
||||
data = json.dumps(payload).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
url, data=data, method="POST",
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
raw = resp.read().decode("utf-8", "replace")
|
||||
return _coerce_json(raw)
|
||||
|
||||
|
||||
def _coerce_json(body: Any) -> Any:
|
||||
"""Accept a dict (mock) or a JSON str/bytes (real HTTP) and normalise."""
|
||||
if isinstance(body, (bytes, bytearray)):
|
||||
body = body.decode("utf-8", "replace")
|
||||
if isinstance(body, str):
|
||||
return json.loads(body)
|
||||
return body
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Analysis orchestration + persistence + tag cloud
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def ensure_tag_table(path: str = relational.DEFAULT_DB) -> None:
|
||||
"""Idempotently create the additive ``bill_tags`` table. Does not touch the
|
||||
preserved flat table."""
|
||||
conn = relational.connect(path, foreign_keys=False)
|
||||
try:
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS " + TAG_TABLE + " ("
|
||||
" id INTEGER PRIMARY KEY,"
|
||||
" bill_id TEXT NOT NULL,"
|
||||
" tag TEXT NOT NULL,"
|
||||
" source TEXT,"
|
||||
" generated_by TEXT NOT NULL,"
|
||||
" generated_at TEXT,"
|
||||
" UNIQUE (bill_id, tag, generated_by)"
|
||||
")"
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _generated_by(model: str) -> str:
|
||||
return "ollama:" + model if model else "ollama"
|
||||
|
||||
|
||||
def tag_bills(bills: Sequence[Dict[str, Any]], cfg: Dict[str, Any],
|
||||
post: Optional[Callable] = None) -> List[Dict[str, Any]]:
|
||||
"""Tag every bill, choosing Ollama or the deterministic fallback.
|
||||
|
||||
Path selection (deterministic degradation):
|
||||
|
||||
* ``enabled`` false -> fallback (reason 'disabled')
|
||||
* ``model`` empty -> fallback (reason 'no model configured')
|
||||
* otherwise attempt Ollama; on ANY error (unreachable / empty / bad parse)
|
||||
-> fallback (reason 'unreachable: ...'), logged, never crashing the build
|
||||
|
||||
Each bill gets a row of ``{"bill_id", "tags": [(tag, source)],
|
||||
"generated_by", "reason"}``.
|
||||
"""
|
||||
enabled = bool(cfg.get("enabled"))
|
||||
model = (cfg.get("model") or "").strip()
|
||||
max_tags = int(cfg.get("default_tags") or 8)
|
||||
endpoint = (cfg.get("endpoint") or "").strip()
|
||||
timeout = int(cfg.get("timeout_seconds") or DEFAULT_TIMEOUT)
|
||||
|
||||
use_ollama = enabled and bool(model) and bool(endpoint)
|
||||
results: List[Dict[str, Any]] = []
|
||||
ran_fallback = False
|
||||
reason = "enabled=true, model=%s" % model
|
||||
|
||||
for b in bills:
|
||||
tags: List[tuple] = []
|
||||
gen = FALLBACK
|
||||
if use_ollama:
|
||||
try:
|
||||
text = _ollama_request(endpoint, model,
|
||||
_build_prompt(b, max_tags),
|
||||
timeout, post=post)
|
||||
parsed = _parse_tags_for_bill(text, max_tags=max_tags)
|
||||
if parsed:
|
||||
tags = [(t, "ollama") for t in parsed]
|
||||
gen = _generated_by(model)
|
||||
else:
|
||||
ran_fallback = True
|
||||
tags = [(t, s) for (t, s) in tag_bill_fallback(b, max_tags)]
|
||||
gen = FALLBACK
|
||||
reason = "ollama empty -> fallback"
|
||||
except Exception as exc: # noqa: BLE001 -- degrade, never crash
|
||||
log.warning("Ollama unreachable/bad-response for %s: %s; "
|
||||
"using fallback", b.get("bill_id"), exc)
|
||||
ran_fallback = True
|
||||
tags = [(t, s) for (t, s) in tag_bill_fallback(b, max_tags)]
|
||||
gen = FALLBACK
|
||||
reason = "unreachable: %s" % (type(exc).__name__)
|
||||
else:
|
||||
if not model:
|
||||
reason = "no model configured"
|
||||
else:
|
||||
reason = "disabled"
|
||||
tags = [(t, s) for (t, s) in tag_bill_fallback(b, max_tags)]
|
||||
gen = FALLBACK
|
||||
ran_fallback = True
|
||||
|
||||
results.append({
|
||||
"bill_id": b.get("bill_id"),
|
||||
"tags": tags,
|
||||
"generated_by": gen,
|
||||
"reason": reason,
|
||||
})
|
||||
|
||||
# Report the dominant path so the log + the summary name what happened.
|
||||
if not use_ollama:
|
||||
log.info("llm path = fallback (%s)", reason)
|
||||
elif ran_fallback:
|
||||
log.info("llm path = ollama with per-bill fallback (%s)", reason)
|
||||
else:
|
||||
log.info("llm path = ollama (model=%s, endpoint=%s)",
|
||||
model, endpoint)
|
||||
return results
|
||||
|
||||
|
||||
def _parse_tags_for_bill(text: str, max_tags: int = 8) -> List[str]:
|
||||
"""Public parse hook used above so the mock test can assert it directly."""
|
||||
return _extract_tags_from_text(text, max_tags=max_tags)
|
||||
|
||||
|
||||
def persist_tags(path: str, results: Sequence[Dict[str, Any]],
|
||||
generated_at: Optional[str] = None) -> int:
|
||||
"""Persist tagged bills into ``bill_tags``. Idempotent on the natural key
|
||||
`` (bill_id, tag, generated_by)``. Returns rows written this pass."""
|
||||
ensure_tag_table(path)
|
||||
conn = relational.connect(path, foreign_keys=False)
|
||||
written = 0
|
||||
try:
|
||||
for r in results:
|
||||
for (tag, src) in r.get("tags") or []:
|
||||
cur = conn.execute(
|
||||
"INSERT OR IGNORE INTO " + TAG_TABLE +
|
||||
" (bill_id, tag, source, generated_by, generated_at) "
|
||||
"VALUES (?,?,?,?,?)",
|
||||
(str(r["bill_id"]) if r.get("bill_id") is not None else "",
|
||||
str(tag), src, r["generated_by"], generated_at))
|
||||
if cur.rowcount:
|
||||
written += 1
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
return written
|
||||
|
||||
|
||||
def build_tag_cloud(db_path: str = relational.DEFAULT_DB,
|
||||
generated_by: Optional[str] = None,
|
||||
top: Optional[int] = None) -> Dict[str, Any]:
|
||||
"""Aggregate ``bill_tags`` into a tag cloud (tag -> count, desc).
|
||||
|
||||
:param generated_by: restrict to rows whose ``generated_by`` starts with
|
||||
this (e.g. ``'fallback'`` or ``'ollama'``); ``None``
|
||||
aggregates all rows.
|
||||
:param top: cap the returned tag list to the N most frequent.
|
||||
"""
|
||||
conn = relational.connect(db_path, foreign_keys=False)
|
||||
try:
|
||||
ensure_tag_table(db_path)
|
||||
if generated_by:
|
||||
rows = conn.execute(
|
||||
"SELECT tag, COUNT(*) AS n, COUNT(DISTINCT bill_id) AS bills "
|
||||
"FROM " + TAG_TABLE +
|
||||
" WHERE generated_by LIKE ? GROUP BY tag ORDER BY n DESC, tag",
|
||||
(generated_by + "%",)).fetchall()
|
||||
total_bills = conn.execute(
|
||||
"SELECT COUNT(DISTINCT bill_id) FROM " + TAG_TABLE +
|
||||
" WHERE generated_by LIKE ?", (generated_by + "%",)).fetchone()[0]
|
||||
else:
|
||||
rows = conn.execute(
|
||||
"SELECT tag, COUNT(*) AS n, COUNT(DISTINCT bill_id) AS bills "
|
||||
"FROM " + TAG_TABLE + " GROUP BY tag ORDER BY n DESC, tag"
|
||||
).fetchall()
|
||||
total_bills = conn.execute(
|
||||
"SELECT COUNT(DISTINCT bill_id) FROM " + TAG_TABLE
|
||||
).fetchone()[0]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
tags = [{"tag": r["tag"], "count": int(r["n"]), "bills": int(r["bills"])}
|
||||
for r in rows]
|
||||
if top is not None:
|
||||
tags = tags[:top]
|
||||
return {
|
||||
"tag_count": len(tags),
|
||||
"bill_count": total_bills,
|
||||
"generated_by": generated_by,
|
||||
"tags": tags,
|
||||
}
|
||||
|
||||
|
||||
def write_tag_cloud(cloud: Dict[str, Any], out_path: str) -> int:
|
||||
"""Write the tag cloud to JSON. Returns the number of tags written.
|
||||
|
||||
The file is diff-stable between runs (tags ordered desc by count, then tag).
|
||||
"""
|
||||
os.makedirs(os.path.dirname(os.path.abspath(out_path)) or ".", exist_ok=True)
|
||||
with open(out_path, "w", encoding="utf-8") as fh:
|
||||
json.dump(cloud, fh, ensure_ascii=False, indent=2, sort_keys=False)
|
||||
fh.write("\n")
|
||||
return int(cloud.get("tag_count", 0))
|
||||
|
||||
|
||||
def analyze(db_path: str = relational.DEFAULT_DB,
|
||||
out_path: Optional[str] = None,
|
||||
cfg: Optional[Dict[str, Any]] = None,
|
||||
post: Optional[Callable] = None,
|
||||
house: Optional[str] = None,
|
||||
generated_at: Optional[str] = None,
|
||||
top: Optional[int] = None,
|
||||
rebuild: bool = False) -> Dict[str, Any]:
|
||||
"""Full ``analyze`` pass: load bills, tag them, persist, build + write cloud.
|
||||
|
||||
Returns a summary:: {bills_tagged, unique_tags, generated_by, reason,
|
||||
cloud_path, top_tags, rows_written, cloud}. The dominant path is in
|
||||
``generated_by`` / ``reason`` (both log the fallback decision).
|
||||
"""
|
||||
cfg = cfg or {}
|
||||
bills = load_bills(db_path, house=house)
|
||||
results = tag_bills(bills, cfg, post=post)
|
||||
|
||||
if rebuild:
|
||||
_drop_tag_rows(db_path, keep_by=None)
|
||||
|
||||
ensure_tag_table(db_path)
|
||||
written = persist_tags(db_path, results, generated_at=generated_at)
|
||||
|
||||
# Determine the dominant generated_by for the cloud + reporting.
|
||||
dominant = _dominant_generated_by(results)
|
||||
cloud = build_tag_cloud(db_path, generated_by=None, top=top)
|
||||
|
||||
if out_path:
|
||||
write_tag_cloud(cloud, out_path)
|
||||
cloud_path = out_path
|
||||
else:
|
||||
cloud_path = None
|
||||
|
||||
reason = results[0]["reason"] if results else "no bills"
|
||||
return {
|
||||
"path": "fallback" if dominant == FALLBACK else "ollama",
|
||||
"generated_by": dominant,
|
||||
"reason": reason,
|
||||
"bills_tagged": len(results),
|
||||
"unique_tags": cloud["tag_count"],
|
||||
"rows_written": written,
|
||||
"cloud": cloud,
|
||||
"cloud_path": cloud_path,
|
||||
"out_path": out_path,
|
||||
"top_tags": cloud["tags"][: (top or 10)],
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internals: rebuild + dominance
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _drop_tag_rows(path: str, keep_by: Optional[str] = None) -> None:
|
||||
"""Clear ``bill_tags`` rows (optionally keep rows for one generated_by)."""
|
||||
ensure_tag_table(path)
|
||||
conn = relational.connect(path, foreign_keys=False)
|
||||
try:
|
||||
if keep_by:
|
||||
conn.execute("DELETE FROM " + TAG_TABLE +
|
||||
" WHERE generated_by NOT LIKE ?", (keep_by + "%",))
|
||||
else:
|
||||
conn.execute(
|
||||
"DELETE FROM " + TAG_TABLE +
|
||||
" WHERE 1=1"
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _dominant_generated_by(results: Sequence[Dict[str, Any]]) -> str:
|
||||
"""Which path won: 'fallback' if any bill fell back, else the ollama model."""
|
||||
counts: Dict[str, int] = {}
|
||||
for r in results:
|
||||
gb = r["generated_by"]
|
||||
counts[gb] = counts.get(gb, 0) + 1
|
||||
if not counts:
|
||||
return FALLBACK
|
||||
if counts.get(FALLBACK, 0) > 0:
|
||||
return FALLBACK
|
||||
first = next(iter(counts))
|
||||
return first
|
||||
@@ -0,0 +1,86 @@
|
||||
# standalone parse subcommand for the parser lane
|
||||
# The shared entry point (`scraper/__main__.py` -> `cli.py`) is jointly owned
|
||||
# by the ingestion task (registers `ingest`/`list`/`check`) and this task
|
||||
# (`parse`). To avoid clobbering the sibling's files, the parse CLI lives in
|
||||
# this module and the shared dispatcher wires it in additively: from
|
||||
# .parse_cli import register, handle_parse; then register(sub) and
|
||||
# handlers['parse'] = handle_parse. Direct invocation: python -m
|
||||
# scraper.parse_main parse --input ...
|
||||
|
||||
from __future__ import annotations
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
from . import parsers
|
||||
from .io import write_jsonl, write_sqlite
|
||||
from .schema import ASSEMBLY_DAILY_FILE, SENATE_DAILY_FILE_PDF
|
||||
|
||||
log = logging.getLogger("scraper.parse")
|
||||
|
||||
|
||||
def _already_registered(sub, name):
|
||||
actions = getattr(sub, "_choices_actions", [])
|
||||
return any(getattr(a, "name", None) == name for a in actions)
|
||||
|
||||
|
||||
def register(sub):
|
||||
if _already_registered(sub, "parse"):
|
||||
return
|
||||
pp = sub.add_parser("parse", help="parse cached Daily File HTML into JSONL / SQLite",
|
||||
description="Parse cached Daily File HTML into JSONL / SQLite records.")
|
||||
pp.add_argument("--input", "-i", required=True, help="HTML file, directory, or glob")
|
||||
pp.add_argument("--output", "-o", help="JSONL output path; omit for stdout")
|
||||
pp.add_argument("--sqlite", dest="sqlite_path", help="optional SQLite output path")
|
||||
pp.add_argument("--table", default="daily_file_records", help="SQLite table name")
|
||||
pp.add_argument("--house", default="assembly", choices=["assembly", "senate"], help="publishing house")
|
||||
pp.add_argument("--source", default=ASSEMBLY_DAILY_FILE, help="source tag for each record")
|
||||
pp.add_argument("--no-hearings", action="store_true", help="omit hearing records")
|
||||
pp.add_argument("--json-include-nulls", action="store_true", help="include null fields")
|
||||
pp.add_argument("--parser", default="lxml", choices=["lxml", "html.parser"], help="backend")
|
||||
pp.set_defaults(func=handle_parse)
|
||||
|
||||
|
||||
def _configure_logging(verbose):
|
||||
level = logging.WARNING
|
||||
if verbose == 1:
|
||||
level = logging.INFO
|
||||
elif verbose >= 2:
|
||||
level = logging.DEBUG
|
||||
fmt = "%(levelname)s %(name)s: %(message)s"
|
||||
logging.basicConfig(level=level, format=fmt)
|
||||
|
||||
|
||||
def handle_parse(args):
|
||||
_configure_logging(getattr(args, "verbose", 0))
|
||||
house = args.house.upper()
|
||||
source = args.source
|
||||
if source == ASSEMBLY_DAILY_FILE and args.house == "senate":
|
||||
source = SENATE_DAILY_FILE_PDF
|
||||
records = parsers.parse(args.input, house=house, source=source,
|
||||
parser=args.parser, include_hearings=not args.no_hearings)
|
||||
out_count = 0
|
||||
if args.output:
|
||||
out_count = write_jsonl(records, args.output, include_nulls=args.json_include_nulls)
|
||||
elif not args.sqlite_path:
|
||||
for rec in records:
|
||||
print(rec.to_json())
|
||||
out_count = len(records)
|
||||
if args.sqlite_path:
|
||||
write_sqlite(records, args.sqlite_path, table=args.table)
|
||||
log.info("parsed %d record(s); jsonL=%s sqlite=%s", len(records),
|
||||
out_count if args.output else "stdout", "on" if args.sqlite_path else "off")
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
p = argparse.ArgumentParser(prog="scraper.parse", description="Standalone parse CLI (parser lane).")
|
||||
p.add_argument("-v", "--verbose", action="count", default=0, help="-v INFO, -vv DEBUG")
|
||||
sub = p.add_subparsers(dest="command", required=True)
|
||||
register(sub)
|
||||
args = p.parse_args(argv)
|
||||
return args.func(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
"""Run the parser CLI as ``python -m scraper.parse_cli parse --input ...``.
|
||||
|
||||
The shared ``python -m scraper`` entrypoint (``scraper/__main__.py``) is jointly
|
||||
owned by the ingestion task, so this module provides a parser-only entry point
|
||||
that does not depend on the sibling's ``cli.py`` / ``__init__.py``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
|
||||
from .parse_cli import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,317 @@
|
||||
"""HTML parsers: transform cached raw Daily File HTML into BillAction records.
|
||||
|
||||
Selector contracts follow DAILY_FILE_PARSE_SPEC.md section 3:
|
||||
|
||||
- Committee hearings : table.committee-hearing-table / tr.committee-hearing-details
|
||||
- Bills on the floor : table.floor-bills-table / tr.floor-bill-details
|
||||
- Bills in committee : table.committee-bills-table / tr[class~=committee-bills-details]
|
||||
|
||||
Robustness (task acceptance criteria):
|
||||
* malformed rows (missing bill link, no number, decode failures) are skipped
|
||||
with a logged warning and never abort the batch;
|
||||
* encoding issues are handled by a charset decode with fallback + logging;
|
||||
* multi-page / multi-file input is handled by iterating every *.html the
|
||||
input resolves to.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import glob
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import List, Optional
|
||||
|
||||
from .bill_id import parse_bill_id, parse_display, parse_supplement, origin_house
|
||||
from .schema import BillAction, ASSEMBLY_DAILY_FILE
|
||||
|
||||
log = logging.getLogger("scraper.parsers")
|
||||
|
||||
# ---- section selectors ---------------------------------------------------
|
||||
HEARING_SELECTOR = "table.committee-hearing-table tr.committee-hearing-details"
|
||||
FLOOR_SELECTOR = "table.floor-bills-table tr.floor-bill-details"
|
||||
COMMITTEE_SELECTOR = "table.committee-bills-table tr[class~=committee-bills-details]"
|
||||
|
||||
# Looser fallbacks in case a table class is missing but rows are still present.
|
||||
HEARING_ROW = "tr[class*=committee-hearing-details]"
|
||||
FLOOR_ROW = "tr[class*=floor-bill-details]"
|
||||
COMMITTEE_ROW = "tr[class*=committee-bills-details]"
|
||||
|
||||
_SUPP = re.compile(r"Supplemental\s+(\d+)", re.IGNORECASE)
|
||||
_DATE_SPLIT = re.compile(r"-\d+$") # strip display suffix like '-001'
|
||||
|
||||
|
||||
# --- cell helpers ---------------------------------------------------------
|
||||
def _attr_or_text(td, prefer: str = "data-sort") -> Optional[str]:
|
||||
"""Return td[prefer] when present else trimmed textContent."""
|
||||
if prefer and td is not None and td.get(prefer):
|
||||
return td.get(prefer).strip()
|
||||
return td.get_text(" ", strip=True) if td is not None else None
|
||||
|
||||
|
||||
def normalise_date(raw: Optional[str]) -> Optional[str]:
|
||||
"""'08/31/26-001' / '08/31/26' -> '2026-08-31'. Non-date -> None."""
|
||||
if not raw:
|
||||
return None
|
||||
s = _DATE_SPLIT.sub("", raw.strip()).strip()
|
||||
if not s:
|
||||
return None
|
||||
for fmt in ("%m/%d/%y", "%m/%d/%Y", "%Y-%m-%d"):
|
||||
try:
|
||||
return datetime.strptime(s, fmt).strftime("%Y-%m-%d")
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
# --- section parsers ------------------------------------------------------
|
||||
def _parse_hearings(soup, house, source, file_date=None):
|
||||
rows = soup.select(HEARING_SELECTOR) or soup.select(HEARING_ROW)
|
||||
records = []
|
||||
for i, tr in enumerate(rows):
|
||||
try:
|
||||
date_td = tr.select_one("td.committee_hearing-date")
|
||||
time_td = tr.select_one("td.committee_hearing-time")
|
||||
comm_td = tr.select_one("td.committee_hearing-name")
|
||||
loc_td = tr.select_one("td.committee_hearing-location")
|
||||
menu = tr.select_one("td.committee_hearing-menu")
|
||||
|
||||
date = normalise_date(_attr_or_text(date_td))
|
||||
committee = _attr_or_text(comm_td, "data-sort")
|
||||
ics = None
|
||||
agenda = None
|
||||
if menu is not None:
|
||||
ics_a = menu.select_one("a[href^='/api/ics/generate']")
|
||||
if ics_a:
|
||||
ics = ics_a.get("href")
|
||||
ag_a = menu.select_one("a[href^='/api/dailyfile/agenda/']")
|
||||
if ag_a:
|
||||
agenda = ag_a.get("href", "").rsplit("/", 1)[-1]
|
||||
|
||||
if not committee and date is None:
|
||||
raise ValueError("hearing row has neither committee nor date")
|
||||
|
||||
records.append(BillAction(
|
||||
date=date,
|
||||
house=house,
|
||||
source=source,
|
||||
record_type="hearing",
|
||||
committee=committee,
|
||||
hearing_time=_attr_or_text(time_td, "text") or None,
|
||||
location=_attr_or_text(loc_td, "data-sort"),
|
||||
ics_url=ics,
|
||||
agenda_id=agenda,
|
||||
raw_selector="committee-hearing-table",
|
||||
))
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("skip malformed hearing row #%d: %s", i, e)
|
||||
# backfill a missing per-row date from the file date
|
||||
for h in records:
|
||||
if not h.date and file_date:
|
||||
h.date = file_date
|
||||
return records
|
||||
|
||||
|
||||
def _extract_bill_measure(measure_td):
|
||||
"""Return (bill_number, bill_id, supplement) from a measure / number cell."""
|
||||
supplement = None
|
||||
a = measure_td.select_one("a")
|
||||
bill_id = None
|
||||
bill_number = None
|
||||
if a is not None and a.get("href"):
|
||||
m = re.search(r"bill_id=([A-Za-z0-9]+)", a["href"])
|
||||
if m:
|
||||
bill_id = m.group(1)
|
||||
dec = parse_bill_id(bill_id)
|
||||
if dec:
|
||||
bill_number = dec["display"]
|
||||
if not bill_number:
|
||||
# Fallback: the display text before any <br> (e.g. "AB 28"). Only accept
|
||||
# a clean "PREFIX N" form; anything with extra text is a malformed row
|
||||
# and is left None so the caller can skip it.
|
||||
display = a.get_text(" ", strip=True) if a is not None else measure_td.get_text(" ", strip=True)
|
||||
m2 = re.match(r"^(AB|SB|ACR|SCR|HR|SJR|ABR)\s*(\d+)$", (display or "").strip())
|
||||
if m2:
|
||||
bill_number = f"{m2.group(1)} {int(m2.group(2))}"
|
||||
bill_id = None # display only; leave bill_id None
|
||||
parsed_supp = parse_supplement(measure_td.get_text(" ", strip=True))
|
||||
if parsed_supp:
|
||||
supplement = parsed_supp
|
||||
return bill_number, bill_id, supplement
|
||||
|
||||
|
||||
def _parse_floor_bills(soup, house, source, file_date):
|
||||
rows = soup.select(FLOOR_SELECTOR) or soup.select(FLOOR_ROW)
|
||||
records = []
|
||||
for i, tr in enumerate(rows):
|
||||
try:
|
||||
measure = tr.select_one("td.floor_bill-measure")
|
||||
if measure is None:
|
||||
raise ValueError("missing floor_bill-measure cell")
|
||||
bill_number, bill_id, supplement = _extract_bill_measure(measure)
|
||||
if not (bill_number or bill_id):
|
||||
raise ValueError("no bill number/id in floor row")
|
||||
|
||||
dec = parse_bill_id(bill_id) if bill_id else None
|
||||
chamber_origin = dec["house"].upper() if dec else origin_house((bill_number or "").split(" ")[0])
|
||||
session = dec["session"] if dec else None
|
||||
|
||||
view = tr.select_one("td.floor_bill-view-details")
|
||||
description = None
|
||||
if view is not None:
|
||||
description = view.get_text(" ", strip=True) or None
|
||||
|
||||
item_td = tr.select_one("td.floor_bill-item")
|
||||
item_text = _attr_or_text(item_td, "data-sort") or _attr_or_text(item_td, "text")
|
||||
item_number = int(item_text.strip()) if item_text and item_text.strip().isdigit() else None
|
||||
|
||||
short_title = _attr_or_text(tr.select_one("td.floor_bill-subject"), "text")
|
||||
records.append(BillAction(
|
||||
date=file_date,
|
||||
house=house,
|
||||
source=source,
|
||||
record_type="bill",
|
||||
bill_number=bill_number,
|
||||
bill_id=bill_id,
|
||||
session=session,
|
||||
chamber_origin=chamber_origin,
|
||||
author=_attr_or_text(tr.select_one("td.floor_bill-author"), "data-sort"),
|
||||
short_title=short_title,
|
||||
description=description or short_title,
|
||||
location=_attr_or_text(tr.select_one("td.floor_bill-location"), "data-sort"),
|
||||
item_number=item_number,
|
||||
supplement=supplement,
|
||||
raw_selector="floor-bills-table",
|
||||
))
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("skip malformed floor row #%d: %s", i, e)
|
||||
return records
|
||||
|
||||
|
||||
def _parse_committee_bills(soup, house, source, file_date):
|
||||
rows = soup.select(COMMITTEE_SELECTOR) or soup.select(COMMITTEE_ROW)
|
||||
records = []
|
||||
for i, tr in enumerate(rows):
|
||||
try:
|
||||
num_td = tr.select_one("td.committee_bill-number")
|
||||
if num_td is None:
|
||||
raise ValueError("missing committee_bill-number cell")
|
||||
bill_number, bill_id, _ = _extract_bill_measure(num_td)
|
||||
if not (bill_number or bill_id):
|
||||
raise ValueError("no bill number/id in committee-bill row")
|
||||
|
||||
dec = parse_bill_id(bill_id) if bill_id else None
|
||||
chamber_origin = dec["house"].upper() if dec else origin_house((bill_number or "").split(" ")[0])
|
||||
short_title = _attr_or_text(tr.select_one("td.committee_bill-subject"), "text")
|
||||
records.append(BillAction(
|
||||
date=file_date,
|
||||
house=house,
|
||||
source=source,
|
||||
record_type="bill",
|
||||
bill_number=bill_number,
|
||||
bill_id=bill_id,
|
||||
session=dec["session"] if dec else None,
|
||||
chamber_origin=chamber_origin,
|
||||
author=_attr_or_text(tr.select_one("td.committee_bill-author"), "data-sort"),
|
||||
short_title=short_title,
|
||||
description=short_title,
|
||||
committee=_attr_or_text(tr.select_one("td.committee_bill-committee"), "data-sort"),
|
||||
raw_selector="committee-bills-table",
|
||||
))
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("skip malformed committee-bill row #%d: %s", i, e)
|
||||
return records
|
||||
|
||||
|
||||
def parse_html(html: str, *, house: str = "assembly",
|
||||
source: str = ASSEMBLY_DAILY_FILE, parser: str = "lxml") -> List[BillAction]:
|
||||
"""Parse a single HTML document into a list of BillAction records.
|
||||
|
||||
The date is derived from the committee-hearing table (the daily file's date)
|
||||
and applied to the bill rows, per spec 3.2 (bills carry no per-row date).
|
||||
"""
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
engine = parser if parser in ("lxml", "html.parser") else "html.parser"
|
||||
try:
|
||||
soup = BeautifulSoup(html, engine)
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.error("failed to parse document (%s): %s", type(e).__name__, e)
|
||||
return []
|
||||
|
||||
hearings = _parse_hearings(soup, house, source, file_date=None)
|
||||
file_date = next((h.date for h in hearings if h.date), None)
|
||||
for h in hearings:
|
||||
if not h.date:
|
||||
h.date = file_date
|
||||
|
||||
# Bills first (floor, then committee), then hearings — so a parsed batch
|
||||
# leads with bill records, each carrying bill_number/session/etc.
|
||||
records = []
|
||||
records.extend(_parse_floor_bills(soup, house, source, file_date))
|
||||
records.extend(_parse_committee_bills(soup, house, source, file_date))
|
||||
records.extend(hearings)
|
||||
return records
|
||||
|
||||
|
||||
def _read_text(path: str) -> str:
|
||||
"""Read a file with charset detection + an encoding fallback."""
|
||||
with open(path, "rb") as fh:
|
||||
raw = fh.read()
|
||||
head = raw[:2048].decode("utf-8", "ignore").lower()
|
||||
charset = "utf-8"
|
||||
meta = re.search(r"content-type=\"[^\"]*?charset=([a-z0-9_-]+)", head)
|
||||
if meta:
|
||||
charset = meta.group(1)
|
||||
else:
|
||||
decl = re.search(r'charset\s*=\s*["\']?\s*([a-z0-9_-]+)', head)
|
||||
if decl:
|
||||
charset = decl.group(1)
|
||||
try:
|
||||
text = raw.decode(charset)
|
||||
except (UnicodeDecodeError, LookupError):
|
||||
log.warning("bad charset %r in %s, falling back to UTF-8/replace", charset, path)
|
||||
text = raw.decode("utf-8", "replace")
|
||||
if "\ufffd" in text:
|
||||
log.warning("replacement chars detected while decoding %s", path)
|
||||
return text
|
||||
|
||||
|
||||
def _resolve_inputs(input_path: str) -> List[str]:
|
||||
"""Expand an input into HTML files: a file, a directory (all *.html), or a glob."""
|
||||
if os.path.isdir(input_path):
|
||||
return sorted(glob.glob(os.path.join(input_path, "*.html")))
|
||||
if any(ch in input_path for ch in "*?[]"):
|
||||
return sorted(glob.glob(input_path))
|
||||
return [input_path]
|
||||
|
||||
|
||||
def parse(input_path: str, *, house: str = "assembly",
|
||||
source: str = ASSEMBLY_DAILY_FILE, parser: str = "lxml",
|
||||
include_hearings: bool = True) -> List[BillAction]:
|
||||
"""Parse one or more cached HTML files into a flat list of records.
|
||||
|
||||
Robust to malformed rows (logged + skipped), bad encodings (logged +
|
||||
fallback), and multi-file/multi-page input. Never aborts the batch on a
|
||||
single bad file or row.
|
||||
"""
|
||||
files = _resolve_inputs(input_path)
|
||||
if not files:
|
||||
log.warning("no input files matched %r", input_path)
|
||||
return []
|
||||
records: List[BillAction] = []
|
||||
for path in files:
|
||||
if not os.path.exists(path):
|
||||
log.warning("input file not found: %s", path)
|
||||
continue
|
||||
try:
|
||||
html = _read_text(path)
|
||||
except OSError as e:
|
||||
log.warning("cannot read %s: %s", path, e)
|
||||
continue
|
||||
doc_records = parse_html(html, house=house, source=source, parser=parser)
|
||||
if not include_hearings:
|
||||
doc_records = [r for r in doc_records if r.record_type != "hearing"]
|
||||
records.extend(doc_records)
|
||||
return records
|
||||
@@ -0,0 +1,129 @@
|
||||
# Additive CLI for the correlation lane (task t_2736f40d, spec 2.5).
|
||||
#
|
||||
# The shared entry point `scraper/__main__.py -> scraper/cli.py` is jointly
|
||||
# owned by the ingestion lane (`ingest`/`list`/`check`) and the parser lane
|
||||
# (`parse`, via scraper/parse_cli.py). To avoid clobbering sibling files this
|
||||
# CLI lives in its own module and is wired into the shared dispatcher
|
||||
# ADDITIVELY + DEFENSIVELY: `cli.py` calls `register(sub)` inside a try/except
|
||||
# and adds its two handler entries without touching any sibling handler. The
|
||||
# `migrate`/`correlate` names do not collide with sibling subcommands.
|
||||
#
|
||||
# Direct invocation (no shared dispatcher needed):
|
||||
# python -m scraper.relate_cli migrate --db db/daily_file.sqlite
|
||||
# python -m scraper.relate_cli correlate --roster rosters.json
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
|
||||
from . import relational
|
||||
|
||||
log = __import__("logging").getLogger("scraper.relate_cli")
|
||||
|
||||
|
||||
def _already_registered(sub, *names: str) -> bool:
|
||||
# Guard: if any of our subcommands is already registered by a sibling,
|
||||
# skip registration rather than shadowing it.
|
||||
actions = getattr(sub, "_choices_actions", [])
|
||||
present = {getattr(a, "name", None) for a in actions}
|
||||
return any(n in present for n in names)
|
||||
|
||||
|
||||
def register(sub) -> None:
|
||||
"""Register the `migrate` and `correlate` subcommands (guarded).
|
||||
|
||||
Mirrors scraper/parse_cli.py::register so the pattern stays uniform across
|
||||
lanes and the shared `cli.py` can wire it in without risk.
|
||||
"""
|
||||
if _already_registered(sub, "migrate", "correlate"):
|
||||
return
|
||||
|
||||
pm = sub.add_parser(
|
||||
"migrate",
|
||||
help="create the relational schema (idempotent; flat table preserved)",
|
||||
description=("Create/upgrade the relational schema; preserves the flat "
|
||||
"daily_file_records table + adds normalized tables & view."),
|
||||
)
|
||||
pm.add_argument("--db", default=relational.DEFAULT_DB,
|
||||
help="SQLite path (default: db/daily_file.sqlite)")
|
||||
pm.add_argument("--check", action="store_true",
|
||||
help="report schema state only (no writes)")
|
||||
pm.set_defaults(func=handle_migrate)
|
||||
|
||||
pc = sub.add_parser(
|
||||
"correlate",
|
||||
help="derive bill<->committee<->member relations from the flat table",
|
||||
description=("Backfill normalized relation tables from "
|
||||
"daily_file_records. Committees with no roster source are "
|
||||
"left with empty membership (never fabricated)."),
|
||||
)
|
||||
pc.add_argument("--db", default=relational.DEFAULT_DB,
|
||||
help="SQLite path (default: db/daily_file.sqlite)")
|
||||
pc.add_argument("--roster", dest="roster", default=None,
|
||||
help="JSON file of committee rosters to backfill members")
|
||||
pc.add_argument("--rebuild", action="store_true",
|
||||
help="clear derived tables first, then backfill cleanly")
|
||||
pc.set_defaults(func=handle_correlate)
|
||||
|
||||
|
||||
def _configure_logging(verbose: int) -> None:
|
||||
import logging
|
||||
level = logging.WARNING
|
||||
if verbose == 1:
|
||||
level = logging.INFO
|
||||
elif verbose >= 2:
|
||||
level = logging.DEBUG
|
||||
logging.basicConfig(level=level,
|
||||
format="%(levelname)s %(name)s: %(message)s")
|
||||
|
||||
|
||||
def handle_migrate(args) -> int:
|
||||
# No --verbose flag on migrate; keep warnings visible so "no roster" gaps
|
||||
# (which are warnings) still surface by default.
|
||||
_configure_logging(0)
|
||||
db = args.db
|
||||
if getattr(args, "check", False):
|
||||
summary = relational.migrate(db)
|
||||
print("schema migration (check):")
|
||||
print(" db:", summary["db"])
|
||||
print(" objects:", ", ".join(summary["objects"]))
|
||||
print(" flat_rows:", summary["flat_rows"], "version:",
|
||||
summary["version"])
|
||||
return 0
|
||||
summary = relational.migrate(db)
|
||||
print("migrated schema at %s" % summary["db"])
|
||||
print(" objects:", ", ".join(summary["objects"]))
|
||||
print(" flat_rows preserved:", summary["flat_rows"])
|
||||
return 0
|
||||
|
||||
|
||||
def handle_correlate(args) -> int:
|
||||
# Verbose default OFF; the per-committee "no roster" warnings that the
|
||||
# never-fabricate rule emits should be visible, so do not suppress log.
|
||||
_configure_logging(0)
|
||||
db = args.db
|
||||
# Ensure the schema exists even if `migrate` was never run on this db.
|
||||
relational.migrate(db)
|
||||
counts = relational.correlate(db, rosters=args.roster,
|
||||
rebuild=args.rebuild, quiet=False)
|
||||
print("correlated %s" % db)
|
||||
print(" counts:", ", ".join("%s=%d" % (k, v)
|
||||
for k, v in sorted(counts.items())))
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
import argparse
|
||||
p = argparse.ArgumentParser(
|
||||
prog="scraper.relate",
|
||||
description="Relational correlation: bills <-> committees <-> members.")
|
||||
p.add_argument("-v", "--verbose", action="count", default=0,
|
||||
help="-v INFO, -vv DEBUG")
|
||||
sub = p.add_subparsers(dest="command", required=True)
|
||||
register(sub)
|
||||
args = p.parse_args(argv)
|
||||
return args.func(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,525 @@
|
||||
"""Relational core for spec §2.5 — bills ↔ committees ↔ members.
|
||||
|
||||
The rest of the pipeline persists a FLAT table (``daily_file_records`` written by
|
||||
``scraper.io.write_sqlite``). QA and the downstream UI read that flat contract,
|
||||
so this lane NEVER mutates it. Instead it ADDS a normalised relational schema
|
||||
and a flat-preserving VIEW on top of the same SQLite database, and derives the
|
||||
relations (bill↔committee, committee↔member) from the flat rows + an optional
|
||||
committee-roster source.
|
||||
|
||||
FROZEN CONTRACTS honoured (see task t_2736f40d / parent t_21cd58a9):
|
||||
|
||||
* ``daily_file_records`` stays fully intact (CREATE IF NOT EXISTS only).
|
||||
* New tables: ``bills``, ``committees``, ``members``, ``committee_members``
|
||||
(committee↔member join), ``bill_committees`` (bill↔committee join), plus a
|
||||
``schema_migrations`` ledger and a ``v_daily_file_records`` view.
|
||||
* FOREIGN KEYS are enabled on every connection; UNIQUE constraints on the
|
||||
natural keys (bill_id+session+chamber; committee name+session+chamber;
|
||||
committee+member).
|
||||
* Migration and correlation are IDEMPOTENT: CREATE IF NOT EXISTS +
|
||||
INSERT OR IGNORE on deterministic keys. Re-running adds no rows / no errors.
|
||||
* NEVER FABRICATE MEMBERS. ``committee_members`` is populated ONLY from a
|
||||
supplied roster source. When a committee seen in the daily file has no roster
|
||||
entry, it is still created (with its ``session``) but is left with an EMPTY
|
||||
member set and a WARN is logged listing which committees are missing rosters —
|
||||
we record the gap, we do not invent members.
|
||||
|
||||
This module uses only the stdlib (sqlite3 + a tiny JSON roster reader). No new
|
||||
third-party dependency is introduced.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
from typing import Any, Dict, Iterable, List, Optional
|
||||
|
||||
log = logging.getLogger("scraper.relate")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Key constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Default database path (spec: db/daily_file.sqlite), resolved relative to the
|
||||
#: repository root so it is stable regardless of cwd.
|
||||
_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
DEFAULT_DB = os.path.join(_REPO_ROOT, "db", "daily_file.sqlite")
|
||||
|
||||
FLAT_TABLE = "daily_file_records"
|
||||
|
||||
#: Schema version stamped into ``schema_migrations`` by :func:`migrate`.
|
||||
SCHEMA_VERSION = 1
|
||||
|
||||
#: The flat columns QA + downstream rely on. Imported so the flat-preserving
|
||||
#: view is byte-for-byte aligned with the contract the rest of the pipeline
|
||||
#: writes via ``scraper.io.write_sqlite``.
|
||||
try: # pragma: no cover - import guard (io is a sibling in the same package)
|
||||
from .io import _COLUMNS as FLAT_COLUMNS
|
||||
except Exception: # pragma: no cover
|
||||
# Fallback if io ever changes; keeps the module importable for unit tests.
|
||||
FLAT_COLUMNS = [
|
||||
("id", "TEXT"), ("date", "TEXT"), ("house", "TEXT"), ("source", "TEXT"),
|
||||
("record_type", "TEXT"), ("bill_number", "TEXT"), ("bill_id", "TEXT"),
|
||||
("session", "TEXT"), ("chamber_origin", "TEXT"), ("author", "TEXT"),
|
||||
("short_title", "TEXT"), ("description", "TEXT"), ("committee", "TEXT"),
|
||||
("hearing_time", "TEXT"), ("location", "TEXT"), ("item_number", "INTEGER"),
|
||||
("supplement", "INTEGER"), ("ics_url", "TEXT"), ("agenda_id", "TEXT"),
|
||||
("raw_selector", "TEXT"),
|
||||
]
|
||||
|
||||
_FLAT_COL_ORDER = [c for c, _ in FLAT_COLUMNS]
|
||||
_FLAT_COL_DEFS = ", ".join(f'"{c}" {t}' for c, t in FLAT_COLUMNS)
|
||||
|
||||
_SCHEMA_TABLES = (
|
||||
"bills", "committees", "members", "committee_members", "bill_committees",
|
||||
"schema_migrations",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Normalisation helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_SLUG_RE = re.compile(r"[^a-z0-9]+")
|
||||
|
||||
|
||||
def normalise_name(raw: Optional[str]) -> str:
|
||||
"""Title-case + trim a committee/author/member display name."""
|
||||
if not raw:
|
||||
return ""
|
||||
s = re.sub(r"\s+", " ", raw.strip())
|
||||
return s
|
||||
|
||||
|
||||
def slug(raw: Optional[str]) -> str:
|
||||
"""Deterministic match key: lowercased, non-alphanumerics -> single '-'.
|
||||
|
||||
Mirrors spec §6.1 ('Normalise by title-casing and stripping
|
||||
non-alphanumerics') so a roster's committee name resolves to the same
|
||||
committee the daily file references.
|
||||
"""
|
||||
if not raw:
|
||||
return ""
|
||||
s = _SLUG_RE.sub("-", raw.strip().lower()).strip("-")
|
||||
return s or "unknown"
|
||||
|
||||
|
||||
def session_key(raw: Optional[str]) -> str:
|
||||
"""Coalesce a NULL/empty session to a stable sentinel ('') so the
|
||||
natural-key UNIQUE constraints stay effective and re-runs idempotent."""
|
||||
return (raw or "").strip()
|
||||
|
||||
|
||||
def chamber_key(raw: Optional[str]) -> str:
|
||||
"""Normalise chamber origin/house to a stable lowercase key."""
|
||||
return (raw or "").strip().lower() or "unknown"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Connection + schema DDL
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def connect(path: str, foreign_keys: bool = True) -> sqlite3.Connection:
|
||||
"""Open the DB, enforcing FK integrity (per-connection pragma)."""
|
||||
os.makedirs(os.path.dirname(os.path.abspath(path)) or ".", exist_ok=True)
|
||||
conn = sqlite3.connect(path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
if foreign_keys:
|
||||
# Must be set on the connection *before* any DDL/DML.
|
||||
conn.execute("PRAGMA foreign_keys = ON")
|
||||
return conn
|
||||
|
||||
|
||||
def _ensure_flat_table(conn: sqlite3.Connection) -> None:
|
||||
"""Create the flat contract table ONLY if missing. Never alters it."""
|
||||
conn.execute(
|
||||
f'CREATE TABLE IF NOT EXISTS {FLAT_TABLE} ('
|
||||
f"{_FLAT_COL_DEFS})"
|
||||
)
|
||||
|
||||
|
||||
def _create_schema(conn: sqlite3.Connection) -> None:
|
||||
"""Create all normalised tables + the flat-preserving view idempotently.
|
||||
|
||||
Every object uses CREATE ... IF NOT EXISTS, so this is safe to run many
|
||||
times. Foreign keys link the relation tables to their parents.
|
||||
"""
|
||||
conn.executescript(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS bills (
|
||||
id INTEGER PRIMARY KEY,
|
||||
bill_id TEXT NOT NULL,
|
||||
bill_number TEXT,
|
||||
session TEXT NOT NULL DEFAULT '',
|
||||
chamber_origin TEXT NOT NULL DEFAULT 'unknown',
|
||||
house TEXT,
|
||||
author TEXT,
|
||||
short_title TEXT,
|
||||
first_seen TEXT,
|
||||
source TEXT,
|
||||
UNIQUE (bill_id, session, chamber_origin)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS committees (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name_key TEXT NOT NULL,
|
||||
name_raw TEXT NOT NULL,
|
||||
session TEXT NOT NULL DEFAULT '',
|
||||
chamber TEXT NOT NULL DEFAULT 'unknown',
|
||||
hearing_time TEXT,
|
||||
location TEXT,
|
||||
source TEXT,
|
||||
UNIQUE (name_key, session, chamber)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS members (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name_norm TEXT NOT NULL,
|
||||
full_name TEXT,
|
||||
chamber TEXT NOT NULL DEFAULT 'unknown',
|
||||
kind TEXT NOT NULL DEFAULT 'roster',
|
||||
source TEXT,
|
||||
UNIQUE (name_norm, chamber, kind)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS committee_members (
|
||||
id INTEGER PRIMARY KEY,
|
||||
committee TEXT NOT NULL,
|
||||
member TEXT NOT NULL,
|
||||
session TEXT NOT NULL DEFAULT '',
|
||||
chamber TEXT NOT NULL DEFAULT 'unknown',
|
||||
role TEXT,
|
||||
source TEXT,
|
||||
UNIQUE (committee, member, session, chamber)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS bill_committees (
|
||||
id INTEGER PRIMARY KEY,
|
||||
bill_id TEXT NOT NULL,
|
||||
committee TEXT NOT NULL,
|
||||
session TEXT NOT NULL DEFAULT '',
|
||||
chamber TEXT NOT NULL DEFAULT 'unknown',
|
||||
source TEXT,
|
||||
UNIQUE (bill_id, committee, session, chamber)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version INTEGER NOT NULL,
|
||||
applied_at TEXT,
|
||||
note TEXT,
|
||||
PRIMARY KEY (version)
|
||||
);
|
||||
"""
|
||||
)
|
||||
# Flat-preserving view: a drop-in mirror of the table QA/UI read. Column
|
||||
# order and row set are identical to daily_file_records, ordered
|
||||
# deterministically so the "flat contract unchanged" test is stable.
|
||||
# (Rebuild every migrate so the view tracks the current flat schema.)
|
||||
conn.execute(f"DROP VIEW IF EXISTS v_{FLAT_TABLE}")
|
||||
cols = ", ".join(f'"{c}"' for c in _FLAT_COL_ORDER)
|
||||
conn.execute(
|
||||
f"CREATE VIEW v_{FLAT_TABLE} AS SELECT {cols} FROM {FLAT_TABLE} "
|
||||
"ORDER BY date, house, bill_id, committee, id"
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def _stamp_migration(conn: sqlite3.Connection, version: int = SCHEMA_VERSION) -> None:
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO schema_migrations (version, applied_at, note) "
|
||||
"VALUES (?, ?, ?)",
|
||||
(version,
|
||||
__import__("datetime").date.today().isoformat(),
|
||||
"relational schema: bills/committees/members/committee_members/"
|
||||
"bill_committees + flat view"),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# migrate / correlate
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def migrate(path: str = DEFAULT_DB, flat_table: str = FLAT_TABLE,
|
||||
drop_flat: bool = False) -> Dict[str, Any]:
|
||||
"""Create the relational schema idempotently, preserving the flat table.
|
||||
|
||||
Safe on a fresh DB (creates everything) and on an existing DB (no-op for
|
||||
objects that already exist). Returns a small summary dict.
|
||||
"""
|
||||
conn = connect(path)
|
||||
try:
|
||||
_ensure_flat_table(conn)
|
||||
_create_schema(conn)
|
||||
_stamp_migration(conn)
|
||||
n_flat = conn.execute(f"SELECT COUNT(*) FROM {FLAT_TABLE}").fetchone()[0]
|
||||
return {"db": path, "migrated": True, "flat_rows": n_flat,
|
||||
"version": SCHEMA_VERSION,
|
||||
"objects": _list_objects(conn)}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _list_objects(conn: sqlite3.Connection) -> List[str]:
|
||||
rows = conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type IN ('table','view') "
|
||||
"ORDER BY type DESC, name").fetchall()
|
||||
return [r["name"] for r in rows]
|
||||
|
||||
|
||||
def _load_flat_rows(conn: sqlite3.Connection) -> Iterable[sqlite3.Row]:
|
||||
return conn.execute(f"SELECT * FROM {FLAT_TABLE}").fetchall()
|
||||
|
||||
|
||||
def load_rosters(path) -> Dict:
|
||||
"""Load committee rosters from a JSON file (the "create new info" path).
|
||||
|
||||
Format (a list of committee blocks)::
|
||||
|
||||
[
|
||||
{"committee": "Budget", "session": "2025-2026", "chamber": "assembly",
|
||||
"members": [
|
||||
{"name": "Roxas", "role": "Chair"},
|
||||
{"name": "Liu", "role": "Vice Chair"}
|
||||
]}
|
||||
]
|
||||
|
||||
Returns a dict keyed by ``(slug, session, chamber)`` -> block, so a
|
||||
committee that appears in distinct (session, chamber) slots keeps separate
|
||||
memberships instead of clobbering. Missing/empty file -> empty dict (caller
|
||||
then logs a WARN for the gap). A malformed file raises ValueError so the CLI
|
||||
can report it cleanly rather than silently producing no members.
|
||||
"""
|
||||
out: Dict[tuple, Dict[str, Any]] = {}
|
||||
if not path:
|
||||
return out
|
||||
with open(path, encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
blocks = data if isinstance(data, list) else data.get("committees", [])
|
||||
for blk in blocks:
|
||||
name = blk.get("committee") or blk.get("name")
|
||||
if not name:
|
||||
continue
|
||||
key = (slug(name), session_key(blk.get("session")),
|
||||
chamber_key(blk.get("chamber")))
|
||||
out[key] = {
|
||||
"name": normalise_name(name),
|
||||
"session": session_key(blk.get("session")),
|
||||
"chamber": chamber_key(blk.get("chamber")),
|
||||
"members": [
|
||||
{"name": normalise_name(m.get("name")),
|
||||
"role": m.get("role")}
|
||||
for m in (blk.get("members") or []) if m.get("name")
|
||||
],
|
||||
}
|
||||
return out
|
||||
|
||||
|
||||
def _ingest_relations(conn: sqlite3.Connection, rosters: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Derive + backfill normalised relations from the flat table.
|
||||
|
||||
* ``bills`` <- flat rows with a bill_id
|
||||
* ``committees`` <- flat rows with a committee (hearing / committee
|
||||
tables) AND committee-like authors ('Committee
|
||||
on ...' strings name a committee too)
|
||||
* ``bill_committees`` <- flat rows that carry BOTH a bill_id and a
|
||||
committee (the "reviewed by" relation)
|
||||
* ``members`` <- roster members (kind='roster') plus bill authors
|
||||
(kind='author-seed') — real names, not guesses
|
||||
* ``committee_members`` <- roster membership ONLY (never fabricated)
|
||||
|
||||
Returns a counts summary + the list of committees missing a roster.
|
||||
"""
|
||||
counts = {"bills": 0, "committees": 0, "bill_committees": 0,
|
||||
"members": 0, "committee_members": 0,
|
||||
"committees_missing_roster": 0}
|
||||
# Helper: run an INSERT OR IGNORE and report whether it changed a row
|
||||
# (rowcount == 0 means the natural key already existed -> idempotent dedup).
|
||||
def ins_added(cur) -> int:
|
||||
return 1 if cur and cur.rowcount else 0
|
||||
|
||||
seen_committees = set() # slugs we saw in the daily file
|
||||
for row in _load_flat_rows(conn):
|
||||
bill_id = row["bill_id"] if "bill_id" in row.keys() else None
|
||||
committee = row["committee"] if "committee" in row.keys() else None
|
||||
author = row["author"] if "author" in row.keys() else None
|
||||
sess = session_key(row["session"] if "session" in row.keys() else None)
|
||||
house = row["house"] if "house" in row.keys() else None
|
||||
origin = row["chamber_origin"] if "chamber_origin" in row.keys() else None
|
||||
# chamber_origin is the origin house of a bill; hearing rows have no
|
||||
# bill so chamber_origin is None -> fall back to the publishing `house`
|
||||
# (e.g. 'assembly') so the committee slot joins to its roster cleanly.
|
||||
cham = chamber_key(origin) if origin else chamber_key(house)
|
||||
seen = "daily_file"
|
||||
|
||||
# --- bills -------------------------------------------------------
|
||||
if bill_id:
|
||||
if ins_added(conn.execute(
|
||||
"INSERT OR IGNORE INTO bills "
|
||||
"(bill_id, bill_number, session, chamber_origin, house, "
|
||||
" author, short_title, first_seen, source) "
|
||||
"VALUES (?,?,?,?,?,?,?,?,?)",
|
||||
(bill_id, row["bill_number"], sess, cham, house,
|
||||
author, row["short_title"],
|
||||
row["first_seen"] if "first_seen" in row.keys()
|
||||
else None, seen))):
|
||||
counts["bills"] += 1
|
||||
|
||||
# --- committee-like author names a committee ---------------------
|
||||
author_committee = None
|
||||
if author and author.strip().lower().startswith("committee"):
|
||||
# "Committee on Budget and Fiscal Review" -> "Budget and Fiscal Review"
|
||||
author_committee = re.sub(r"^committees?\s+on\s+", "",
|
||||
author.strip(), flags=re.I) or author.strip()
|
||||
cham = chamber_key(row["chamber_origin"] if "chamber_origin"
|
||||
in row.keys() else None)
|
||||
for cname in {c for c in (committee, author_committee) if c}:
|
||||
ck = slug(cname)
|
||||
seen_committees.add((ck, sess, cham))
|
||||
if ins_added(conn.execute(
|
||||
"INSERT OR IGNORE INTO committees "
|
||||
"(name_key, name_raw, session, chamber, hearing_time, "
|
||||
" location, source) VALUES (?,?,?,?,?,?,?)",
|
||||
(ck, normalise_name(cname), sess, cham,
|
||||
row["hearing_time"] if "hearing_time" in row.keys()
|
||||
and committee else None,
|
||||
row["location"] if "location" in row.keys()
|
||||
and committee else None, seen))):
|
||||
counts["committees"] += 1
|
||||
|
||||
# --- bill <-> committee ("reviewed by") --------------------------
|
||||
if bill_id and committee:
|
||||
if ins_added(conn.execute(
|
||||
"INSERT OR IGNORE INTO bill_committees "
|
||||
"(bill_id, committee, session, chamber, source) "
|
||||
"VALUES (?,?,?,?,?)",
|
||||
(bill_id, slug(committee), sess, cham, seen))):
|
||||
counts["bill_committees"] += 1
|
||||
|
||||
# --- author -> member seed (a real legislator, not a guess) -----
|
||||
if author and not author.strip().lower().startswith("committee"):
|
||||
stem = re.sub(r"\s+et al\.$", "", author.strip())
|
||||
if re.fullmatch(r"[\w'\-]+", stem):
|
||||
if ins_added(conn.execute(
|
||||
"INSERT OR IGNORE INTO members "
|
||||
"(name_norm, full_name, chamber, kind, source) "
|
||||
"VALUES (?,?,?,?,?)",
|
||||
(slug(stem), stem, cham, "author-seed", seen))):
|
||||
counts["members"] += 1
|
||||
|
||||
# --- roster-driven committee_members (NEVER fabricated) -------------
|
||||
# `rosters` is keyed by (slug, session, chamber); populate members for each
|
||||
# roster's own slot. A roster slot may use a (session, chamber) chosen by the
|
||||
# roster source; it is not required to mirror every seen slot exactly.
|
||||
roster_slugs = {k[0] for k in rosters}
|
||||
for blk in rosters.values():
|
||||
for m in blk["members"]:
|
||||
if not m["name"]:
|
||||
continue
|
||||
if ins_added(conn.execute(
|
||||
"INSERT OR IGNORE INTO members "
|
||||
"(name_norm, full_name, chamber, kind, source) "
|
||||
"VALUES (?,?,?,?,?)",
|
||||
(slug(m["name"]), m["name"], blk["chamber"], "roster",
|
||||
"roster"))):
|
||||
counts["members"] += 1
|
||||
if ins_added(conn.execute(
|
||||
"INSERT OR IGNORE INTO committee_members "
|
||||
"(committee, member, session, chamber, role, source) "
|
||||
"VALUES (?,?,?,?,?,?)",
|
||||
(slug(blk["name"]), slug(m["name"]), blk["session"],
|
||||
blk["chamber"], m.get("role"), "roster"))):
|
||||
counts["committee_members"] += 1
|
||||
|
||||
# --- gap reporting: committees seen but with NO roster (slug level) --
|
||||
# A seen committee slot is "missing a roster" when no roster block covers
|
||||
# that slug at all -- we still created the committee row, but its member
|
||||
# set is left empty. Per the "create new info / never fabricate" rule we
|
||||
# record the gap (+WARN) instead of inventing members.
|
||||
missing = set()
|
||||
for (ck, sess, cham) in seen_committees:
|
||||
if ck not in roster_slugs:
|
||||
missing.add((ck, sess, cham))
|
||||
counts["committees_missing_roster"] = len(missing)
|
||||
for (ck, sess, cham) in sorted(missing):
|
||||
log.warning(
|
||||
"committee %r (session=%s chamber=%s) has no roster source; "
|
||||
"membership left empty (not fabricated). Provide --roster to "
|
||||
"backfill.", ck, sess or "-", cham)
|
||||
|
||||
conn.commit()
|
||||
return counts
|
||||
|
||||
|
||||
def correlate(path: str = DEFAULT_DB, rosters: Optional[str] = None,
|
||||
rebuild: bool = False, quiet: bool = False) -> Dict[str, Any]:
|
||||
"""Derive relations from the flat table into the normalised tables.
|
||||
|
||||
Idempotent by default (INSERT OR IGNORE on deterministic natural keys).
|
||||
``rebuild=True`` clears the derived join/parent tables first so a run
|
||||
reflects the current flat snapshot exactly.
|
||||
"""
|
||||
conn = connect(path)
|
||||
try:
|
||||
_ensure_flat_table(conn)
|
||||
_create_schema(conn)
|
||||
_stamp_migration(conn)
|
||||
roster_data = load_rosters(rosters) if rosters else {}
|
||||
|
||||
if rebuild:
|
||||
for t in ("bill_committees", "committee_members", "members",
|
||||
"committees", "bills"):
|
||||
conn.execute(f"DELETE FROM {t}")
|
||||
conn.commit()
|
||||
|
||||
counts = _ingest_relations(conn, roster_data)
|
||||
# Final absolute counts (post-dedup) for reporting.
|
||||
for k in ("bills", "committees", "bill_committees", "members",
|
||||
"committee_members"):
|
||||
counts[k] = conn.execute(f"SELECT COUNT(*) FROM {k}").fetchone()[0]
|
||||
if not quiet:
|
||||
log.info("correlate: %s", json.dumps(counts, sort_keys=True))
|
||||
return counts
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Demonstration / verification helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def sample_selects(path: str = DEFAULT_DB, limit: int = 5) -> Dict[str, Any]:
|
||||
"""Pull a few human-readable sample SELECTs for the completion report."""
|
||||
conn = connect(path)
|
||||
try:
|
||||
def q(sql: str, args: tuple = ()):
|
||||
return [dict(r) for r in conn.execute(sql, args).fetchall()]
|
||||
return {
|
||||
"counts": {t: conn.execute(f"SELECT COUNT(*) FROM {t}").fetchone()[0]
|
||||
for t in ("bills", "committees", "members",
|
||||
"committee_members", "bill_committees")},
|
||||
"sample_bill_committees": q(
|
||||
"SELECT bc.bill_id, c.name_raw AS committee, bc.chamber, "
|
||||
"bc.session FROM bill_committees bc "
|
||||
"JOIN committees c "
|
||||
" ON c.name_key = bc.committee "
|
||||
" AND c.chamber = bc.chamber "
|
||||
" AND c.session = bc.session LIMIT ?", (limit,)),
|
||||
"sample_committee_members": q(
|
||||
"SELECT DISTINCT c.name_raw AS committee, m.full_name AS "
|
||||
"member, cm.role, cm.chamber FROM committee_members cm "
|
||||
"JOIN committees c ON c.name_key = cm.committee "
|
||||
"JOIN members m ON m.name_norm = cm.member "
|
||||
"ORDER BY c.name_raw, m.full_name LIMIT ?", (limit,)),
|
||||
"committees_without_roster": q(
|
||||
"SELECT name_raw, chamber, session FROM committees c "
|
||||
"WHERE NOT EXISTS (SELECT 1 FROM committee_members cm "
|
||||
" WHERE cm.committee = c.name_key "
|
||||
" AND cm.chamber = c.chamber "
|
||||
" AND cm.session = c.session)"),
|
||||
}
|
||||
finally:
|
||||
conn.close()
|
||||
@@ -0,0 +1,31 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class RequestLog:
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = Path(path)
|
||||
|
||||
def record(self, event):
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(self.path, "a", encoding="utf-8") as fh:
|
||||
fh.write(json.dumps(event, sort_keys=True) + "\n")
|
||||
|
||||
@classmethod
|
||||
def read(cls, path):
|
||||
p = Path(path)
|
||||
if not p.exists():
|
||||
return []
|
||||
out = []
|
||||
with open(p, "r", encoding="utf-8") as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if line:
|
||||
try:
|
||||
out.append(json.loads(line))
|
||||
except ValueError:
|
||||
continue
|
||||
return out
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Normalised output schema for a Daily File entry.
|
||||
|
||||
One :class:`BillAction` per record, matching DAILY_FILE_PARSE_SPEC.md §7.
|
||||
Records serialise cleanly to JSON Lines and to a flat SQLite table.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from dataclasses import dataclass, fields, asdict
|
||||
from typing import Any, Optional, List
|
||||
|
||||
|
||||
# Source identifiers (spec §7). Floor / committee bills come from the Assembly
|
||||
# daily file; Senate data would arrive from the PDF path or the leginfo alt.
|
||||
ASSEMBLY_DAILY_FILE = "ASSEMBLY_DAILY_FILE"
|
||||
SENATE_DAILY_FILE_PDF = "SENATE_DAILY_FILE_PDF"
|
||||
LEGINFO_BILL_RESULTS = "LEGINFO_BILL_RESULTS"
|
||||
|
||||
# House = the publishing chamber; chamber_origin = the bill's origin (spec §8 #7).
|
||||
HOUSE = "assembly" # default house for the Assembly Daily File HTML source
|
||||
|
||||
|
||||
@dataclass
|
||||
class BillAction:
|
||||
"""A single normalised daily-file record.
|
||||
|
||||
Field names mirror the §7 JSON schema. ``id`` is an optional deterministic
|
||||
content hash (sha1 of ``house|bill_number|date|committee``); leave it ``None``
|
||||
to auto-compute on serialise.
|
||||
"""
|
||||
|
||||
# --- required identity / context -------------------------------------
|
||||
date: Optional[str] = None # "2026-08-31" (normalised ISO) when available
|
||||
house: str = HOUSE # ASSEMBLY / SENATE (publishing house)
|
||||
source: str = ASSEMBLY_DAILY_FILE # source tag (see module constants)
|
||||
record_type: str = "bill" # "bill" | "hearing"
|
||||
# --- bill fields ------------------------------------------------------
|
||||
bill_number: Optional[str] = None # "AB 28" (4-digit normalised display)
|
||||
bill_id: Optional[str] = None # canonical "202520260AB28"
|
||||
session: Optional[str] = None # "2025-2026"
|
||||
chamber_origin: Optional[str] = None # ASSEMBLY / SENATE (origin of the bill)
|
||||
author: Optional[str] = None
|
||||
short_title: Optional[str] = None # the bill's subject / short title
|
||||
description: Optional[str] = None # the action description for the day's item
|
||||
committee: Optional[str] = None # None for floor items
|
||||
hearing_time: Optional[str] = None # for hearing rows: "9:30am" / "Upon Call of the Chair"
|
||||
location: Optional[str] = None
|
||||
item_number: Optional[int] = None # int, un-padded (floor items)
|
||||
supplement: Optional[int] = None # edition, from "Supplemental N"
|
||||
ics_url: Optional[str] = None
|
||||
agenda_id: Optional[str] = None
|
||||
# --- provenance -------------------------------------------------------
|
||||
id: Optional[str] = None # auto content hash when None
|
||||
raw_selector: Optional[str] = None # which table the row came from (debug/provenance)
|
||||
|
||||
def with_id(self, content: Optional[str] = None) -> "BillAction":
|
||||
"""Return the record with ``id`` filled in from a content hash.
|
||||
|
||||
The spec's key is ``sha1(house|bill_number|date|committee)``. Missing
|
||||
components are rendered as empty strings so the hash stays deterministic.
|
||||
"""
|
||||
if content is None:
|
||||
parts = [
|
||||
self.house or "",
|
||||
self.bill_number or "",
|
||||
self.date or "",
|
||||
self.committee or "",
|
||||
]
|
||||
content = "|".join(parts)
|
||||
self.id = hashlib.sha1(content.encode("utf-8")).hexdigest()
|
||||
return self
|
||||
|
||||
def to_json(self) -> str:
|
||||
"""Serialise to a single-line JSON string (the JSON Lines record)."""
|
||||
import json
|
||||
|
||||
with_id = self if self.id is not None else self.with_id()
|
||||
return json.dumps(
|
||||
{k: v for k, v in asdict(with_id).items() if v is not None},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
)
|
||||
|
||||
def to_dict(self, *, fill_id: bool = True, include_nulls: bool = False) -> dict:
|
||||
rec = self.with_id() if fill_id and self.id is None else self
|
||||
data = asdict(rec)
|
||||
if not include_nulls:
|
||||
data = {k: v for k, v in data.items() if v is not None}
|
||||
return data
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict) -> "BillAction":
|
||||
known = {f.name for f in fields(cls)}
|
||||
return cls(**{k: v for k, v in data.items() if k in known})
|
||||
|
||||
@staticmethod
|
||||
def field_names() -> List[str]:
|
||||
return [f.name for f in fields(BillAction)]
|
||||
@@ -0,0 +1,93 @@
|
||||
# Resolve a chamber source for a given date -> the URL(s) to fetch + cache file
|
||||
# extension. Neither daily-file page is date-addressable by URL, so the date only
|
||||
# drives the Senate PDF filename and the cache key; the Assembly URL is constant.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
|
||||
# Senate daily-file PDF filename pattern: SEN_{type}_{year}_R_{YYYYMMDD}.PDF
|
||||
SENATE_PDF_RE = re.compile(
|
||||
r"SEN_(?P<type>[A-Z]+)_(?P<year>\d{4})_R_(?P<date>\d{8})\.PDF$"
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SourcePlan:
|
||||
source: str # "assembly_daily_file" | "senate_daily_file"
|
||||
date: str # requested YYYY-MM-DD
|
||||
kind: str # "html" | "pdf"
|
||||
primary_url: str # URL that is actually fetched
|
||||
ext: str # cache extension ("html" | "pdf")
|
||||
pdf_url: Optional[str] = None
|
||||
as_of: Optional[str] = None # date the live page itself represents
|
||||
extra_urls: list = field(default_factory=list)
|
||||
|
||||
|
||||
def build_source_plan(name: str, date_iso: str, src_cfg: dict) -> SourcePlan:
|
||||
kind = src_cfg.get("kind")
|
||||
if kind == "html":
|
||||
# Assembly: constant URL, always the current live file.
|
||||
return SourcePlan(
|
||||
source=name,
|
||||
date=date_iso,
|
||||
kind="html",
|
||||
primary_url=src_cfg["url"],
|
||||
ext="html",
|
||||
as_of=None, # filled by the Ingestor after reading the body
|
||||
)
|
||||
|
||||
if kind == "pdf":
|
||||
# Senate: index is live HTML; the actual file is a date-encoded PDF.
|
||||
pdf_base = src_cfg.get("pdf_base_url", "").rstrip("/") + "/"
|
||||
filename = _senate_pdf_filename(src_cfg, date_iso)
|
||||
pdf_url = pdf_base + filename
|
||||
return SourcePlan(
|
||||
source=name,
|
||||
date=date_iso,
|
||||
kind="pdf",
|
||||
primary_url=src_cfg["index_url"],
|
||||
ext="pdf",
|
||||
pdf_url=pdf_url,
|
||||
as_of=date_iso, # the PDF filename encodes the date
|
||||
)
|
||||
|
||||
raise ValueError("Unknown source kind '%s' for '%s'" % (kind, name))
|
||||
|
||||
|
||||
def _senate_pdf_filename(cfg: dict, date_iso: str) -> str:
|
||||
pattern = cfg.get("pdf_filename_pattern", "SEN_{type}_{year}_R_{date}.PDF")
|
||||
ydate = date_iso.replace("-", "")
|
||||
year = _session_year(cfg, date_iso)
|
||||
ptype = cfg.get("default_pdf_type", "R")
|
||||
return pattern.format(type=ptype, year=year, date=ydate)
|
||||
|
||||
|
||||
def _session_year(cfg: dict, date_iso: str) -> str:
|
||||
# For the 2025-2026 session, any date in 2026 uses session-start year 2025.
|
||||
if not cfg.get("session_year_from_date"):
|
||||
return date_iso[:4]
|
||||
try:
|
||||
y = int(date_iso[:4])
|
||||
except (ValueError, TypeError):
|
||||
return date_iso[:4]
|
||||
if y % 2 == 0:
|
||||
return str(y - 1)
|
||||
return str(y)
|
||||
|
||||
|
||||
def parse_senate_pdf_filename(name: str):
|
||||
# Reverse the encode: filename -> {type, year, date_iso, raw}; None if nmatch.
|
||||
m = SENATE_PDF_RE.search(name.split("/")[-1])
|
||||
if not m:
|
||||
return None
|
||||
ydate = m.group("date")
|
||||
date_iso = ydate[0:4] + "-" + ydate[4:6] + "-" + ydate[6:8]
|
||||
return {
|
||||
"type": m.group("type"),
|
||||
"year": m.group("year"),
|
||||
"date_iso": date_iso,
|
||||
"raw": name,
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Deliberately-malformed daily-file fragment used to prove the parser skips
|
||||
bad rows (logged warning, no batch abort) and survives encoding issues.
|
||||
|
||||
Three problem rows are embedded:
|
||||
R1 a floor-bill row with NO bill link / no number -> must skip
|
||||
R2 a floor-bill row whose item cell is non-numeric garbage-> must skip
|
||||
R3 a hearing row with no committee AND no date -> must skip
|
||||
The surrounding rows are valid, so the batch must still yield all valid ones.
|
||||
"""
|
||||
|
||||
VALID = (
|
||||
'<table class="floor-bills-table">'
|
||||
'<tr class="floor-bill-details">'
|
||||
'<td class="floor_bill-measure" data-sort="AB 00100">'
|
||||
'<a href="https://leginfo.legislature.ca.gov/faces/billNavClient.xhtml?bill_id=202520260AB100">AB 100</a></td>'
|
||||
'<td class="floor_bill-author" data-sort="Smith">Smith</td>'
|
||||
'<td class="floor_bill-subject">An act to do X.</td>'
|
||||
'<td class="floor_bill-location" data-sort="State Capitol">State Capitol</td>'
|
||||
'<td class="floor_bill-item" data-sort="0000001">1</td>'
|
||||
'<td class="floor_bill-view-details">Read third time. </td>'
|
||||
'</tr>'
|
||||
'</table>'
|
||||
)
|
||||
|
||||
BAD_NO_LINK = (
|
||||
'<table class="floor-bills-table">'
|
||||
'<tr class="floor-bill-details">'
|
||||
'<td class="floor_bill-measure" data-sort="AB 00200">'
|
||||
'AB 200 (no hyperlink here)</td>'
|
||||
'<td class="floor_bill-author" data-sort="Jones">Jones</td>'
|
||||
'<td class="floor_bill-subject">Missing bill link row.</td>'
|
||||
'<td class="floor_bill-item" data-sort="0000002">2</td>'
|
||||
'</tr>'
|
||||
'</table>'
|
||||
)
|
||||
|
||||
BAD_ITEM = (
|
||||
'<table class="floor-bills-table">'
|
||||
'<tr class="floor-bill-details">'
|
||||
'<td class="floor_bill-measure" data-sort="AB 00300">'
|
||||
'<a href="https://leginfo.legislature.ca.gov/faces/billNavClient.xhtml?bill_id=202520260AB300">AB 300</a></td>'
|
||||
'<td class="floor_bill-author" data-sort="Lee">Lee</td>'
|
||||
'<td class="floor_bill-subject">Non-numeric item row.</td>'
|
||||
'<td class="floor_bill-item">not-a-number</td>'
|
||||
'</tr>'
|
||||
'</table>'
|
||||
)
|
||||
|
||||
BAD_HEARING = (
|
||||
'<table class="committee-hearing-table">'
|
||||
'<tr class="committee-hearing-details">'
|
||||
'<td class="committee_hearing-time">9:00am</td>'
|
||||
'<td class="committee_hearing-menu">no agenda link</td>'
|
||||
'</tr>'
|
||||
'</table>'
|
||||
)
|
||||
|
||||
def malformed_document() -> str:
|
||||
"""A concatenated fragment: valid, bad, bad, bad rows."""
|
||||
return VALID + BAD_NO_LINK + BAD_ITEM + BAD_HEARING
|
||||
|
||||
def write_fixtures(out_dir: str) -> None:
|
||||
import os
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
# A clean single-file fixture for the "good" acceptance test.
|
||||
with open(os.path.join(out_dir, "malformed_fragment.html"), "w", encoding="utf-8") as fh:
|
||||
fh.write(malformed_document())
|
||||
# A tiny encoding-edge-case file: UTF-16 with a BOM so the charset path runs.
|
||||
with open(os.path.join(out_dir, "utf16_sample.html"), "w", encoding="utf-16") as fh:
|
||||
fh.write(VALID)
|
||||
|
||||
if __name__ == "__main__":
|
||||
write_fixtures(".parser_fixtures")
|
||||
import sys
|
||||
print("wrote fixtures to .parser_fixtures/", file=sys.stderr)
|
||||
Reference in new issue
Block a user