WesternTechnologies/cali_scraper
2
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
03eeb6bb6276572567764134ea191cd99e2425e1
cali_scraper/tests/test_dashboard_api.py
T
kfj001 03eeb6bb62 feat(dashboard): add forward "what's next" signal + fix upcoming floor 
The daily-file bill queue is a backward snapshot, so "upcoming" could only
resurface the latest past day (the 08-31 snapshot read as today's queue) and
there was no "what is scheduled next." leginfo bill status/history are
backward-only, so the genuine future-dated signal is the chambers' "upcoming
events" calendars.

- api.py: floor the "upcoming" view at *today* (not the DB max date) when no
  as_of is pinned; surface next_legislative_day + scheduled_events (next 7
  days) on /bills. Add _today_iso / _next_legislative_day / _scheduled_events,
  all degrading to a recess marker / [] (never 5xx).
- relational.py: schema v2 adds the scheduled_events table +
  store_scheduled_events (idempotent per-day refresh), load_scheduled_events,
  next_legislative_day.
- parsers.py: parse_events_page + classify_event_kind for the chamber events
  calendar; skip empty-title rows (decompose the <time> before the fallback).
- ingest.py + cli.py: --store-events lane (cache-first, recess-safe) that
  parses the calendar HTML into scheduled_events.
- sources.py + config.yaml: register assembly_events / senate_events
  (kind: events, empty_is_ok: true).
- run_ingest.sh: run the events lane after the daily-file lane as non-fatal
  (skippable via CALI_SCRAPER_SKIP_EVENTS=1); daily-file exit still gates cron.
- frontend: "Next legislative day" line + "What's next · next 7 days" band
  with honest recess empty states.
- DASHBOARD.md: document the forward signal + known limitations (leginfo
  backward-only gap, recess empty state, 08-31 snapshot is analysis-only).
- tests: 5 new API tests + 18 parse-events tests; 170 pass, 92% coverage.
2026-09-06 19:50:02 -07:00

454 lines
20 KiB
Python
Raw Blame History

# Tests for the dashboard API layer against the frozen API_CONTRACT.md shapes.
# Drives handlers via api.run(Request(...)) — no sockets — plus one in-process
# ThreadingHTTPServer test. Builds a small temp DB (flat + relational +
# bill_tags) so shapes + error codes + empty states are asserted.
from __future__ import annotations
import json
import threading
import urllib.request
import pytest
from scraper import relational
from scraper.schema import BillAction
from scraper.io import write_sqlite
from scraper.dashboard import api
from scraper.dashboard.api import Request, Services
from scraper.dashboard.prefs import PrefsStore, LocalAuth
# ---- fixture: a small, controlled DB --------------------------------------
def build_small_db(path):
recs = [
BillAction(date="2026-09-10", house="assembly",
record_type="bill", bill_number="AB 100",
bill_id="202520260AB100", session="2025-2026",
chamber_origin="assembly", author="Schiavo et al.",
short_title="Waste: recycling standards",
description="d", source="ASSEMBLY_DAILY_FILE"),
BillAction(date="2026-09-10", house="assembly",
record_type="bill", bill_number="AB 200",
bill_id="202520260AB200", session="2025-2026",
chamber_origin="assembly", author="Macedo",
short_title="Education: K-12 funding",
description="d", source="ASSEMBLY_DAILY_FILE"),
BillAction(date="2026-09-05", house="assembly",
record_type="bill", bill_number="SB 300",
bill_id="202520260SB300", session="2025-2026",
chamber_origin="assembly", author="Unrelated",
short_title="Old bill, past date",
description="d", source="ASSEMBLY_DAILY_FILE"),
]
n = write_sqlite(recs, path, table=relational.FLAT_TABLE)
assert n == 3
relational.correlate(path, quiet=True)
conn = relational.connect(path, foreign_keys=False)
try:
conn.execute(
"CREATE TABLE IF NOT EXISTS bill_tags ("
"id INTEGER PRIMARY KEY, bill_id TEXT NOT NULL, "
"tag TEXT NOT NULL, source TEXT, generated_by TEXT NOT NULL, "
"generated_at TEXT, UNIQUE (bill_id, tag, generated_by))")
for bid, tag in ("202520260AB100", "waste"), \
("202520260AB200", "education"), \
("202520260AB200", "health"):
conn.execute(
"INSERT OR IGNORE INTO bill_tags "
"(bill_id,tag,generated_by) VALUES (?,?,?)",
(bid, tag, "fallback"))
conn.commit()
finally:
conn.close()
return path
@pytest.fixture
def small_db(tmp_path):
return build_small_db(str(tmp_path / "dash.sqlite"))
@pytest.fixture
def services(tmp_path, small_db):
prefs = PrefsStore(str(tmp_path / "prefs.json"))
auth = LocalAuth(str(tmp_path / "auth.json"))
return Services(small_db, prefs, auth)
def _req(method, path, services, token=None, query=None, body=None,
user_id=None):
return Request(method, path, query=query or {}, body=body or {},
token=token, services=services, user_id=user_id)
# ---- /bills envelope (§1/§2) ----------------------------------------------
def test_bills_envelope_shape(services):
status, body = api.run(_req("GET", "/api/v1/bills", services,
query={"upcoming": "1",
"as_of": "2026-09-06"}))
assert status == 200, body
for key in ("session", "as_of", "chamber_availability",
"items", "groups"):
assert key in body
assert isinstance(body["items"], list)
assert isinstance(body["groups"], dict)
ids = {i["bill_id"] for i in body["items"]}
assert "202520260AB100" in ids
assert "202520260AB200" in ids
assert "202520260SB300" not in ids
item = body["items"][0]
for key in ("bill_id", "bill_number", "chamber_origin", "author",
"short_title", "hearing_sort", "tags"):
assert key in item
assert isinstance(item["hearing_sort"], int)
def test_bills_personal_returns_relevance_and_floats(services):
reg = services.auth.register("u@example.com")
# user watches "schiavo" and opts into "education"
api.run(_req("POST", "/api/v1/prefs/watch", services,
token=reg["token"],
body={"legislator_id": "schiavo"}))
api.run(_req("POST", "/api/v1/prefs/topics", services,
token=reg["token"],
body={"tag": "education"}))
# personal + full + upcoming (as_of=2026-09-04, before all 3 bills' date)
status, body = api.run(_req("GET", "/api/v1/bills", services,
token=reg["token"],
query={"upcoming": "1", "as_of": "2026-09-04",
"personal": "1", "full": "1"}))
assert status == 200, body
by = {i["bill_id"]: i for i in body["items"]}
assert "relevance" in by["202520260AB100"]
assert by["202520260AB100"]["relevance"]["score"] > 0
assert by["202520260AB200"]["relevance"]["score"] > 0
# SB300 not relevant to the user (no match) — its score is 0
assert "202520260SB300" in by
assert by["202520260SB300"]["relevance"]["score"] == 0
# float-to-top: the first item has a positive score (a floated bill)
first = by[body["items"][0]["bill_id"]]
assert first["relevance"]["score"] > 0
assert by["202520260AB100"]["bill_id"] in \
[i["bill_id"] for i in body["items"][:1]]
# SB300 comes after the floated bills in the ordering
order = [i["bill_id"] for i in body["items"]]
assert order.index("202520260SB300") > \
max(order.index("202520260AB100"),
order.index("202520260AB200")), order
def test_bills_personal_requires_auth(services):
status, body = api.run(_req("GET", "/api/v1/bills", services,
query={"personal": "1"}))
assert status == 401, body
assert body["error"]["code"] == "not_authenticated"
# ---- §5 "upcoming" item 2: each item carries status + reason -------------
def test_bills_items_carry_status_reason_and_possibly_closed(services):
# Every item must expose the "why is this here?" triad so a closed/past
# bill doesn't read as a vague "pending" (API_CONTRACT §5 item 2).
status, body = api.run(_req("GET", "/api/v1/bills", services,
query={"upcoming": "1"}))
assert status == 200, body
items = body["items"]
assert items, "expected at least one item"
for it in items:
for key in ("status", "reason", "possibly_closed"):
assert key in it, key
assert it["status"] in ("open", "closed", "unknown")
assert isinstance(it["reason"], str) and it["reason"]
assert it["possibly_closed"] in (True, False, None)
# ---- six empty states -> 200 items:[] (never 4xx/5xx) ---------------------
def test_empty_no_next_hearing(services):
status, body = api.run(_req("GET", "/api/v1/bills", services,
query={"from": "2030-01-01",
"to": "2030-12-31"}))
assert status == 200 and body["items"] == [] and body["groups"] == {}
def test_empty_filter_returns_nothing(services):
status, body = api.run(_req("GET", "/api/v1/bills", services,
query={"upcoming": "1",
"as_of": "2030-01-01",
"tags": "education"}))
assert status == 200 and body["items"] == []
def test_empty_prefs_not_set(services):
reg = services.auth.register("nobody@example.com")
status, body = api.run(_req("GET", "/api/v1/prefs", services,
token=reg["token"]))
# register seeds an empty-blob, so 200 with empty prefs is valid;
# a truly-empty store would 404 with prefs_not_set — accept both,
# never 5xx.
assert status in (200, 404)
if status == 404:
assert body["error"]["code"] == "prefs_not_set"
def test_empty_no_tagged_bills_and_chamber_pending(tmp_path, services):
# a flat DB with no tag rows -> /tags returns 200 tags [] (§6.6)
empty = str(tmp_path / "notags.sqlite")
write_sqlite([BillAction(bill_number="AB 1",
bill_id="202520260AB1", session="2025-2026",
chamber_origin="assembly", author="Solo",
short_title="x",
source="ASSEMBLY_DAILY_FILE")],
empty, table=relational.FLAT_TABLE)
relational.migrate(empty)
svc = Services(empty, PrefsStore(str(tmp_path / "p.json")),
LocalAuth(str(tmp_path / "a.json")))
status, body = api.run(_req("GET", "/api/v1/tags", svc))
assert status == 200 and body["tags"] == []
# and a Senate-only-empty window stays 200 (chamber pending, not error)
status, body = api.run(_req("GET", "/api/v1/bills", svc,
query={"from": "2030-01-01",
"chamber": "SENATE"}))
assert status == 200 and body["items"] == []
# ---- forward "upcoming" source: scheduled_events + next_legislative_day ----
def _events_db(tmp_path):
"""A small DB whose flat snapshot is in the PAST but whose chamber
scheduled_events table holds future-dated events (the forward source)."""
p = str(tmp_path / "fwd.sqlite")
write_sqlite([
BillAction(date="2026-08-31", house="assembly", record_type="bill",
bill_number="AB 1099", bill_id="202520260AB1099",
session="2025-2026", chamber_origin="assembly",
author="Bryan", short_title="Developmental services",
description="Read third time. Passed. Ordered to the Assembly.",
source="ASSEMBLY_DAILY_FILE")], p, table=relational.FLAT_TABLE)
relational.migrate(p)
relational.store_scheduled_events([
{"date": "2026-09-08", "chamber": "ASSEMBLY", "kind": "FLOOR",
"title": "Assembly Floor Session",
"source_url": "https://www.assembly.ca.gov/events/upcoming",
"source": "assembly_events"},
{"date": "2026-09-09", "chamber": "ASSEMBLY", "kind": "COMMITTEE",
"title": "Judiciary Committee", "source": "assembly_events"},
], path=p, fetched_at="2026-09-06T00:00:00Z")
return p
def test_upcoming_defaults_to_not_past(tmp_path):
"""The bug that motivated this work: with upcoming=1 and no as_of, the
bill queue must floor at *today* (the past snapshot drops out), not at the
most-recent data date (which would re-surface yesterday as "upcoming")."""
fwd = _events_db(tmp_path)
svc = Services(fwd, PrefsStore(str(tmp_path / "p.json")),
LocalAuth(str(tmp_path / "a.json")))
status, body = api.run(_req("GET", "/api/v1/bills", svc,
query={"upcoming": "1"}))
assert status == 200, body
# the 08-31 snapshot is in the past relative to today -> not "upcoming"
assert body["items"] == []
# ...but the chamber calendar still answers "next":
assert body["next_legislative_day"]["date"] is not None
assert body["next_legislative_day"]["recess"] is False
assert len(body["scheduled_events"]) >= 1
def test_envelope_carries_forward_fields(tmp_path):
"""Envelope must carry the forward fields with a 7-day window."""
fwd = _events_db(tmp_path)
svc = Services(fwd, PrefsStore(str(tmp_path / "p.json")),
LocalAuth(str(tmp_path / "a.json")))
status, body = api.run(_req("GET", "/api/v1/bills", svc,
query={"upcoming": "1", "as_of": "2026-09-06"}))
assert status == 200, body
for key in ("next_legislative_day", "scheduled_events"):
assert key in body
nd = body["next_legislative_day"]
assert nd["date"] == "2026-09-08"
assert nd["kind"] == "FLOOR"
# 7-day window from 09-06 reaches 09-08 and 09-09, not further:
dates = {e["date"] for e in body["scheduled_events"]}
assert "2026-09-08" in dates and "2026-09-09" in dates
def test_next_day_recess_when_no_forward_events(tmp_path):
"""A pre-v2-style DB with no scheduled_events degrades to a recess marker
(never a 5xx): next_legislative_day is a {date: None, recess: True}."""
p = str(tmp_path / "empty.sqlite")
relational.migrate(p)
svc = Services(p, PrefsStore(str(tmp_path / "p.json")),
LocalAuth(str(tmp_path / "a.json")))
status, body = api.run(_req("GET", "/api/v1/bills", svc,
query={"upcoming": "1"}))
assert status == 200, body
assert body["next_legislative_day"] == {"date": None, "recess": True}
assert body["scheduled_events"] == []
def test_all_view_still_returns_history(tmp_path):
"""Omitting upcoming (the "All"/history view) must still return the past
snapshot rows -- the forward fix only changes the *upcoming* view."""
fwd = _events_db(tmp_path)
svc = Services(fwd, PrefsStore(str(tmp_path / "p.json")),
LocalAuth(str(tmp_path / "a.json")))
status, body = api.run(_req("GET", "/api/v1/bills", svc, query={}))
assert status == 200
assert len(body["items"]) >= 1
assert any(i["bill_number"] == "AB 1099" for i in body["items"])
# ---- /tags, /legislators, /districts (§4.1 / §3.2 / §4.2) -----------------
def test_tags_cloud(services):
status, body = api.run(_req("GET", "/api/v1/tags", services))
assert status == 200, body
assert "tags" in body
labels = {t["tag_id"] for t in body["tags"]}
assert "tag:waste" in labels
assert "tag:education" in labels
assert all("count" in t and "label" in t for t in body["tags"])
def test_legislators_browse(services):
status, body = api.run(_req("GET", "/api/v1/legislators", services))
assert status == 200
assert "items" in body
ids = {x["legislator_id"] for x in body["items"]}
assert "schiavo" in ids
def test_districts_list(services):
status, body = api.run(_req("GET", "/api/v1/districts", services))
assert status == 200
assert "items" in body and body["items"]
# ---- /districts/resolve (§3.1) — confidence, proposes (no persist) ---------
def test_resolve_single(services):
status, body = api.run(_req("POST", "/api/v1/districts/resolve",
services, body={"zip": "90210"}))
assert status == 200, body
assert body["confidence"] == "single"
assert body["resolved_from"] == "zip"
assert body["suggested_watch"] and len(body["districts"]) == 1
def test_resolve_ambiguous_lists_candidates(services):
status, body = api.run(_req("POST", "/api/v1/districts/resolve",
services, body={"zip": "90012"}))
assert status == 200, body
assert body["confidence"] == "ambiguous"
assert len(body["districts"]) >= 2
def test_resolve_zero_and_out_of_state(services):
s0, b0 = api.run(_req("POST", "/api/v1/districts/resolve",
services, body={"zip": "00001"}))
assert s0 == 404 and b0["error"]["code"] == "district_not_found"
# out-of-state prefix (99xxx / 990xx) -> 400 out_of_state
s2, b2 = api.run(_req("POST", "/api/v1/districts/resolve",
services, body={"zip": "99001"}))
assert s2 == 400 and b2["error"]["code"] == "out_of_state"
# manual pick is the fallback and always resolves (single)
s3, b3 = api.run(_req("POST", "/api/v1/districts/resolve",
services, body={"district": "A-14"}))
assert s3 == 200 and b3["confidence"] == "single"
def test_resolve_missing_input_is_400(services):
status, body = api.run(_req("POST", "/api/v1/districts/resolve",
services, body={}))
assert status == 400 and "field" in body["error"]
# ---- prefs + watch/topic (§3.3/§3.4/§3.5) ---------------------------------
def test_prefs_put_and_get_roundtrip(services):
reg = services.auth.register("prefs@example.com")
token = reg["token"]
status, body = api.run(_req("PUT", "/api/v1/prefs", services,
token=token,
body={"zip": "90210",
"watched_legislators": ["schiavo"],
"opted_in_tags": ["waste"]}))
assert status == 200, body
_, got = api.run(_req("GET", "/api/v1/prefs", services, token=token))
assert got["zip"] == "90210"
assert "schiavo" in got["watched_legislators"]
assert "waste" in got["opted_in_tags"]
def test_prefs_put_rejects_unknown_tag(services):
reg = services.auth.register("bad@example.com")
status, body = api.run(_req("PUT", "/api/v1/prefs", services,
token=reg["token"],
body={"opted_in_tags": ["nope-not-tag"]}))
assert status == 400 and body["error"]["code"] == "tag_unknown"
def test_add_remove_watch_is_idempotent(services):
reg = services.auth.register("w@example.com")
token = reg["token"]
api.run(_req("POST", "/api/v1/prefs/watch", services, token=token,
body={"legislator_id": "schiavo"}))
# add again -> idempotent, still exactly one
api.run(_req("POST", "/api/v1/prefs/watch", services, token=token,
body={"legislator_id": "schiavo"}))
_, got = api.run(_req("GET", "/api/v1/prefs", services, token=token))
assert got["watched_legislators"] == ["schiavo"]
# remove -> gone
api.run(_req("DELETE", "/api/v1/prefs/watch/schiavo", services,
token=token))
_, got2 = api.run(_req("GET", "/api/v1/prefs", services, token=token))
assert "schiavo" not in got2["watched_legislators"]
def test_add_unknown_legislator_is_400(services):
reg = services.auth.register("l@example.com")
status, body = api.run(_req("POST", "/api/v1/prefs/watch", services,
token=reg["token"],
body={"legislator_id": "phantom-leg"}))
assert status == 400 and body["error"]["code"] == "legislator_unknown"
# ---- router: unknown endpoint -> 404, not a crash -------------------------
def test_unknown_endpoint_404(services):
status, body = api.run(_req("GET", "/api/v1/nope", services))
assert status == 404 and body["error"]["code"] == "not_found"
# ---- live server (real socket, in-process, no external port) --------------
def test_live_server_serves_bills_and_static(tmp_path, small_db):
from scraper.dashboard import server as srv
prefs = PrefsStore(str(tmp_path / "prefs.json"))
auth = LocalAuth(str(tmp_path / "auth.json"))
services = Services(small_db, prefs, auth)
handler = srv.make_handler(services)
httpd = srv.ThreadingHTTPServer(("127.0.0.1", 0), handler)
port = httpd.server_address[1]
t = threading.Thread(target=httpd.serve_forever, daemon=True)
t.start()
try:
url = ("http://127.0.0.1:%d/api/v1/bills?upcoming=1"
"&as_of=2026-09-06" % port)
with urllib.request.urlopen(url, timeout=5) as resp:
assert resp.status == 200
payload = json.loads(resp.read().decode("utf-8"))
assert "items" in payload
with urllib.request.urlopen(
"http://127.0.0.1:%d/index.html" % port, timeout=5) as resp:
assert resp.status == 200
html = resp.read().decode("utf-8")
assert "<html" in html.lower()
finally:
httpd.shutdown()
httpd.server_close()
Reference in new issue View Git Blame Copy Permalink