Initial commit
This commit is contained in:
kfj001 committed
commit
5f2db08b43
100 files changed
+16985
No files matched your search
@@ -0,0 +1,5 @@
|
||||
"""QA test-suite package for the California Legislature Daily File scraper.
|
||||
|
||||
Tests live in this package; shared fixtures/builders are in ``fixtures`` and the
|
||||
shared fakes/handlers in the root ``conftest``.
|
||||
"""
|
||||
@@ -0,0 +1,255 @@
|
||||
"""Fixture builders for the QA test-suite (task t_8e5a3821).
|
||||
|
||||
The task asks for parser unit tests over "archived sample HTML fixtures (at
|
||||
least 3 dates, one per chamber, plus one edge-case file)". California publishes
|
||||
its daily file with NO historical date parameter -- the Assembly page always
|
||||
renders the current live file and the Senate page is a PDF only (see
|
||||
DAILY_FILE_PARSE_SPEC.md section 1 / section 8 #2). So a "different date" in these
|
||||
tests means a *different legislative day's content*; the dates are carried as data
|
||||
values in the hearing table (the parser derives the file date from it, spec 3.2).
|
||||
|
||||
The builders below reproduce the canonical table markup verbatim from the archived
|
||||
sample (class + data-sort names copied from samples/assembly_dailyfile_latest.html
|
||||
and spec section 3) so the parser's date/hearing/selector machinery runs on genuine
|
||||
markup, combined with freshly-constructed floor / committee rows. Each builder emits
|
||||
one distinct legislative date and covers BOTH chambers' origin, giving the three
|
||||
"different dates, one per chamber" fixtures plus the edge-case file the task wants.
|
||||
|
||||
The builders are pure except that they decode a bill_id through the real
|
||||
``scraper.bill_id.parse_bill_id`` codec -- itself a unit-tested subject -- so the
|
||||
generated HTML is always well-formed and the id is guaranteed decodable. No network,
|
||||
no file I/O: the fixtures are fully reproducible.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
|
||||
from scraper.bill_id import parse_bill_id
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- markup bits
|
||||
|
||||
def _url(raw: str) -> str:
|
||||
return f"https://leginfo.legislature.ca.gov/faces/billNavClient.xhtml?bill_id={raw}"
|
||||
|
||||
|
||||
# A hearing row: class + data-sort names copied verbatim from the archived sample
|
||||
# so _parse_hearings / normalise_date run on genuine markup.
|
||||
_HEARING_ROW = (
|
||||
'<table class="committee-hearing-table">'
|
||||
'<tr class="committee-hearing-details">'
|
||||
'<td class="committee_hearing-date" data-sort="{date}">{date_short}</td>'
|
||||
'<td class="committee_hearing-time">{time}</td>'
|
||||
'<td class="committee_hearing-name" data-sort="{committee}">{committee}</td>'
|
||||
'<td class="committee_hearing-location" data-sort="{location}">{location}</td>'
|
||||
'<td class="committee_hearing-menu">'
|
||||
'<a href="/api/ics/generate?name={ics}&start=1&end=2">ics</a>'
|
||||
'<a href="/api/dailyfile/agenda/{agenda}">agenda</a>'
|
||||
'</td>'
|
||||
'</tr></table>'
|
||||
)
|
||||
|
||||
# A floor bill row: canonical floor-bills-table shape (spec section 3.2). The
|
||||
# measure cell carries bill_id in the anchor href (decoded by scraper.bill_id)
|
||||
# plus a "Supplemental N" note after a <br> when supplied.
|
||||
_FLOOR_ROW = (
|
||||
'<table class="floor-bills-table">'
|
||||
'<tr class="floor-bill-details">'
|
||||
'<td class="floor_bill-measure" data-sort="{bill}">'
|
||||
'<a href="{url}" target="_blank">{disp}</a>{supp}</td>'
|
||||
'<td class="floor_bill-author" data-sort="{author}">{author}</td>'
|
||||
'<td class="floor_bill-subject">{title}</td>'
|
||||
'<td class="floor_bill-location" data-sort="{location}">{location}</td>'
|
||||
'<td class="floor_bill-item" data-sort="{item_padded}">{item}</td>'
|
||||
'<td class="floor_bill-view-details"><dialog class="was-dialog">ignore-me</dialog></td>'
|
||||
'</tr></table>'
|
||||
)
|
||||
|
||||
# A committee bill row: canonical committee-bills-table shape (spec section 3.3).
|
||||
_COMMITTEE_ROW = (
|
||||
'<table class="committee-bills-table">'
|
||||
'<tr class="committee-bills-details">'
|
||||
'<td class="committee_bill-number" data-sort="{bill}">'
|
||||
'<a href="{url}" target="_blank">{disp}</a></td>'
|
||||
'<td class="committee_bill-author" data-sort="{author}">{author}</td>'
|
||||
'<td class="committee_bill-subject">{title}</td>'
|
||||
'<td class="committee_bill-committee" data-sort="{committee}">{committee}</td>'
|
||||
'</tr></table>'
|
||||
)
|
||||
|
||||
# The page wrapper is a minimal, well-formed HTML5 doc.
|
||||
_WRAPPER = (
|
||||
'<!doctype html><html lang="en"><head>'
|
||||
'<meta charset="utf-8"><title>Daily File {date}</title>'
|
||||
'</head><body>{body}</body></html>'
|
||||
)
|
||||
|
||||
|
||||
# --------------------- edge-case fragments (mirror test_fixtures.py) ---------------------
|
||||
|
||||
EDGE_BAD_NO_LINK = (
|
||||
'<table class="floor-bills-table"><tr class="floor-bill-details">'
|
||||
'<td class="floor_bill-measure" data-sort="AB 00200">AB 200 (no hyperlink)</td>'
|
||||
'<td class="floor_bill-author" data-sort="Jones">Jones</td>'
|
||||
'<td class="floor_bill-subject">Missing bill link.</td>'
|
||||
'<td class="floor_bill-item" data-sort="0000002">2</td></tr></table>'
|
||||
)
|
||||
EDGE_BAD_ITEM = (
|
||||
'<table class="floor-bills-table"><tr class="floor-bill-details">'
|
||||
'<td class="floor_bill-measure">'
|
||||
'<a href="https://leginfo.legislature.ca.gov/faces/billNavClient.xhtml'
|
||||
'?bill_id=202520260AB300">AB 300</a></td>'
|
||||
'<td class="floor_bill-author" data-sort="Lee">Lee</td>'
|
||||
'<td class="floor_bill-subject">Non-numeric item cell.</td>'
|
||||
'<td class="floor_bill-item">not-a-number</td></tr></table>'
|
||||
)
|
||||
EDGE_HEARING_NO_COMMITTEE = (
|
||||
'<table class="committee-hearing-table"><tr class="committee-hearing-details">'
|
||||
'<td class="committee_hearing-time">9:00am</td>'
|
||||
'<td class="committee_hearing-menu">no agenda link</td></tr></table>'
|
||||
)
|
||||
# A floor row whose link/text is only a clean "AB NNNNN" display, recoverable via
|
||||
# the parser's display-text fallback.
|
||||
EDGE_DISPLAY_ONLY = (
|
||||
'<table class="floor-bills-table"><tr class="floor-bill-details">'
|
||||
'<td class="floor_bill-measure">AB 150</td>'
|
||||
'<td class="floor_bill-author" data-sort="Adams">Adams</td>'
|
||||
'<td class="floor_bill-subject">Display-only recovery.</td>'
|
||||
'<td class="floor_bill-item" data-sort="0000150">150</td></tr></table>'
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- builders
|
||||
|
||||
_SUPP_SUFFIX = '<br>Supplemental {n}'
|
||||
|
||||
|
||||
def _short(date_iso: str) -> str:
|
||||
year, month, day = date_iso.split("-")
|
||||
return f"{month}/{day}/{year[2:]}" # '2026-08-31' -> '08/31/26'
|
||||
|
||||
|
||||
def hearing(date_iso, time, committee, location, agenda, ics):
|
||||
s = _short(date_iso)
|
||||
return _HEARING_ROW.format(
|
||||
date=f"{s}-001", # the real data-sort carries a '-001' display suffix
|
||||
date_short=s,
|
||||
time=time,
|
||||
committee=committee,
|
||||
location=location,
|
||||
ics=ics,
|
||||
agenda=agenda,
|
||||
)
|
||||
|
||||
|
||||
def floor_bill(raw_id, author, title, item, location="State Capitol",
|
||||
supplement: Optional[int] = None):
|
||||
dec = parse_bill_id(raw_id)
|
||||
if dec is None:
|
||||
raise ValueError(f"floor_bill builder got undecodable id {raw_id!r}")
|
||||
supp = _SUPP_SUFFIX.format(n=supplement) if supplement else ""
|
||||
return _FLOOR_ROW.format(
|
||||
bill=f"{dec['chamber']} {dec['number']:05d}",
|
||||
url=_url(raw_id),
|
||||
disp=f"{dec['chamber']} {dec['number']}",
|
||||
supp=supp,
|
||||
author=author,
|
||||
title=title,
|
||||
location=location,
|
||||
item=str(item),
|
||||
item_padded=f"{item:07d}",
|
||||
)
|
||||
|
||||
|
||||
def committee_bill(raw_id, author, title, committee):
|
||||
dec = parse_bill_id(raw_id)
|
||||
if dec is None:
|
||||
raise ValueError(f"committee_bill builder got undecodable id {raw_id!r}")
|
||||
return _COMMITTEE_ROW.format(
|
||||
bill=f"{dec['chamber']} {dec['number']:05d}",
|
||||
url=_url(raw_id),
|
||||
disp=f"{dec['chamber']} {dec['number']}",
|
||||
author=author,
|
||||
title=title,
|
||||
committee=committee,
|
||||
)
|
||||
|
||||
|
||||
# -------- the three "different dates, one per chamber" daily-file fixtures --------
|
||||
|
||||
def assembly_daily_file_html() -> str:
|
||||
date_iso = "2026-08-31" # verbatim from the archived hearing sample
|
||||
body = (
|
||||
hearing(date_iso, "Upon Call of the Chair", "Budget",
|
||||
"1021 O Street, Room 1100", "20361", "name=Budget")
|
||||
+ floor_bill("202520260AB28", "Schiavo et al.",
|
||||
"An act relating to solid waste, and making an appropriation "
|
||||
"therefor.", 32,
|
||||
"Unfinished Business - Concurrence In Senate Amendments",
|
||||
supplement=2)
|
||||
# A Senate-origin bill on the Assembly file (cross-chamber referral,
|
||||
# spec section 8 #7) -> chamber_origin must resolve to SENATE.
|
||||
+ floor_bill("202520260SB16", "Blakespear et al.",
|
||||
"An act relating to mental health.", 62,
|
||||
"Senate Third Reading File")
|
||||
+ committee_bill("202520260SB795", "Archuleta",
|
||||
"Air pollution: gasoline vapor control systems: "
|
||||
"ethanol blends.", "Emergency Management")
|
||||
)
|
||||
return _WRAPPER.format(date=date_iso, body=body)
|
||||
|
||||
|
||||
def senate_daily_file_html() -> str:
|
||||
date_iso = "2026-09-02" # the Senate committee daily-file date (spec section 1.2)
|
||||
body = (
|
||||
hearing(date_iso, "10:00am", "Government Operations",
|
||||
"700 N Street, Room 6035", "30481", "name=GovernmentOperations")
|
||||
+ floor_bill("202520260SB801", "Garcia et al.",
|
||||
"An act relating to water conservation.", 12,
|
||||
"State Capitol, Room 417")
|
||||
)
|
||||
return _WRAPPER.format(date=date_iso, body=body)
|
||||
|
||||
|
||||
def extra_daily_file_html() -> str:
|
||||
date_iso = "2026-09-03" # an additional legislative day -> 3rd distinct date
|
||||
body = (
|
||||
hearing(date_iso, "Upon Call of the Chair", "Rules",
|
||||
"1021 O Street, Room 2000", "30512", "name=Rules")
|
||||
+ floor_bill("202520260AB42", "Nguyen",
|
||||
"An act relating to public schools.", 7,
|
||||
"Unfinished Business")
|
||||
)
|
||||
return _WRAPPER.format(date=date_iso, body=body)
|
||||
|
||||
|
||||
# -------- edge cases --------
|
||||
|
||||
def edge_case_html() -> str:
|
||||
"""Every documented malformed-row class plus one recoverable display-only row
|
||||
and one good row, so a good batch must skip the bad rows (warning each),
|
||||
recover the display-only row, and keep the good one.
|
||||
|
||||
Mirrors scraper.test_fixtures.malformed_document() so the parser's
|
||||
robustness contract has one authoritative edge fixture here too.
|
||||
"""
|
||||
good = floor_bill("202520260AB100", "Smith", "An act to do X.", 1, "State Capitol")
|
||||
body = good + EDGE_BAD_NO_LINK + EDGE_BAD_ITEM + EDGE_HEARING_NO_COMMITTEE \
|
||||
+ EDGE_DISPLAY_ONLY
|
||||
return _WRAPPER.format(date="2026-08-31", body=body)
|
||||
|
||||
|
||||
def edge_case_bytes(html: str = "AB 100 <html>not-utf8</html>") -> bytes:
|
||||
"""A UTF-16 (BOM-prefixed) file. The parser's charset sniff reads the BOM,
|
||||
decodes it, and the text carries replacement chars -> the parser must log a
|
||||
warning and return [] rather than crash the batch."""
|
||||
return html.encode("utf-16")
|
||||
|
||||
|
||||
# Dated + chamber coverage for the test-suite's own bookkeeping.
|
||||
FIXTURE_DATES = {
|
||||
"assembly_daily_file_html": ("Assembly", "2026-08-31"),
|
||||
"senate_daily_file_html": ("Senate", "2026-09-02"),
|
||||
"extra_daily_file_html": ("Assembly", "2026-09-03"),
|
||||
"edge_case_html": ("Assembly", "2026-08-31"),
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
"""Unit tests for scraper.bill_id -- the canonical bill_id codec (spec section 4).
|
||||
|
||||
These tests are the source of truth for the codec the parser depends on, so they
|
||||
run first and are the most brittle-to-change. Any change to the codec that
|
||||
breaks any of these must be a conscious decision, not an accidental refactor.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from scraper.bill_id import (
|
||||
normalise_session,
|
||||
origin_house,
|
||||
parse_bill_id,
|
||||
parse_display,
|
||||
parse_supplement,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- parse_bill_id
|
||||
|
||||
class TestParseBillId:
|
||||
def test_assembly_bill_ab28(self):
|
||||
dec = parse_bill_id("202520260AB28")
|
||||
assert dec is not None
|
||||
assert dec["session"] == "2025-2026"
|
||||
assert dec["session_raw"] == "20252026"
|
||||
assert dec["chamber"] == "AB"
|
||||
assert dec["number"] == 28
|
||||
assert dec["display"] == "AB 28"
|
||||
assert dec["house"] == "assembly"
|
||||
assert dec["raw"] == "202520260AB28"
|
||||
|
||||
def test_senate_bill_sb795(self):
|
||||
dec = parse_bill_id("202520260SB795")
|
||||
assert dec is not None
|
||||
assert dec["chamber"] == "SB"
|
||||
assert dec["session"] == "2025-2026"
|
||||
assert dec["number"] == 795
|
||||
assert dec["display"] == "SB 795"
|
||||
assert dec["house"] == "senate"
|
||||
|
||||
def test_assembly_prefixes(self):
|
||||
for raw, chamber, house in [
|
||||
("202520260ACR5", "ACR", "assembly"),
|
||||
("202520260HR7", "HR", "assembly"),
|
||||
("202520260ABR1", "ABR", "assembly"),
|
||||
]:
|
||||
dec = parse_bill_id(raw)
|
||||
assert dec is not None
|
||||
assert dec["chamber"] == chamber, raw
|
||||
assert dec["house"] == house, raw
|
||||
|
||||
def test_senate_prefixes(self):
|
||||
for raw, chamber in [
|
||||
("202520260SCR12", "SCR"),
|
||||
("202520260SJR3", "SJR"),
|
||||
]:
|
||||
dec = parse_bill_id(raw)
|
||||
assert dec is not None
|
||||
assert dec["chamber"] == chamber, raw
|
||||
assert dec["house"] == "senate", raw
|
||||
|
||||
def test_invalid_inputs_return_none(self):
|
||||
for bad in [
|
||||
"",
|
||||
"garbage",
|
||||
"202520260", # no chamber/number tail
|
||||
"202520260AB", # chamber with no number
|
||||
"202520260AB28JUNK",# trailing junk
|
||||
"20250AB28", # wrong session length
|
||||
"202520260XY12", # unknown chamber prefix
|
||||
]:
|
||||
assert parse_bill_id(bad) is None, f"expected None for {bad!r}"
|
||||
|
||||
def test_whitespace_is_stripped(self):
|
||||
dec = parse_bill_id(" 202520260AB28 ")
|
||||
assert dec is not None
|
||||
assert dec["display"] == "AB 28"
|
||||
assert dec["raw"] == " 202520260AB28 "
|
||||
|
||||
def test_session_decimal_boundary(self):
|
||||
dec = parse_bill_id("202520260AB40")
|
||||
assert dec is not None
|
||||
assert dec["number"] == 40
|
||||
assert dec["display"] == "AB 40"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- normalise_session
|
||||
|
||||
class TestNormaliseSession:
|
||||
def test_standard_session(self):
|
||||
assert normalise_session("20252026") == "2025-2026"
|
||||
|
||||
def test_empty(self):
|
||||
assert normalise_session("") == ""
|
||||
|
||||
def test_non_session_passthrough(self):
|
||||
assert normalise_session("garbage") == "garbage"
|
||||
|
||||
def test_four_digit_passthrough(self):
|
||||
assert normalise_session("2025") == "2025"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- origin_house
|
||||
|
||||
class TestOriginHouse:
|
||||
def test_assembly_prefixes(self):
|
||||
for chamber in ("AB", "ACR", "HR", "ABR"):
|
||||
assert origin_house(chamber) == "assembly", chamber
|
||||
|
||||
def test_senate_prefixes(self):
|
||||
for chamber in ("SB", "SCR", "SJR"):
|
||||
assert origin_house(chamber) == "senate", chamber
|
||||
|
||||
def test_unknown(self):
|
||||
assert origin_house("ZZ") == "unknown"
|
||||
assert origin_house("") == "unknown"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- parse_display
|
||||
|
||||
class TestParseDisplay:
|
||||
def test_padded(self):
|
||||
assert parse_display("AB 00028") == ("AB", 28)
|
||||
|
||||
def test_unpadded(self):
|
||||
assert parse_display("AB 28") == ("AB", 28)
|
||||
|
||||
def test_various_prefixes(self):
|
||||
assert parse_display("SB 795") == ("SB", 795)
|
||||
assert parse_display("ACR 5") == ("ACR", 5)
|
||||
assert parse_display("SCR 12") == ("SCR", 12)
|
||||
|
||||
def test_invalid(self):
|
||||
for garbage in ("", "SB", "AB 7 X", "XYZ 5", " "):
|
||||
assert parse_display(garbage) is None, garbage
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- parse_supplement
|
||||
|
||||
class TestParseSupplement:
|
||||
def test_standard(self):
|
||||
assert parse_supplement("Supplemental 2") == 2
|
||||
|
||||
def test_case_insensitive(self):
|
||||
assert parse_supplement("supplemental 1") == 1
|
||||
|
||||
def test_no_match(self):
|
||||
for text in ("", "plain text", "Supplemental", "Supp 3"):
|
||||
assert parse_supplement(text) is None, text
|
||||
@@ -0,0 +1,347 @@
|
||||
# Tests for the dashboard API layer against the frozen API_CONTRACT.md shapes.
|
||||
# Drives handlers via api.run(Request(...)) — no sockets — plus one in-process
|
||||
# ThreadingHTTPServer test. Builds a small temp DB (flat + relational +
|
||||
# bill_tags) so shapes + error codes + empty states are asserted.
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
import urllib.request
|
||||
|
||||
import pytest
|
||||
|
||||
from scraper import relational
|
||||
from scraper.schema import BillAction
|
||||
from scraper.io import write_sqlite
|
||||
from scraper.dashboard import api
|
||||
from scraper.dashboard.api import Request, Services
|
||||
from scraper.dashboard.prefs import PrefsStore, LocalAuth
|
||||
|
||||
|
||||
# ---- fixture: a small, controlled DB --------------------------------------
|
||||
|
||||
def build_small_db(path):
|
||||
recs = [
|
||||
BillAction(date="2026-09-10", house="assembly",
|
||||
record_type="bill", bill_number="AB 100",
|
||||
bill_id="202520260AB100", session="2025-2026",
|
||||
chamber_origin="assembly", author="Schiavo et al.",
|
||||
short_title="Waste: recycling standards",
|
||||
description="d", source="ASSEMBLY_DAILY_FILE"),
|
||||
BillAction(date="2026-09-10", house="assembly",
|
||||
record_type="bill", bill_number="AB 200",
|
||||
bill_id="202520260AB200", session="2025-2026",
|
||||
chamber_origin="assembly", author="Macedo",
|
||||
short_title="Education: K-12 funding",
|
||||
description="d", source="ASSEMBLY_DAILY_FILE"),
|
||||
BillAction(date="2026-09-05", house="assembly",
|
||||
record_type="bill", bill_number="SB 300",
|
||||
bill_id="202520260SB300", session="2025-2026",
|
||||
chamber_origin="assembly", author="Unrelated",
|
||||
short_title="Old bill, past date",
|
||||
description="d", source="ASSEMBLY_DAILY_FILE"),
|
||||
]
|
||||
n = write_sqlite(recs, path, table=relational.FLAT_TABLE)
|
||||
assert n == 3
|
||||
relational.correlate(path, quiet=True)
|
||||
conn = relational.connect(path, foreign_keys=False)
|
||||
try:
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS bill_tags ("
|
||||
"id INTEGER PRIMARY KEY, bill_id TEXT NOT NULL, "
|
||||
"tag TEXT NOT NULL, source TEXT, generated_by TEXT NOT NULL, "
|
||||
"generated_at TEXT, UNIQUE (bill_id, tag, generated_by))")
|
||||
for bid, tag in ("202520260AB100", "waste"), \
|
||||
("202520260AB200", "education"), \
|
||||
("202520260AB200", "health"):
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO bill_tags "
|
||||
"(bill_id,tag,generated_by) VALUES (?,?,?)",
|
||||
(bid, tag, "fallback"))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def small_db(tmp_path):
|
||||
return build_small_db(str(tmp_path / "dash.sqlite"))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def services(tmp_path, small_db):
|
||||
prefs = PrefsStore(str(tmp_path / "prefs.json"))
|
||||
auth = LocalAuth(str(tmp_path / "auth.json"))
|
||||
return Services(small_db, prefs, auth)
|
||||
|
||||
|
||||
def _req(method, path, services, token=None, query=None, body=None,
|
||||
user_id=None):
|
||||
return Request(method, path, query=query or {}, body=body or {},
|
||||
token=token, services=services, user_id=user_id)
|
||||
|
||||
|
||||
# ---- /bills envelope (§1/§2) ----------------------------------------------
|
||||
|
||||
def test_bills_envelope_shape(services):
|
||||
status, body = api.run(_req("GET", "/api/v1/bills", services,
|
||||
query={"upcoming": "1",
|
||||
"as_of": "2026-09-06"}))
|
||||
assert status == 200, body
|
||||
for key in ("session", "as_of", "chamber_availability",
|
||||
"items", "groups"):
|
||||
assert key in body
|
||||
assert isinstance(body["items"], list)
|
||||
assert isinstance(body["groups"], dict)
|
||||
ids = {i["bill_id"] for i in body["items"]}
|
||||
assert "202520260AB100" in ids
|
||||
assert "202520260AB200" in ids
|
||||
assert "202520260SB300" not in ids
|
||||
item = body["items"][0]
|
||||
for key in ("bill_id", "bill_number", "chamber_origin", "author",
|
||||
"short_title", "hearing_sort", "tags"):
|
||||
assert key in item
|
||||
assert isinstance(item["hearing_sort"], int)
|
||||
|
||||
|
||||
def test_bills_personal_returns_relevance_and_floats(services):
|
||||
reg = services.auth.register("u@example.com")
|
||||
# user watches "schiavo" and opts into "education"
|
||||
api.run(_req("POST", "/api/v1/prefs/watch", services,
|
||||
token=reg["token"],
|
||||
body={"legislator_id": "schiavo"}))
|
||||
api.run(_req("POST", "/api/v1/prefs/topics", services,
|
||||
token=reg["token"],
|
||||
body={"tag": "education"}))
|
||||
# personal + full + upcoming (as_of=2026-09-04, before all 3 bills' date)
|
||||
status, body = api.run(_req("GET", "/api/v1/bills", services,
|
||||
token=reg["token"],
|
||||
query={"upcoming": "1", "as_of": "2026-09-04",
|
||||
"personal": "1", "full": "1"}))
|
||||
assert status == 200, body
|
||||
by = {i["bill_id"]: i for i in body["items"]}
|
||||
assert "relevance" in by["202520260AB100"]
|
||||
assert by["202520260AB100"]["relevance"]["score"] > 0
|
||||
assert by["202520260AB200"]["relevance"]["score"] > 0
|
||||
# SB300 not relevant to the user (no match) — its score is 0
|
||||
assert "202520260SB300" in by
|
||||
assert by["202520260SB300"]["relevance"]["score"] == 0
|
||||
# float-to-top: the first item has a positive score (a floated bill)
|
||||
first = by[body["items"][0]["bill_id"]]
|
||||
assert first["relevance"]["score"] > 0
|
||||
assert by["202520260AB100"]["bill_id"] in \
|
||||
[i["bill_id"] for i in body["items"][:1]]
|
||||
# SB300 comes after the floated bills in the ordering
|
||||
order = [i["bill_id"] for i in body["items"]]
|
||||
assert order.index("202520260SB300") > \
|
||||
max(order.index("202520260AB100"),
|
||||
order.index("202520260AB200")), order
|
||||
|
||||
|
||||
def test_bills_personal_requires_auth(services):
|
||||
status, body = api.run(_req("GET", "/api/v1/bills", services,
|
||||
query={"personal": "1"}))
|
||||
assert status == 401, body
|
||||
assert body["error"]["code"] == "not_authenticated"
|
||||
|
||||
|
||||
# ---- six empty states -> 200 items:[] (never 4xx/5xx) ---------------------
|
||||
|
||||
def test_empty_no_next_hearing(services):
|
||||
status, body = api.run(_req("GET", "/api/v1/bills", services,
|
||||
query={"from": "2030-01-01",
|
||||
"to": "2030-12-31"}))
|
||||
assert status == 200 and body["items"] == [] and body["groups"] == {}
|
||||
|
||||
|
||||
def test_empty_filter_returns_nothing(services):
|
||||
status, body = api.run(_req("GET", "/api/v1/bills", services,
|
||||
query={"upcoming": "1",
|
||||
"as_of": "2030-01-01",
|
||||
"tags": "education"}))
|
||||
assert status == 200 and body["items"] == []
|
||||
|
||||
|
||||
def test_empty_prefs_not_set(services):
|
||||
reg = services.auth.register("nobody@example.com")
|
||||
status, body = api.run(_req("GET", "/api/v1/prefs", services,
|
||||
token=reg["token"]))
|
||||
# register seeds an empty-blob, so 200 with empty prefs is valid;
|
||||
# a truly-empty store would 404 with prefs_not_set — accept both,
|
||||
# never 5xx.
|
||||
assert status in (200, 404)
|
||||
if status == 404:
|
||||
assert body["error"]["code"] == "prefs_not_set"
|
||||
|
||||
|
||||
def test_empty_no_tagged_bills_and_chamber_pending(tmp_path, services):
|
||||
# a flat DB with no tag rows -> /tags returns 200 tags [] (§6.6)
|
||||
empty = str(tmp_path / "notags.sqlite")
|
||||
write_sqlite([BillAction(bill_number="AB 1",
|
||||
bill_id="202520260AB1", session="2025-2026",
|
||||
chamber_origin="assembly", author="Solo",
|
||||
short_title="x",
|
||||
source="ASSEMBLY_DAILY_FILE")],
|
||||
empty, table=relational.FLAT_TABLE)
|
||||
relational.migrate(empty)
|
||||
svc = Services(empty, PrefsStore(str(tmp_path / "p.json")),
|
||||
LocalAuth(str(tmp_path / "a.json")))
|
||||
status, body = api.run(_req("GET", "/api/v1/tags", svc))
|
||||
assert status == 200 and body["tags"] == []
|
||||
# and a Senate-only-empty window stays 200 (chamber pending, not error)
|
||||
status, body = api.run(_req("GET", "/api/v1/bills", svc,
|
||||
query={"from": "2030-01-01",
|
||||
"chamber": "SENATE"}))
|
||||
assert status == 200 and body["items"] == []
|
||||
|
||||
|
||||
# ---- /tags, /legislators, /districts (§4.1 / §3.2 / §4.2) -----------------
|
||||
|
||||
def test_tags_cloud(services):
|
||||
status, body = api.run(_req("GET", "/api/v1/tags", services))
|
||||
assert status == 200, body
|
||||
assert "tags" in body
|
||||
labels = {t["tag_id"] for t in body["tags"]}
|
||||
assert "tag:waste" in labels
|
||||
assert "tag:education" in labels
|
||||
assert all("count" in t and "label" in t for t in body["tags"])
|
||||
|
||||
|
||||
def test_legislators_browse(services):
|
||||
status, body = api.run(_req("GET", "/api/v1/legislators", services))
|
||||
assert status == 200
|
||||
assert "items" in body
|
||||
ids = {x["legislator_id"] for x in body["items"]}
|
||||
assert "schiavo" in ids
|
||||
|
||||
|
||||
def test_districts_list(services):
|
||||
status, body = api.run(_req("GET", "/api/v1/districts", services))
|
||||
assert status == 200
|
||||
assert "items" in body and body["items"]
|
||||
|
||||
|
||||
# ---- /districts/resolve (§3.1) — confidence, proposes (no persist) ---------
|
||||
|
||||
def test_resolve_single(services):
|
||||
status, body = api.run(_req("POST", "/api/v1/districts/resolve",
|
||||
services, body={"zip": "90210"}))
|
||||
assert status == 200, body
|
||||
assert body["confidence"] == "single"
|
||||
assert body["resolved_from"] == "zip"
|
||||
assert body["suggested_watch"] and len(body["districts"]) == 1
|
||||
|
||||
|
||||
def test_resolve_ambiguous_lists_candidates(services):
|
||||
status, body = api.run(_req("POST", "/api/v1/districts/resolve",
|
||||
services, body={"zip": "90012"}))
|
||||
assert status == 200, body
|
||||
assert body["confidence"] == "ambiguous"
|
||||
assert len(body["districts"]) >= 2
|
||||
|
||||
|
||||
def test_resolve_zero_and_out_of_state(services):
|
||||
s0, b0 = api.run(_req("POST", "/api/v1/districts/resolve",
|
||||
services, body={"zip": "00001"}))
|
||||
assert s0 == 404 and b0["error"]["code"] == "district_not_found"
|
||||
# out-of-state prefix (99xxx / 990xx) -> 400 out_of_state
|
||||
s2, b2 = api.run(_req("POST", "/api/v1/districts/resolve",
|
||||
services, body={"zip": "99001"}))
|
||||
assert s2 == 400 and b2["error"]["code"] == "out_of_state"
|
||||
# manual pick is the fallback and always resolves (single)
|
||||
s3, b3 = api.run(_req("POST", "/api/v1/districts/resolve",
|
||||
services, body={"district": "A-14"}))
|
||||
assert s3 == 200 and b3["confidence"] == "single"
|
||||
|
||||
|
||||
def test_resolve_missing_input_is_400(services):
|
||||
status, body = api.run(_req("POST", "/api/v1/districts/resolve",
|
||||
services, body={}))
|
||||
assert status == 400 and "field" in body["error"]
|
||||
|
||||
|
||||
# ---- prefs + watch/topic (§3.3/§3.4/§3.5) ---------------------------------
|
||||
|
||||
def test_prefs_put_and_get_roundtrip(services):
|
||||
reg = services.auth.register("prefs@example.com")
|
||||
token = reg["token"]
|
||||
status, body = api.run(_req("PUT", "/api/v1/prefs", services,
|
||||
token=token,
|
||||
body={"zip": "90210",
|
||||
"watched_legislators": ["schiavo"],
|
||||
"opted_in_tags": ["waste"]}))
|
||||
assert status == 200, body
|
||||
_, got = api.run(_req("GET", "/api/v1/prefs", services, token=token))
|
||||
assert got["zip"] == "90210"
|
||||
assert "schiavo" in got["watched_legislators"]
|
||||
assert "waste" in got["opted_in_tags"]
|
||||
|
||||
|
||||
def test_prefs_put_rejects_unknown_tag(services):
|
||||
reg = services.auth.register("bad@example.com")
|
||||
status, body = api.run(_req("PUT", "/api/v1/prefs", services,
|
||||
token=reg["token"],
|
||||
body={"opted_in_tags": ["nope-not-tag"]}))
|
||||
assert status == 400 and body["error"]["code"] == "tag_unknown"
|
||||
|
||||
|
||||
def test_add_remove_watch_is_idempotent(services):
|
||||
reg = services.auth.register("w@example.com")
|
||||
token = reg["token"]
|
||||
api.run(_req("POST", "/api/v1/prefs/watch", services, token=token,
|
||||
body={"legislator_id": "schiavo"}))
|
||||
# add again -> idempotent, still exactly one
|
||||
api.run(_req("POST", "/api/v1/prefs/watch", services, token=token,
|
||||
body={"legislator_id": "schiavo"}))
|
||||
_, got = api.run(_req("GET", "/api/v1/prefs", services, token=token))
|
||||
assert got["watched_legislators"] == ["schiavo"]
|
||||
# remove -> gone
|
||||
api.run(_req("DELETE", "/api/v1/prefs/watch/schiavo", services,
|
||||
token=token))
|
||||
_, got2 = api.run(_req("GET", "/api/v1/prefs", services, token=token))
|
||||
assert "schiavo" not in got2["watched_legislators"]
|
||||
|
||||
|
||||
def test_add_unknown_legislator_is_400(services):
|
||||
reg = services.auth.register("l@example.com")
|
||||
status, body = api.run(_req("POST", "/api/v1/prefs/watch", services,
|
||||
token=reg["token"],
|
||||
body={"legislator_id": "phantom-leg"}))
|
||||
assert status == 400 and body["error"]["code"] == "legislator_unknown"
|
||||
|
||||
|
||||
# ---- router: unknown endpoint -> 404, not a crash -------------------------
|
||||
|
||||
def test_unknown_endpoint_404(services):
|
||||
status, body = api.run(_req("GET", "/api/v1/nope", services))
|
||||
assert status == 404 and body["error"]["code"] == "not_found"
|
||||
|
||||
|
||||
# ---- live server (real socket, in-process, no external port) --------------
|
||||
|
||||
def test_live_server_serves_bills_and_static(tmp_path, small_db):
|
||||
from scraper.dashboard import server as srv
|
||||
prefs = PrefsStore(str(tmp_path / "prefs.json"))
|
||||
auth = LocalAuth(str(tmp_path / "auth.json"))
|
||||
services = Services(small_db, prefs, auth)
|
||||
handler = srv.make_handler(services)
|
||||
httpd = srv.ThreadingHTTPServer(("127.0.0.1", 0), handler)
|
||||
port = httpd.server_address[1]
|
||||
t = threading.Thread(target=httpd.serve_forever, daemon=True)
|
||||
t.start()
|
||||
try:
|
||||
url = ("http://127.0.0.1:%d/api/v1/bills?upcoming=1"
|
||||
"&as_of=2026-09-06" % port)
|
||||
with urllib.request.urlopen(url, timeout=5) as resp:
|
||||
assert resp.status == 200
|
||||
payload = json.loads(resp.read().decode("utf-8"))
|
||||
assert "items" in payload
|
||||
with urllib.request.urlopen(
|
||||
"http://127.0.0.1:%d/index.html" % port, timeout=5) as resp:
|
||||
assert resp.status == 200
|
||||
html = resp.read().decode("utf-8")
|
||||
assert "<html" in html.lower()
|
||||
finally:
|
||||
httpd.shutdown()
|
||||
httpd.server_close()
|
||||
@@ -0,0 +1,216 @@
|
||||
"""Tests for the dashboard ranking / float-to-top logic (spec §5, DESIGN §4).
|
||||
|
||||
Pure-logic coverage of the heart of §5, no DB:
|
||||
|
||||
* a WATCHED-LEGISLATOR match raises a bill's score so it floats to the top;
|
||||
* a TAG match (opted-in topic) raises a bill's score so it floats to the top;
|
||||
* ANON view has NO relevance and NO personalized sort (personal=off excludes it);
|
||||
* hearing_sort orders "Upon Call of the Chair" last, times asc;
|
||||
* a T-1 day is excluded from an upcoming view;
|
||||
* "Why is this here?" (matched[]) transparency carries the trigger.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from scraper.dashboard import ranking
|
||||
from scraper.dashboard.ranking import (
|
||||
hearing_sort, score_bill, normalise_bill, float_to_top,
|
||||
personal_band, full_queue, is_floated, Weights,
|
||||
)
|
||||
from scraper.dashboard.context import RelevanceContext, from_prefs
|
||||
|
||||
|
||||
def _bill(bill_id, author, committee=None, tags=None, chamber="ASSEMBLY",
|
||||
hearing_time="9:30am", date="2026-09-07",
|
||||
short_title="t", description="d"):
|
||||
b = {
|
||||
"bill_id": bill_id,
|
||||
"bill_number": bill_id,
|
||||
"chamber_origin": chamber,
|
||||
"author": author,
|
||||
"short_title": short_title,
|
||||
"description": description,
|
||||
"committee": committee,
|
||||
"hearing_time": hearing_time,
|
||||
"location": "Room 1",
|
||||
"date": date,
|
||||
"session": "2025-2026",
|
||||
"item_number": 1,
|
||||
"supplement": 1,
|
||||
"source": "ASSEMBLY_DAILY_FILE",
|
||||
"tags": tags,
|
||||
}
|
||||
b["hearing_sort"] = hearing_sort(hearing_time)
|
||||
return b
|
||||
|
||||
|
||||
def _bills():
|
||||
# Three bills, all on the same day. B2 is authored by a watched legislator;
|
||||
# B3 is in a topic (education) the user opted into; B1 matches nothing.
|
||||
return [
|
||||
_bill("A1", "Unrelated McPerson", committee="General",
|
||||
tags=["budget"], date="2026-09-07"),
|
||||
_bill("A2", "Schiavo et al.", committee="Resources and Water",
|
||||
date="2026-09-07"), # author watched path
|
||||
_bill("A3", "Macedo", committee=None, tags=["education", "health"],
|
||||
date="2026-09-07"), # tag match path
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# hearing_sort
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_hearing_sort_walltime_ascending():
|
||||
assert hearing_sort("9:30am") == 9 * 3600 + 30 * 60
|
||||
assert hearing_sort("10:00am") > hearing_sort("9:30am")
|
||||
assert hearing_sort("1:00pm") > hearing_sort("12:00pm") # 1pm = 13:00
|
||||
assert hearing_sort("12:00am") == 0 # midnight
|
||||
|
||||
|
||||
def test_hearing_sort_call_of_chair_and_null_sort_last():
|
||||
last = hearing_sort("Upon Call of the Chair")
|
||||
assert last == hearing_sort("Call of the Chair")
|
||||
assert last == hearing_sort(None) # missing time sinks (time TBD)
|
||||
assert last == hearing_sort("")
|
||||
# a real wall time sorts before call-of-the-chair / nulls.
|
||||
assert hearing_sort("9:30am") < last
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# score_bill — the OR rule (author OR tag suffices to float)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_watched_author_floats():
|
||||
b = _bills()[1] # authored by "Schiavo et al."
|
||||
rel = score_bill(b, watched_authors={"schiavo"},
|
||||
watched_committees=set(), opted_in_tags=set())
|
||||
assert rel["score"] >= 3, rel
|
||||
assert is_floated({**b, "relevance": rel})
|
||||
# transparency: matched[] names the rule (a "Why is this here?" trigger)
|
||||
types = [m["type"] for m in rel["matched"]]
|
||||
assert "watched-legislator" in types
|
||||
|
||||
|
||||
def test_opted_in_tag_floats():
|
||||
# A bill that is NOT authored by a watched leg but carries an opted-in
|
||||
# topic floats on the TAG path alone.
|
||||
b = _bills()[2] # tags ["education","health"]
|
||||
rel = score_bill(b, watched_authors=set(),
|
||||
watched_committees=set(),
|
||||
opted_in_tags={"education"})
|
||||
assert rel["score"] > 0, rel
|
||||
assert is_floated({**b, "relevance": rel})
|
||||
types = [m["type"] for m in rel["matched"]]
|
||||
assert "tag" in types
|
||||
# matched carries the actual tag that triggered it
|
||||
assert any(m["value"].lower().startswith("topic")
|
||||
for m in rel["matched"])
|
||||
|
||||
|
||||
def test_no_match_does_not_float():
|
||||
# A bill matching nothing scores 0 and does NOT float.
|
||||
b = _bills()[0]
|
||||
rel = score_bill(b, watched_authors=set(),
|
||||
watched_committees=set(), opted_in_tags=set())
|
||||
assert rel["score"] == 0
|
||||
assert not is_floated({**b, "relevance": rel})
|
||||
|
||||
|
||||
def test_both_paths_stack_score():
|
||||
# Author watched + an opted-in tag both present -> score > either alone.
|
||||
b = _bill("A2", "Schiavo et al.", tags=["education"])
|
||||
rel = score_bill(b, watched_authors={"schiavo"},
|
||||
watched_committees=set(), opted_in_tags={"education"})
|
||||
alone = score_bill(b, watched_authors={"schiavo"},
|
||||
watched_committees=set(), opted_in_tags=set())
|
||||
assert rel["score"] > alone["score"]
|
||||
assert {"watched-legislator", "tag"}.issubset(
|
||||
{m["type"] for m in rel["matched"]})
|
||||
|
||||
|
||||
def test_committee_of_watched_legislator_path():
|
||||
# A bill in a committee a watched legislator chairs/serves on floats even
|
||||
# with no author/tag match (committee->members join supplied).
|
||||
b = _bill("A4", "Otherperson", committee="Emergency Management",
|
||||
tags=[], )
|
||||
rel = score_bill(b, watched_authors=set(),
|
||||
watched_committees={"emergency-management"},
|
||||
opted_in_tags=set())
|
||||
assert rel["score"] > 0, rel
|
||||
assert "committee-of-watched-legislator" in [
|
||||
m["type"] for m in rel["matched"]]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# float-to-top order + anon exclusion of personalized sort
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _personal_bills():
|
||||
bills = _bills()
|
||||
ctx = from_prefs({"watched_legislators": ["schiavo"],
|
||||
"opted_in_tags": ["education"]})
|
||||
for b in bills:
|
||||
b["relevance"] = ctx.score(b)
|
||||
return bills, ctx
|
||||
|
||||
|
||||
def test_float_to_top_puts_relevant_first():
|
||||
bills, ctx = _personal_bills()
|
||||
assert ctx.is_personalised
|
||||
ordered = float_to_top(bills)
|
||||
# A2 (author) and A3 (tag) float ahead of A1 (no match).
|
||||
ids = [b["bill_id"] for b in ordered]
|
||||
assert ids.index("A2") < ids.index("A1")
|
||||
assert ids.index("A3") < ids.index("A1")
|
||||
# floated band = top N of the floated bills
|
||||
band = personal_band(bills)
|
||||
assert {b["bill_id"] for b in band} == {"A2", "A3"}
|
||||
assert all(is_floated(b) for b in band)
|
||||
|
||||
|
||||
def test_personal_band_bounded_to_n():
|
||||
bills, ctx = _personal_bills()
|
||||
band = personal_band(bills, n=1)
|
||||
assert len(band) == 1
|
||||
# higher score first (A2 = author+nothing vs A3 = tag+nothing both score
|
||||
# 3 and 2 respectively; A2 should lead when scores differ)
|
||||
a2 = next(b for b in bills if b["bill_id"] == "A2")
|
||||
a3 = next(b for b in bills if b["bill_id"] == "A3")
|
||||
assert a2["relevance"]["score"] >= a3["relevance"]["score"]
|
||||
assert band[0]["bill_id"] == "A2"
|
||||
|
||||
|
||||
def test_full_queue_promotes_but_hides_nothing():
|
||||
# Every bill remains present; only order changes (promote, not hide).
|
||||
bills, ctx = _personal_bills()
|
||||
ordered = full_queue(bills)
|
||||
assert {b["bill_id"] for b in ordered} == {"A1", "A2", "A3"}
|
||||
assert len(ordered) == 3
|
||||
|
||||
|
||||
def test_anon_view_has_no_relevance_and_no_personal_sort():
|
||||
# The anon envelope carries NO relevance block and is NOT float-sorted:
|
||||
# the personal=off path must not reorder anything.
|
||||
bills = _bills()
|
||||
anon = [dict(b, **{"relevance": None}) for b in bills]
|
||||
# strip relevance to mirror the anon projection
|
||||
anon = [{k: v for k, v in b.items() if k != "relevance"} for b in _bills()]
|
||||
assert all("relevance" not in b for b in anon)
|
||||
# anon ordering == stable input order (no float applied)
|
||||
ordered = float_to_top([
|
||||
{k: v for k, v in b.items() if k != "relevance"} for b in _bills()])
|
||||
assert [b["bill_id"] for b in ordered] == ["A1", "A2", "A3"]
|
||||
|
||||
|
||||
def test_weights_are_tunable():
|
||||
# D3: weights are knobs. Bumping the tag weight changes band ordering.
|
||||
bills, ctx = _personal_bills()
|
||||
hi = Weights(tag_watched=10)
|
||||
for b in bills:
|
||||
b["relevance"] = score_bill(b,
|
||||
watched_authors=ctx.watched_authors,
|
||||
watched_committees=ctx.watched_committees,
|
||||
opted_in_tags=ctx.opted_in_tags, weights=hi)
|
||||
ordered = float_to_top(bills)
|
||||
# A3 (tag, weight 10) now outranks A2 (author, weight 3)
|
||||
assert ordered[0]["bill_id"] == "A3"
|
||||
@@ -0,0 +1,286 @@
|
||||
"""Integration tests: ingest -> parse end-to-end, asserting the JSONL contract.
|
||||
|
||||
Scope for task t_8e5a3821 item (2): a test that runs ingest -> parse on a
|
||||
fixture and asserts the JSONL schema, plus the ingest-layer failure modes
|
||||
(expected-404 "not_found", antibot, transient/retry, cache-hit, request-log
|
||||
recording, alert-resilience) that the parser never sees.
|
||||
|
||||
No live network: the HTTP transport is faked (conftest.make_ingestor) so the
|
||||
REAL retry/backoff loop and the REAL cache are exercised against a fixture body.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
import pytest
|
||||
|
||||
from scraper import parsers
|
||||
from scraper.cache import RawCache
|
||||
from scraper.io import write_jsonl
|
||||
from scraper.request_log import RequestLog
|
||||
from scraper.alerts import ALERT_REGISTRY, register
|
||||
from scraper.schema import BillAction
|
||||
from conftest import make_ingestor, read_jsonl, CONFIG_PATH
|
||||
from tests import fixtures as F
|
||||
|
||||
|
||||
# The canonical JSONL field set a downstream consumer may see (spec section 7).
|
||||
KNOWN_FIELDS = set(BillAction.field_names())
|
||||
BILL_ID_RE = re.compile(r"^\d{8}\d?(AB|SB|ACR|SCR|HR|SJR|ABR)\d+$")
|
||||
|
||||
|
||||
def assert_schema(rows):
|
||||
"""Every JSONL row must contain only known, well-typed fields."""
|
||||
assert rows, "no records were written"
|
||||
for row in rows:
|
||||
extra = set(row.keys()) - KNOWN_FIELDS
|
||||
assert not extra, "unknown fields in JSONL row: %s" % extra
|
||||
assert isinstance(row.get("record_type"), str)
|
||||
assert row["record_type"] in ("bill", "hearing")
|
||||
if row["record_type"] == "bill":
|
||||
if row.get("bill_id"):
|
||||
assert BILL_ID_RE.match(row["bill_id"]), row["bill_id"]
|
||||
assert row["bill_id"][:8].isdigit(), row["bill_id"]
|
||||
assert row.get("bill_number") is not None or row.get("bill_id")
|
||||
assert row.get("id")
|
||||
assert len(row["id"]) == 40, "bad id length: %s" % row
|
||||
|
||||
|
||||
class TestIngestToParseEndToEnd:
|
||||
def test_happy_path_ingest_then_parse_then_jsonl(self, tmp_path,
|
||||
monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
asm = F.assembly_daily_file_html()
|
||||
raw_dir = str(tmp_path / "cache" / "raw")
|
||||
log_path = str(tmp_path / "cache" / "req.jsonl")
|
||||
ing, transport = make_ingestor(
|
||||
CONFIG_PATH, raw_dir, log_path,
|
||||
route=lambda url: (200, asm.encode("utf-8")),
|
||||
)
|
||||
res = ing.ingest("2026-08-31", source="assembly_daily_file")
|
||||
assert res.all_ok
|
||||
cr = res.chambers[0]
|
||||
assert cr.status == "ok"
|
||||
assert cr.bytes == len(asm.encode("utf-8"))
|
||||
assert cr.path is not None and cr.path.exists()
|
||||
|
||||
# The cached raw file must itself be parseable by the real parser.
|
||||
recs = parsers.parse(str(cr.path))
|
||||
assert recs, "parser produced no records from the cached HTML"
|
||||
# Re-parse the same cache file: deterministic, same record count.
|
||||
recs2 = parsers.parse(str(cr.path))
|
||||
assert len(recs2) == len(recs)
|
||||
|
||||
# Serialise to JSONL and assert the schema contract on the real output.
|
||||
out = tmp_path / "records.jsonl"
|
||||
written = write_jsonl(recs, str(out))
|
||||
assert written == len(recs)
|
||||
rows = read_jsonl(str(out))
|
||||
assert len(rows) == len(recs)
|
||||
assert_schema(rows)
|
||||
# The fixture's known records must survive the whole pipeline.
|
||||
bns = {r["bill_number"] for r in rows if r["record_type"] == "bill"}
|
||||
assert "AB 28" in bns and "SB 795" in bns
|
||||
|
||||
def test_request_log_records_the_ingest(self, tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
asm = F.assembly_daily_file_html()
|
||||
log_path = str(tmp_path / "cache" / "req.jsonl")
|
||||
ing, _ = make_ingestor(
|
||||
CONFIG_PATH, str(tmp_path / "cache" / "raw"), log_path,
|
||||
route=lambda url: (200, asm.encode()),
|
||||
)
|
||||
ing.ingest("2026-08-31", source="assembly_daily_file")
|
||||
events = RequestLog.read(log_path)
|
||||
assert events, "request log is empty"
|
||||
for e in events:
|
||||
for key in ("ts", "source", "url", "status", "bytes",
|
||||
"attempt", "outcome", "log_ts"):
|
||||
assert key in e, "missing key %s in request-log event" % key
|
||||
assert e["status"] == 200
|
||||
assert e["source"] == "assembly_daily_file"
|
||||
|
||||
|
||||
class TestIngestFailureModes:
|
||||
def test_expected_404_is_not_an_error(self, tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
ing, _ = make_ingestor(
|
||||
CONFIG_PATH, str(tmp_path / "cache" / "raw"),
|
||||
str(tmp_path / "req.jsonl"),
|
||||
route=lambda url: (404, b""),
|
||||
)
|
||||
res = ing.ingest("2026-09-04", source="senate_daily_file")
|
||||
cr = res.chambers[0]
|
||||
assert cr.status == "not_found", cr.status
|
||||
assert "expected" in cr.note.lower()
|
||||
assert cr.status not in ("antibot", "transient")
|
||||
|
||||
def test_unexpected_404_on_assembly_is_a_warning_not_silent(
|
||||
self, tmp_path, monkeypatch):
|
||||
# The Assembly source has no missing_is_expected_404, so its 404 is a
|
||||
# real problem: not_found with an "unexpect..." note, not "expected".
|
||||
monkeypatch.chdir(tmp_path)
|
||||
ing, _ = make_ingestor(
|
||||
CONFIG_PATH, str(tmp_path / "cache" / "raw"),
|
||||
str(tmp_path / "req.jsonl"),
|
||||
route=lambda url: (404, b""),
|
||||
)
|
||||
res = ing.ingest("2026-09-04", source="assembly_daily_file")
|
||||
cr = res.chambers[0]
|
||||
assert cr.status == "not_found", cr.status
|
||||
# Match on "unexpect": the substring "expected" also appears inside
|
||||
# the word "unEXPEcted", so a naive substring check is ambiguous.
|
||||
note = cr.note.lower()
|
||||
assert "unexpect" in note, cr.note
|
||||
|
||||
def test_antibot_403_is_flagged(self, tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
ing, _ = make_ingestor(
|
||||
CONFIG_PATH, str(tmp_path / "cache" / "raw"),
|
||||
str(tmp_path / "req.jsonl"),
|
||||
route=lambda url: (403, b"<html>antibot</html>"),
|
||||
)
|
||||
res = ing.ingest("2026-09-02", source="senate_daily_file")
|
||||
cr = res.chambers[0]
|
||||
assert cr.status == "antibot", cr.status
|
||||
assert res.all_ok is False
|
||||
|
||||
def test_transient_retries_three_times(self, tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
ing, transport = make_ingestor(
|
||||
CONFIG_PATH, str(tmp_path / "cache" / "raw"),
|
||||
str(tmp_path / "req.jsonl"),
|
||||
route=lambda url: (503, b"busy"),
|
||||
)
|
||||
res = ing.ingest("2026-09-02", source="assembly_daily_file")
|
||||
cr = res.chambers[0]
|
||||
assert cr.status == "transient", cr.status
|
||||
assert cr.attempts == 3, cr.attempts
|
||||
# Three attempts -> two back-off sleeps (1s, 2s), recorded not slept.
|
||||
assert len(transport.calls) == 3, transport.calls
|
||||
assert ing._sleeps == [1.0, 2.0], ing._sleeps
|
||||
|
||||
|
||||
# QA FINDING (P2): scraper/http_client.py has a catch-all "return 'transient'"
|
||||
# for any non-404/non-antibot status, and the retry loop treats *any* transient
|
||||
# as retryable. Net effect: a permanent 4xx such as 400 is retry-stormed 3x
|
||||
# exactly like a transient 5xx, even though the code's own comment says it
|
||||
# "should not retry-storm". The xfail guard below pins the CURRENT (buggy)
|
||||
# behaviour as a regression trip-wire: when the engineer fixes the retry guard
|
||||
# to give up on non-retryable 4xx, the xfail PASSES and -- being strict -- flips
|
||||
# to a failure signalling "finding resolved, update the guard". See QA_CHECKLIST.
|
||||
class TestRetryStormRegressionGuard:
|
||||
@pytest.mark.xfail(
|
||||
strict=True,
|
||||
reason="P2 finding: non-retryable 4xx (400) is retry-stormed 3x instead "
|
||||
"of giving up on attempt 1 (http_client.py catch-all transient). "
|
||||
"See QA_CHECKLIST.md. Remove this guard once the engine is fixed.")
|
||||
def test_non_retryable_4xx_must_give_up_on_attempt_1(
|
||||
self, tmp_path, monkeypatch):
|
||||
# Desired behaviour (currently unmet): a 400 is neither a 404, an
|
||||
# antibot, nor a transient-network fault, so it must NOT retry-storm --
|
||||
# one attempt, zero back-off sleeps.
|
||||
monkeypatch.chdir(tmp_path)
|
||||
ing, transport = make_ingestor(
|
||||
CONFIG_PATH, str(tmp_path / "cache" / "raw"),
|
||||
str(tmp_path / "req.jsonl"),
|
||||
route=lambda url: (400, b"bad request"),
|
||||
)
|
||||
res = ing.ingest("2026-09-02", source="assembly_daily_file")
|
||||
cr = res.chambers[0]
|
||||
assert cr.status == "transient", cr.status
|
||||
assert cr.attempts == 1, "non-retryable 4xx must give up on attempt 1"
|
||||
assert len(ing._sleeps) == 0, "no backoff for a non-retryable 4xx"
|
||||
|
||||
def test_retry_storm_on_4xx_is_currently_3_attempts(self, tmp_path,
|
||||
monkeypatch):
|
||||
# Current, PASSING behaviour (the buggy side): a 400 really is retried
|
||||
# 3x with back-off [1.0, 2.0]. Pinned so the gate stays green while the
|
||||
# strict-xfail guard above records the defect. Remove when fixed.
|
||||
monkeypatch.chdir(tmp_path)
|
||||
ing, transport = make_ingestor(
|
||||
CONFIG_PATH, str(tmp_path / "cache" / "raw"),
|
||||
str(tmp_path / "req.jsonl"),
|
||||
route=lambda url: (400, b"bad request"),
|
||||
)
|
||||
res = ing.ingest("2026-09-02", source="assembly_daily_file")
|
||||
cr = res.chambers[0]
|
||||
assert cr.status == "transient", cr.status
|
||||
assert cr.attempts == 3, cr.attempts
|
||||
assert ing._sleeps == [1.0, 2.0], ing._sleeps
|
||||
|
||||
|
||||
class TestCacheIntegrity:
|
||||
def test_cache_hit_serves_from_disk_without_network(self, tmp_path,
|
||||
monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
asm = F.assembly_daily_file_html()
|
||||
raw_dir = str(tmp_path / "cache" / "raw")
|
||||
log_path = str(tmp_path / "cache" / "req.jsonl")
|
||||
# 1) First ingest: network miss -> downloads + caches.
|
||||
ing1, tp1 = make_ingestor(
|
||||
CONFIG_PATH, raw_dir, log_path,
|
||||
route=lambda url: (200, asm.encode()),
|
||||
)
|
||||
res1 = ing1.ingest("2026-08-31", source="assembly_daily_file")
|
||||
assert res1.chambers[0].status == "ok"
|
||||
assert len(tp1.calls) == 1
|
||||
|
||||
# 2) Second ingest, same cache, force=False -> cache hit, no network.
|
||||
ing2, tp2 = make_ingestor(
|
||||
CONFIG_PATH, raw_dir, log_path, route=None,
|
||||
)
|
||||
ing2.force = False
|
||||
res2 = ing2.ingest("2026-08-31", source="assembly_daily_file")
|
||||
cr2 = res2.chambers[0]
|
||||
assert cr2.status == "cached", cr2.status
|
||||
assert cr2.cache_hit is True
|
||||
assert tp2 is None # no fake transport wired -> no network on hit
|
||||
|
||||
def test_alert_sink_failure_does_not_crash_run(self, tmp_path,
|
||||
monkeypatch):
|
||||
# An alerting sink that raises must not crash the ingest run; the
|
||||
# failure is logged and falls back to the log sink.
|
||||
monkeypatch.chdir(tmp_path)
|
||||
asm = F.assembly_daily_file_html()
|
||||
|
||||
def _boom(ctx):
|
||||
raise RuntimeError("alert sink on fire")
|
||||
|
||||
register("_test_boom_sink", _boom)
|
||||
ing, _ = make_ingestor(
|
||||
CONFIG_PATH, str(tmp_path / "cache" / "raw"),
|
||||
str(tmp_path / "req.jsonl"),
|
||||
route=lambda url: (200, asm.encode()),
|
||||
)
|
||||
# Point the alert sink at the bomb (config is a live dict).
|
||||
ing.cfg._alerts["sink"] = "_test_boom_sink"
|
||||
res = ing.ingest("2026-08-31", source="assembly_daily_file")
|
||||
assert res.chambers[0].status == "ok", "alert failure must not break ingest"
|
||||
# Clean up the registry so the bomb sink does not leak into other tests.
|
||||
ALERT_REGISTRY.pop("_test_boom_sink", None)
|
||||
|
||||
def test_cache_roundtrip_preserves_bytes(self, tmp_path):
|
||||
cache = RawCache(str(tmp_path / "raw"))
|
||||
data = F.assembly_daily_file_html().encode("utf-8")
|
||||
path = cache.put(
|
||||
"assembly_daily_file", "2026-09-01", "html", data,
|
||||
url="http://example/a", status=200,
|
||||
fetched_at="2026-09-01T00:00:00+00:00",
|
||||
)
|
||||
assert path is not None
|
||||
got = cache.get("assembly_daily_file", "2026-09-01", "html")
|
||||
assert got == data
|
||||
assert cache.exists("assembly_daily_file", "2026-09-01", "html")
|
||||
|
||||
def test_cache_corruption_is_treated_as_a_miss(self, tmp_path):
|
||||
# A corrupt file (sha mismatch vs its meta) must be a cache miss, not a
|
||||
# poisoned hit.
|
||||
cache = RawCache(str(tmp_path / "raw"))
|
||||
bad = tmp_path / "raw" / "assembly_daily_file" / "2026-09-01.html"
|
||||
bad.parent.mkdir(parents=True, exist_ok=True)
|
||||
bad.write_text("garbage-not-the-cached-bytes")
|
||||
meta = cache.meta_path("assembly_daily_file", "2026-09-01")
|
||||
meta.write_text('{"sha256": "deadbeef", "bytes": 8}\n')
|
||||
assert cache.get("assembly_daily_file", "2026-09-01", "html") is None
|
||||
assert cache.exists("assembly_daily_file", "2026-09-01", "html") is False
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Optional live-URL smoke test (scope item 3).
|
||||
|
||||
Hits the real Assembly Daily File URL once, end-to-end through the real HTTP
|
||||
client, and parses the result. By default this test is SKIPPED so a CI run never
|
||||
depends on the external servers being up. Enable it with::
|
||||
|
||||
CALI_SCRAPER_SMOKE=1 python3 -m pytest tests/test_live_smoke.py -m smoke
|
||||
|
||||
The "smoke" marker is registered in pyproject.toml (--strict-markers). Because
|
||||
the live URL is the only external dependency in the whole suite, keeping it
|
||||
opt-in is what makes the rest of the CI signal deterministic.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _live_enabled() -> bool:
|
||||
# "1", "true", "yes", "on" (case-insensitive) all enable the smoke test.
|
||||
return os.environ.get("CALI_SCRAPER_SMOKE", "").strip().lower() in {
|
||||
"1", "true", "yes", "on",
|
||||
}
|
||||
|
||||
|
||||
# Skip the whole module unless the operator opts in.
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not _live_enabled(),
|
||||
reason="live smoke test disabled (set CALI_SCRAPER_SMOKE=1 to run)",
|
||||
)
|
||||
|
||||
|
||||
class TestLiveAssembly:
|
||||
"""Opt-in: fetch the live Assembly Daily File and parse it."""
|
||||
|
||||
def test_live_assembly_daily_file_returns_body(self, tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
from scraper.ingest import Ingestor
|
||||
config_path = os.environ.get("CALI_SCRAPER_CONFIG") \
|
||||
or os.path.join(os.path.dirname(os.path.dirname(__file__)),
|
||||
"config.yaml")
|
||||
ing = Ingestor(config_path=config_path, force=True)
|
||||
# Use today's date as the cache/manifest key; the Assembly HTML is the
|
||||
# always-current live file (spec section 1).
|
||||
import datetime
|
||||
today = datetime.date.today().isoformat()
|
||||
res = ing.ingest(today, source="assembly_daily_file")
|
||||
cr = res.chambers[0]
|
||||
# The only hard requirement for a smoke test: we got *something* back
|
||||
# that is a real HTML document (not an antibot/transient/404).
|
||||
assert cr.status == "ok", f"unexpected ingest status: {cr.status}"
|
||||
assert cr.bytes > 0, "empty body from live source"
|
||||
assert cr.path is not None and cr.path.exists()
|
||||
# And the parser must handle whatever it is.
|
||||
import scraper.parsers as parsers
|
||||
recs = parsers.parse(str(cr.path))
|
||||
# The Assembly file is always non-empty in practice, but a transient
|
||||
# skeleton day (recess) could legitimately yield zero records -- so we
|
||||
# only assert the batch did not crash, not a count.
|
||||
assert isinstance(recs, list)
|
||||
|
||||
def test_live_http_reachable_or_documented_antibot(self):
|
||||
# A second, independent liveness check: the Senate index (which is the
|
||||
# human page, not the antibot PDF) must answer without raising.
|
||||
try:
|
||||
import httpx
|
||||
except Exception as exc: # pragma: no cover - httpx is a hard dep
|
||||
pytest.skip(f"httpx unavailable: {exc}")
|
||||
with httpx.Client(timeout=20, follow_redirects=True,
|
||||
headers={"User-Agent": "research-scanner/1.0"}) as cli:
|
||||
try:
|
||||
resp = cli.get("https://www.senate.ca.gov/dailyfile")
|
||||
except httpx.HTTPError as exc:
|
||||
pytest.skip(f"network unavailable: {exc}")
|
||||
# A 2xx/308/403 (antibot) all mean "the server responded"; anything that
|
||||
# is not a connection failure is a pass for a *reachability* smoke test.
|
||||
assert resp.status_code in (200, 301, 302, 308, 403), resp.status_code
|
||||
@@ -0,0 +1,305 @@
|
||||
"""Tests for the Ollama / LLM analysis lane (spec 4, task t_950780e0).
|
||||
|
||||
Covers the acceptance criteria:
|
||||
|
||||
(a) fallback path with ``enabled=false`` -> ZERO network (built-in tagger);
|
||||
(b) a mocked Ollama response is parsed + stored (generated_by='ollama:m');
|
||||
(c) endpoint unreachable -> fallback, no crash;
|
||||
plus the tag-cloud aggregation + additive-table contract and the CLI wiring
|
||||
(analyze registers without clobbering sibling handlers).
|
||||
|
||||
The ollama path is never touched against the real network: a canned response is
|
||||
fed through the injectable ``post`` callable, and the fallback path is proven to
|
||||
need no network by asserting ``urllib.request.urlopen`` is never called.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
from scraper import llm
|
||||
from scraper import relational
|
||||
from scraper import analyze_cli
|
||||
from scraper.schema import BillAction
|
||||
from scraper.io import write_sqlite
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- helpers ---
|
||||
|
||||
def build_flat_db(path):
|
||||
recs = [
|
||||
BillAction(date="2026-08-31", house="assembly",
|
||||
record_type="hearing", committee="Emergency Management",
|
||||
hearing_time="9:30am", location="Room 444",
|
||||
source="ASSEMBLY_DAILY_FILE"),
|
||||
BillAction(date="2026-08-31", house="assembly", record_type="bill",
|
||||
bill_number="AB 28", bill_id="202520260AB28",
|
||||
session="2025-2026", chamber_origin="assembly",
|
||||
author="Macedo et al.",
|
||||
short_title=("An act relating to solid waste, and making "
|
||||
"an appropriation therefor."),
|
||||
committee="Emergency Management",
|
||||
source="ASSEMBLY_DAILY_FILE"),
|
||||
BillAction(date="2026-08-31", house="assembly", record_type="bill",
|
||||
bill_number="AB 568", bill_id="202520260AB568",
|
||||
session="2025-2026", chamber_origin="assembly",
|
||||
author="Macedo", short_title="An act relating to water "
|
||||
"districts.", source="ASSEMBLY_DAILY_FILE"),
|
||||
BillAction(date="2026-08-31", house="assembly", record_type="bill",
|
||||
bill_number="AB 7", bill_id="202520260AB7",
|
||||
session="2025-2026", chamber_origin="assembly",
|
||||
author="Ortega",
|
||||
short_title="An act relating to housing.",
|
||||
source="ASSEMBLY_DAILY_FILE"),
|
||||
]
|
||||
write_sqlite(recs, path, table=relational.FLAT_TABLE)
|
||||
return path
|
||||
|
||||
|
||||
def _fake_ollama_json(tags):
|
||||
return {"choices": [{"message": {"content": json.dumps(tags)}}]}
|
||||
|
||||
|
||||
def _fake_post(tags=None, raises=False):
|
||||
calls = []
|
||||
|
||||
def post(url, payload, timeout):
|
||||
calls.append(url)
|
||||
if raises:
|
||||
raise urllib.error.URLError("connection refused")
|
||||
return _fake_ollama_json(tags or [])
|
||||
|
||||
post.calls = calls # type: ignore[attr-defined]
|
||||
return post
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# (a) fallback path, enabled=false -> ZERO network
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_fallback_when_disabled_is_zero_network(tmp_path, monkeypatch):
|
||||
db = str(tmp_path / "ll_a.sqlite")
|
||||
build_flat_db(db)
|
||||
|
||||
def _deny(*a, **k):
|
||||
raise AssertionError("fallback path must not touch the network")
|
||||
|
||||
monkeypatch.setattr(urllib.request, "urlopen", _deny)
|
||||
|
||||
results = llm.tag_bills(llm.load_bills(db),
|
||||
cfg={"enabled": False, "model": "llama3",
|
||||
"endpoint": "http://localhost:11434"})
|
||||
assert len(results) == 3, "3 distinct bills expected"
|
||||
assert all(r["generated_by"] == llm.FALLBACK for r in results)
|
||||
assert all(r["tags"] for r in results), "every bill must get >=1 tag"
|
||||
|
||||
ab28 = next(r for r in results if r["bill_id"] == "202520260AB28")
|
||||
tags = [t for (t, _s) in ab28["tags"]]
|
||||
assert "solid waste" in tags, tags
|
||||
assert "solid waste & recycling" in tags or "budget" in tags, tags
|
||||
|
||||
|
||||
def test_tag_bill_fallback_never_invents():
|
||||
bill = {"bill_id": "X", "short_title": "An act relating to zzzzz."}
|
||||
tags = [t for (t, _s) in llm.tag_bill_fallback(bill)]
|
||||
assert tags, "must never return an empty tag list"
|
||||
|
||||
bill2 = {"bill_id": "Y", "short_title": "An act relating to water districts."}
|
||||
tags2 = [t for (t, _s) in llm.tag_bill_fallback(bill2)]
|
||||
assert "water" in tags2, tags2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# (b) mocked ollama response parsed + stored
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_mocked_ollama_parsed_and_stored(tmp_path):
|
||||
db = str(tmp_path / "ll_b.sqlite")
|
||||
build_flat_db(db)
|
||||
|
||||
post = _fake_post(tags=["housing", "residential rental"])
|
||||
results = llm.tag_bills(llm.load_bills(db),
|
||||
cfg={"enabled": True, "model": "llama3",
|
||||
"endpoint": "http://localhost:11434",
|
||||
"default_tags": 8},
|
||||
post=post)
|
||||
assert results, "expected results"
|
||||
assert all(r["generated_by"] == "ollama:llama3" for r in results), \
|
||||
[r["generated_by"] for r in results]
|
||||
ab568 = next(r for r in results if r["bill_id"] == "202520260AB568")
|
||||
tags = {t for (t, _s) in ab568["tags"]}
|
||||
assert "housing" in tags and "residential rental" in tags, tags
|
||||
assert all(s == "ollama" for (_t, s) in ab568["tags"]), ab568["tags"]
|
||||
|
||||
written = llm.persist_tags(db, results)
|
||||
rows = _fetch(db, "SELECT bill_id, tag, generated_by FROM bill_tags "
|
||||
"ORDER BY bill_id, tag")
|
||||
assert written > 0
|
||||
assert any(gb == "ollama:llama3" for (_b, _t, gb) in rows)
|
||||
assert len(rows) >= 2 * 3, "3 bills x >=2 tags each"
|
||||
|
||||
|
||||
def test_ollama_parse_json_and_fenced():
|
||||
# JSON array inside a code fence must be parsed (non-filler tags).
|
||||
assert llm._parse_tags_for_bill('```json\n["housing","bills"]\n```') == \
|
||||
["housing", "bills"]
|
||||
# Bare JSON array.
|
||||
assert llm._parse_tags_for_bill('[ "housing" , "foreclosure" ]') == \
|
||||
["housing", "foreclosure"]
|
||||
out = llm._parse_tags_for_bill("housing, foreclosure, education")
|
||||
assert "housing" in out and "foreclosure" in out, out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# (c) endpoint unreachable -> fallback, no crash
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_unreachable_endpoint_falls_back_no_crash(tmp_path):
|
||||
db = str(tmp_path / "ll_c.sqlite")
|
||||
build_flat_db(db)
|
||||
|
||||
post = _fake_post(raises=True)
|
||||
results = llm.tag_bills(llm.load_bills(db),
|
||||
cfg={"enabled": True, "model": "llama3",
|
||||
"endpoint": "http://localhost:11434",
|
||||
"timeout_seconds": 1},
|
||||
post=post)
|
||||
assert results, "must return results, not raise"
|
||||
assert all(r["generated_by"] == llm.FALLBACK for r in results)
|
||||
assert all(r["tags"] for r in results), "fallback must tag every bill"
|
||||
assert all("unreachable" in r["reason"] for r in results), \
|
||||
[r["reason"] for r in results]
|
||||
|
||||
|
||||
def test_mixed_run_reports_fallback_path(tmp_path):
|
||||
db = str(tmp_path / "ll_m.sqlite")
|
||||
build_flat_db(db)
|
||||
|
||||
def flaky(url, payload, timeout):
|
||||
return _fake_ollama_json([])
|
||||
|
||||
results = llm.tag_bills(llm.load_bills(db),
|
||||
cfg={"enabled": True, "model": "llama3",
|
||||
"endpoint": "http://localhost:11434"},
|
||||
post=flaky)
|
||||
dom = llm._dominant_generated_by(results)
|
||||
assert dom == llm.FALLBACK, "empty ollama answer -> fallback path reported"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# tag-cloud aggregation + additive table contract
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_build_tag_cloud_aggregates(tmp_path):
|
||||
db = str(tmp_path / "ll_cloud.sqlite")
|
||||
build_flat_db(db)
|
||||
llm.analyze(db_path=db, out_path=str(tmp_path / "cloud.json"),
|
||||
cfg={"enabled": False}, rebuild=True)
|
||||
cloud = llm.build_tag_cloud(db_path=db)
|
||||
assert cloud["bill_count"] == 3, cloud
|
||||
assert cloud["tag_count"] > 0
|
||||
counts = [t["count"] for t in cloud["tags"]]
|
||||
assert counts == sorted(counts, reverse=True), counts
|
||||
with open(str(tmp_path / "cloud.json"), encoding="utf-8") as fh:
|
||||
loaded = json.load(fh)
|
||||
assert loaded["bill_count"] == 3
|
||||
assert set(loaded) >= {"tag_count", "bill_count", "tags"}
|
||||
|
||||
|
||||
def test_bill_tags_table_is_additive_and_idempotent(tmp_path):
|
||||
db = str(tmp_path / "ll_idem.sqlite")
|
||||
build_flat_db(db)
|
||||
s1 = llm.analyze(db_path=db, cfg={"enabled": False}, rebuild=True)
|
||||
s2 = llm.analyze(db_path=db, cfg={"enabled": False}, rebuild=False)
|
||||
assert s1["rows_written"] > 0
|
||||
assert s2["rows_written"] == 0, "second run must add no rows (idempotent)"
|
||||
assert _count(db, relational.FLAT_TABLE) == 4
|
||||
|
||||
|
||||
def test_ensure_tag_table_does_not_touch_flat(tmp_path):
|
||||
db = str(tmp_path / "ll_flat.sqlite")
|
||||
build_flat_db(db)
|
||||
before = _count(db, relational.FLAT_TABLE)
|
||||
llm.ensure_tag_table(db)
|
||||
assert _count(db, relational.FLAT_TABLE) == before == 4
|
||||
assert llm.TAG_TABLE == "bill_tags"
|
||||
cols = _cols(db, relational.FLAT_TABLE)
|
||||
assert "tag" not in cols and "generated_by" not in cols
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI wiring: additive + guarded
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_analyze_registers_without_clobbering():
|
||||
from scraper import cli
|
||||
parser = cli.build_parser()
|
||||
a = parser.parse_args(["analyze", "--top", "5"])
|
||||
assert a.command == "analyze"
|
||||
assert a.top == 5
|
||||
assert parser.parse_args(["parse", "--input", "f.html"]).command == "parse"
|
||||
assert parser.parse_args(["migrate"]).command == "migrate"
|
||||
assert parser.parse_args(
|
||||
["ingest", "--date", "2026-09-01"]).command == "ingest"
|
||||
|
||||
|
||||
def test_handle_analyze_end_to_end(tmp_path):
|
||||
import argparse
|
||||
from conftest import CONFIG_PATH
|
||||
db = str(tmp_path / "ll_cli.sqlite")
|
||||
build_flat_db(db)
|
||||
out = str(tmp_path / "cloud.json")
|
||||
ns = argparse.Namespace(db=db, out_path=out, house=None, top=5,
|
||||
rebuild=True, force_llm=False, list_config=False,
|
||||
config=CONFIG_PATH)
|
||||
rc = analyze_cli.handle_analyze(ns)
|
||||
assert rc == 0
|
||||
with open(out, encoding="utf-8") as fh:
|
||||
cloud = json.load(fh)
|
||||
assert cloud["bill_count"] == 3
|
||||
assert cloud["tag_count"] > 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# config wiring
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_config_llm_section_is_additive():
|
||||
from scraper.config import load_config
|
||||
from conftest import CONFIG_PATH
|
||||
cfg = load_config(CONFIG_PATH)
|
||||
llm_cfg = cfg.llm
|
||||
assert llm_cfg["enabled"] is False
|
||||
assert llm_cfg["backend"] == "ollama"
|
||||
assert "http://localhost:11434" in llm_cfg["endpoint"]
|
||||
assert "http" in cfg.data and "sources" in cfg.data
|
||||
assert llm_cfg["default_tags"] == 8
|
||||
assert llm_cfg["timeout_seconds"] == 30
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# test helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _fetch(db, sql):
|
||||
c = relational.connect(db, foreign_keys=False)
|
||||
try:
|
||||
return [tuple(r) for r in c.execute(sql).fetchall()]
|
||||
finally:
|
||||
c.close()
|
||||
|
||||
|
||||
def _count(db, table):
|
||||
return _fetch(db, "SELECT COUNT(*) FROM " + table)[0][0]
|
||||
|
||||
|
||||
def _cols(db, table):
|
||||
c = relational.connect(db, foreign_keys=False)
|
||||
try:
|
||||
cur = c.execute("PRAGMA table_info(" + table + ")")
|
||||
return [row[1] for row in cur.fetchall()]
|
||||
finally:
|
||||
c.close()
|
||||
@@ -0,0 +1,286 @@
|
||||
"""Unit tests for scraper.parsers -- the HTML -> record transformation (spec section 3).
|
||||
|
||||
Scope for task t_8e5a3821 item (1):
|
||||
* fixtures for at least 3 dates, one per chamber, plus one edge-case file;
|
||||
* the archived Assembly sample must still parse to the expected record set;
|
||||
* malformed rows are skipped with a logged warning and never abort the batch;
|
||||
* cross-chamber origin, committee vs floor discrimination, and the
|
||||
data-sort preference are all asserted.
|
||||
|
||||
The parser is the coverage-gated module, so these tests aim for near-complete
|
||||
branch coverage of scraper.parsers and scraper.bill_id.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from scraper import parsers
|
||||
from scraper.parsers import normalise_date
|
||||
from tests import fixtures as F
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- fixtures
|
||||
#
|
||||
# Three legislative-day fixtures, both chambers, plus the edge-case file. Three
|
||||
# dates come from assembly_daily_file_html (2026-08-31), senate_daily_file_html
|
||||
# (2026-09-02) and extra_daily_file_html (2026-09-03).
|
||||
|
||||
class TestFixtureCoverage:
|
||||
def test_daily_file_fixtures_parse_without_raising(self):
|
||||
for name in ("assembly_daily_file_html", "senate_daily_file_html",
|
||||
"extra_daily_file_html"):
|
||||
recs = parsers.parse_html(getattr(F, name)())
|
||||
# Each full daily file carries at least one bill and one hearing.
|
||||
assert len(recs) >= 2, f"{name} produced no records"
|
||||
assert any(r.record_type == "bill" for r in recs), f"{name}: no bills"
|
||||
assert any(r.record_type == "hearing" for r in recs), f"{name}: no hearings"
|
||||
|
||||
def test_edge_fixture_parses_without_raising(self):
|
||||
# The edge-case file is deliberately bills-only and malformed; it must
|
||||
# parse without raising and must keep the recoverable rows.
|
||||
recs = parsers.parse_html(F.edge_case_html())
|
||||
assert {r.bill_number for r in recs} >= {"AB 100", "AB 300", "AB 150"}
|
||||
|
||||
def test_three_distinct_dates(self):
|
||||
dates = set()
|
||||
for name in ("assembly_daily_file_html", "senate_daily_file_html",
|
||||
"extra_daily_file_html"):
|
||||
recs = parsers.parse_html(getattr(F, name)())
|
||||
d = next((r.date for r in recs if r.date), None)
|
||||
assert d is not None, f"{name}: no date derived from hearing table"
|
||||
dates.add(d)
|
||||
assert len(dates) == 3, f"expected 3 distinct dates, got {dates}"
|
||||
|
||||
def test_both_chambers_covered(self):
|
||||
origins = set()
|
||||
for name in ("assembly_daily_file_html", "senate_daily_file_html",
|
||||
"extra_daily_file_html"):
|
||||
for r in parsers.parse_html(getattr(F, name)()):
|
||||
if r.chamber_origin:
|
||||
origins.add(r.chamber_origin.upper())
|
||||
assert {"ASSEMBLY", "SENATE"} <= origins, f"missing a chamber: {origins}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the archived sample
|
||||
|
||||
class TestArchivedSample:
|
||||
def test_sample_parses_expected_counts(self):
|
||||
recs = parsers.parse("samples/assembly_dailyfile_latest.html")
|
||||
by = {}
|
||||
for r in recs:
|
||||
by[r.record_type] = by.get(r.record_type, 0) + 1
|
||||
# Grounded values (verified 2026-09-05 against the 401 192-byte sample):
|
||||
# 156 floor + 1 committee = 157 bills, 4 hearings.
|
||||
assert by.get("bill", 0) >= 150, by
|
||||
assert by.get("hearing", 0) >= 4, by
|
||||
|
||||
def test_sample_has_ab28(self):
|
||||
recs = parsers.parse("samples/assembly_dailyfile_latest.html")
|
||||
ab28 = next((r for r in recs if r.bill_number == "AB 28"), None)
|
||||
assert ab28 is not None
|
||||
assert ab28.bill_id == "202520260AB28"
|
||||
assert ab28.session == "2025-2026"
|
||||
assert ab28.chamber_origin == "ASSEMBLY"
|
||||
assert ab28.item_number == 32
|
||||
assert ab28.supplement == 2
|
||||
|
||||
def test_sample_cross_chamber(self):
|
||||
recs = parsers.parse("samples/assembly_dailyfile_latest.html")
|
||||
origins = {r.chamber_origin for r in recs if r.record_type == "bill"}
|
||||
assert "ASSEMBLY" in origins and "SENATE" in origins, origins
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- floor / committee
|
||||
|
||||
_FIXTURE_BUILDERS = ("assembly_daily_file_html", "senate_daily_file_html",
|
||||
"extra_daily_file_html")
|
||||
|
||||
|
||||
class TestRecordTypes:
|
||||
@pytest.mark.parametrize("builder_name", _FIXTURE_BUILDERS)
|
||||
def test_bills_cannot_be_hearings(self, builder_name):
|
||||
recs = parsers.parse_html(getattr(F, builder_name)())
|
||||
for r in recs:
|
||||
if r.record_type == "bill":
|
||||
assert not r.hearing_time, "bill should not carry a hearing time"
|
||||
assert r.raw_selector in ("floor-bills-table", "committee-bills-table")
|
||||
if r.record_type == "hearing":
|
||||
assert r.hearing_time is not None
|
||||
assert r.raw_selector == "committee-hearing-table"
|
||||
# Hearings carry no bill identity.
|
||||
assert r.bill_number is None
|
||||
assert r.bill_id is None
|
||||
|
||||
def test_floor_vs_committee_discrimination(self):
|
||||
recs = parsers.parse_html(F.assembly_daily_file_html())
|
||||
floor = [r for r in recs if r.raw_selector == "floor-bills-table"]
|
||||
comm = [r for r in recs if r.raw_selector == "committee-bills-table"]
|
||||
assert floor, "no floor records"
|
||||
assert comm, "no committee records"
|
||||
# Floor rows have an item_number; committee rows do not.
|
||||
assert all(r.item_number is not None for r in floor)
|
||||
assert all(r.item_number is None for r in comm)
|
||||
# Committee rows carry a committee name; floor rows do not.
|
||||
assert all(r.committee is None for r in floor)
|
||||
assert all(r.committee for r in comm)
|
||||
|
||||
def test_committee_bill_has_session_from_id(self):
|
||||
recs = parsers.parse_html(F.assembly_daily_file_html())
|
||||
sb795 = next(r for r in recs if r.bill_number == "SB 795")
|
||||
assert sb795.session == "2025-2026"
|
||||
assert sb795.chamber_origin == "SENATE"
|
||||
assert sb795.committee == "Emergency Management"
|
||||
|
||||
def test_display_only_row_falls_back_to_display_text(self):
|
||||
# EDGE_DISPLAY_ONLY has no bill_id in the link; the parser must recover
|
||||
# "AB 150" from the clean display text (bill_id stays None).
|
||||
recs = parsers.parse_html(F.edge_case_html())
|
||||
ab150 = next((r for r in recs if r.bill_number == "AB 150"), None)
|
||||
assert ab150 is not None
|
||||
assert ab150.bill_id is None # no decodable id available
|
||||
assert ab150.chamber_origin == "assembly" # origin derived from display
|
||||
assert ab150.item_number == 150
|
||||
|
||||
def test_data_sort_preferred_over_text(self):
|
||||
# A floor row whose text-content and data-sort disagree: the parser must
|
||||
# take the clean data-sort value for author/location.
|
||||
import bs4
|
||||
row = (
|
||||
'<table class="floor-bills-table"><tr class="floor-bill-details">'
|
||||
'<td class="floor_bill-measure">'
|
||||
'<a href="https://leginfo.legislature.ca.gov/faces/'
|
||||
'billNavClient.xhtml?bill_id=202520260AB777">AB 777</a></td>'
|
||||
'<td class="floor_bill-author" data-sort="Rivera">Rivera (with extra spaces)</td>'
|
||||
'<td class="floor_bill-subject">Subject text.</td>'
|
||||
'<td class="floor_bill-location" data-sort="Capitol Room 444">Capitol Room 444 </td>'
|
||||
'<td class="floor_bill-item" data-sort="0000777">777</td></tr></table>'
|
||||
)
|
||||
recs = parsers.parse_html(row)
|
||||
assert len(recs) == 1
|
||||
r = recs[0]
|
||||
assert r.author == "Rivera" # data-sort is clean, not textContent
|
||||
assert r.location == "Capitol Room 444"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- date handling
|
||||
|
||||
class TestDateHandling:
|
||||
def test_normalise_date_ms(self):
|
||||
assert normalise_date("08/31/26-001") == "2026-08-31"
|
||||
|
||||
def test_normalise_date_ms_no_suffix(self):
|
||||
assert normalise_date("08/31/26") == "2026-08-31"
|
||||
|
||||
def test_normalise_date_prefers_mm_slash_dd(self):
|
||||
# NOTE (documented quirk): "2026-08-31" (ISO) returns None -- the
|
||||
# _DATE_SPLIT regex (designed for "08/31/26-001") strips the trailing
|
||||
# "-31" before the ISO format is tried, leaving "2026-08" which fails
|
||||
# all three strptime attempts. This is a known limitation, but the
|
||||
# parser only ever feeds normalise_date a data-sort value (the
|
||||
# site emits "MM/DD/YY" with "-NNN" suffix), so it is safe in practice.
|
||||
# Pin the actual behaviour so a future fix surfaces as a test change.
|
||||
assert normalise_date("2026-08-31") is None
|
||||
assert normalise_date("08/31/26") == "2026-08-31"
|
||||
assert normalise_date("08/31/26-001") == "2026-08-31"
|
||||
|
||||
@pytest.mark.parametrize("junk", ["", "not-a-date", "13/40/99", "2026/13/99"])
|
||||
def test_normalise_date_invalid(self, junk):
|
||||
assert normalise_date(junk) is None
|
||||
|
||||
def test_file_date_backfill_from_hearing(self):
|
||||
recs = parsers.parse_html(F.assembly_daily_file_html())
|
||||
dates = {r.date for r in recs}
|
||||
assert dates == {"2026-08-31"}, f"expected one backfilled date, got {dates}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- robustness
|
||||
|
||||
class TestRobustness:
|
||||
def test_malformed_rows_skipped_not_aborted(self, logcap):
|
||||
cap = logcap("scraper.parsers")
|
||||
recs = parsers.parse_html(F.edge_case_html())
|
||||
bns = {r.bill_number for r in recs}
|
||||
# The good + recoverable rows survive.
|
||||
assert "AB 100" in bns, "good row lost"
|
||||
assert "AB 300" in bns, "recoverable row lost"
|
||||
assert "AB 150" in bns, "display-only recovery failed"
|
||||
# The no-link row must not produce a record.
|
||||
assert "AB 200" not in bns, "a no-link row must not produce a record"
|
||||
# Each skipped row logs a warning (deterministic failure logging).
|
||||
assert len(cap.warnings) >= 2, f"expected >=2 skip warnings, got {cap.warnings}"
|
||||
|
||||
def test_utf16_edge_case_does_not_crash(self, logcap, tmp_path):
|
||||
cap = logcap("scraper.parsers")
|
||||
p = tmp_path / "utf16.html"
|
||||
p.write_bytes(F.edge_case_bytes())
|
||||
recs = parsers.parse(str(p))
|
||||
assert recs == []
|
||||
assert any("charset" in w.lower() or "replacement" in w.lower()
|
||||
for w in cap.warnings), f"expected an encoding warning: {cap.warnings}"
|
||||
|
||||
def test_empty_input_returns_empty(self):
|
||||
assert parsers.parse_html("") == []
|
||||
|
||||
def test_garbage_html_returns_empty_not_crash(self):
|
||||
assert parsers.parse_html("<!!not-html>>><<<") == []
|
||||
|
||||
def test_missing_input_file_is_skipped(self, logcap, tmp_path):
|
||||
cap = logcap("scraper.parsers")
|
||||
miss = str(tmp_path / "does_not_exist.html")
|
||||
recs = parsers.parse(miss)
|
||||
assert recs == []
|
||||
assert any("not found" in w.lower() for w in cap.warnings)
|
||||
|
||||
def test_directory_of_html_files_is_aggregated(self, tmp_path):
|
||||
d = tmp_path / "multi"
|
||||
d.mkdir()
|
||||
for i in range(3):
|
||||
(d / f"day{i}.html").write_text(F.extra_daily_file_html())
|
||||
recs = parsers.parse(str(d))
|
||||
assert len(recs) == 3 * 2, f"expected 6, got {len(recs)}"
|
||||
|
||||
def test_include_hearings_false_drops_hearings(self, tmp_path):
|
||||
p = tmp_path / "asm.html"
|
||||
p.write_text(F.assembly_daily_file_html())
|
||||
recs = parsers.parse(str(p), include_hearings=False)
|
||||
assert all(r.record_type != "hearing" for r in recs)
|
||||
assert any(r.record_type == "bill" for r in recs)
|
||||
|
||||
def test_house_argument_sets_house_field(self):
|
||||
recs = parsers.parse_html(F.senate_daily_file_html(), house="senate")
|
||||
assert recs, "senate fixture parsed nothing"
|
||||
assert all(r.house == "senate" for r in recs)
|
||||
|
||||
def test_invalid_parser_engine_falls_back(self):
|
||||
# An unrecognised parser engine must fall back to html.parser, not raise.
|
||||
recs = parsers.parse_html(F.assembly_daily_file_html(), parser="nosuchparser")
|
||||
assert recs, "bad parser engine produced no records"
|
||||
|
||||
def test_supplement_extraction(self):
|
||||
recs = parsers.parse_html(F.assembly_daily_file_html())
|
||||
ab28 = next(r for r in recs if r.bill_number == "AB 28")
|
||||
assert ab28.supplement == 2
|
||||
|
||||
def test_glob_input_matches_files(self, tmp_path):
|
||||
d = tmp_path / "glob"
|
||||
d.mkdir()
|
||||
(d / "a.html").write_text(F.senate_daily_file_html())
|
||||
(d / "b.html").write_text(F.extra_daily_file_html())
|
||||
recs = parsers.parse(str(d / "*.html"))
|
||||
assert len(recs) == 2 * 2, f"expected 4, got {len(recs)}"
|
||||
|
||||
def test_no_inputs_matched_returns_empty(self, tmp_path, logcap):
|
||||
# A glob that matches nothing -> the "no input files matched" branch.
|
||||
cap = logcap("scraper.parsers")
|
||||
missing_glob = str(tmp_path / "does-not-match-*.html")
|
||||
recs = parsers.parse(missing_glob)
|
||||
assert recs == []
|
||||
assert any("no input files" in w.lower() for w in cap.warnings)
|
||||
|
||||
def test_missing_single_file_is_skipped(self, tmp_path, logcap):
|
||||
# A single path that exists as a name but not on disk -> "not found".
|
||||
cap = logcap("scraper.parsers")
|
||||
missing = str(tmp_path / "no-such-dir.html")
|
||||
recs = parsers.parse(missing)
|
||||
assert recs == []
|
||||
assert any("not found" in w.lower() for w in cap.warnings)
|
||||
@@ -0,0 +1,269 @@
|
||||
"""Tests for the relational correlation lane (spec 2.5, task t_2736f40d).
|
||||
|
||||
Covers the acceptance criteria:
|
||||
(a) migrate is idempotent on a fresh db and on an existing db;
|
||||
(b) a bill with a non-null ``committee`` ends up in ``bill_committees``;
|
||||
(c) a committee member (from a roster) appears in the committee<->member join;
|
||||
(d) the flat ``v_daily_file_records`` view preserves the flat contract.
|
||||
Plus the "create new info / never fabricate" rule: a committee with no roster
|
||||
source is created with an EMPTY member set (a WARN is logged, no members
|
||||
invented).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from scraper import relational
|
||||
from scraper import relate_cli
|
||||
from scraper.schema import BillAction
|
||||
from scraper.io import write_sqlite
|
||||
|
||||
|
||||
def build_flat_db(path):
|
||||
recs = [
|
||||
BillAction(date="2026-08-31", house="assembly",
|
||||
record_type="hearing", committee="Emergency Management",
|
||||
hearing_time="9:30am", location="Room 444",
|
||||
source="ASSEMBLY_DAILY_FILE"),
|
||||
BillAction(date="2026-08-31", house="assembly", record_type="bill",
|
||||
bill_number="AB 28", bill_id="202520260AB28",
|
||||
session="2025-2026", chamber_origin="assembly",
|
||||
author="Macedo et al.", short_title="solid waste",
|
||||
committee="Emergency Management",
|
||||
source="ASSEMBLY_DAILY_FILE"),
|
||||
BillAction(date="2026-08-31", house="assembly", record_type="bill",
|
||||
bill_number="AB 568", bill_id="202520260AB568",
|
||||
session="2025-2026", chamber_origin="assembly",
|
||||
author="Macedo", short_title="water districts",
|
||||
source="ASSEMBLY_DAILY_FILE"),
|
||||
BillAction(date="2026-08-31", house="assembly", record_type="bill",
|
||||
bill_number="SB 133", bill_id="202520260SB133",
|
||||
session="2025-2026", chamber_origin="senate",
|
||||
author="Committee on Budget and Fiscal Review",
|
||||
short_title="budget fix", source="ASSEMBLY_DAILY_FILE"),
|
||||
]
|
||||
write_sqlite(recs, path, table=relational.FLAT_TABLE)
|
||||
return path
|
||||
|
||||
|
||||
def write_roster_file(path):
|
||||
data = [
|
||||
{"committee": "Emergency Management",
|
||||
"session": "2025-2026",
|
||||
"chamber": "assembly",
|
||||
"members": [
|
||||
{"name": "Smith", "role": "Chair"},
|
||||
{"name": "Jones", "role": "Member"},
|
||||
]},
|
||||
]
|
||||
with open(path, "w", encoding="utf-8") as fh:
|
||||
json.dump(data, fh)
|
||||
return path
|
||||
|
||||
|
||||
def _fetch_rows(db, sql):
|
||||
c = relational.connect(db)
|
||||
try:
|
||||
cur = c.execute(sql)
|
||||
return [tuple(r) for r in cur.fetchall()]
|
||||
finally:
|
||||
c.close()
|
||||
|
||||
|
||||
def _count_table(db, table):
|
||||
c = relational.connect(db)
|
||||
try:
|
||||
cur = c.execute("SELECT COUNT(*) FROM " + table)
|
||||
return cur.fetchone()[0]
|
||||
finally:
|
||||
c.close()
|
||||
|
||||
|
||||
def _columns(db, table):
|
||||
c = relational.connect(db)
|
||||
try:
|
||||
cur = c.execute("PRAGMA table_info(" + table + ")")
|
||||
return [row[1] for row in cur.fetchall()]
|
||||
finally:
|
||||
c.close()
|
||||
|
||||
|
||||
def _is_view(db, name):
|
||||
c = relational.connect(db)
|
||||
try:
|
||||
cur = c.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='view' AND name=?",
|
||||
(name,))
|
||||
return cur.fetchone() is not None
|
||||
finally:
|
||||
c.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# (a) idempotency: fresh db, then re-run
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_migrate_fresh_db_is_idempotent(tmp_path):
|
||||
db = str(tmp_path / "rel.sqlite")
|
||||
s1 = relational.migrate(db)
|
||||
assert s1["migrated"] is True
|
||||
for obj in ("bills", "committees", "members", "committee_members",
|
||||
"bill_committees", "schema_migrations", "v_daily_file_records"):
|
||||
assert obj in s1["objects"], obj
|
||||
s2 = relational.migrate(db)
|
||||
assert set(s1["objects"]) == set(s2["objects"])
|
||||
assert s2["flat_rows"] == 0
|
||||
assert s2["version"] == s1["version"]
|
||||
|
||||
|
||||
def test_migrate_on_existing_flat_db_preserves_rows(tmp_path):
|
||||
db = str(tmp_path / "rel_existing.sqlite")
|
||||
build_flat_db(db)
|
||||
assert _count_table(db, relational.FLAT_TABLE) == 4
|
||||
s = relational.migrate(db)
|
||||
assert s["flat_rows"] == 4
|
||||
assert _count_table(db, relational.FLAT_TABLE) == 4
|
||||
for obj in ("bills", "committees", "bill_committees"):
|
||||
assert obj in s["objects"]
|
||||
|
||||
|
||||
def test_correlate_is_idempotent(tmp_path):
|
||||
db = str(tmp_path / "rel_c.sqlite")
|
||||
build_flat_db(db)
|
||||
relational.migrate(db)
|
||||
c1 = relational.correlate(db, quiet=True)
|
||||
c2 = relational.correlate(db, quiet=True)
|
||||
assert c1["bills"] == c2["bills"]
|
||||
assert c1["committees"] == c2["committees"]
|
||||
assert c1["bill_committees"] == c2["bill_committees"]
|
||||
assert c1["members"] == c2["members"]
|
||||
assert c1["committee_members"] == c2["committee_members"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# (b) bill<->committee
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_bill_with_committee_lands_in_bill_committees(tmp_path):
|
||||
db = str(tmp_path / "rel_bc.sqlite")
|
||||
build_flat_db(db)
|
||||
relational.correlate(db, quiet=True)
|
||||
rows = _fetch_rows(db,
|
||||
"SELECT bill_id, committee "
|
||||
"FROM bill_committees WHERE bill_id='202520260AB28'")
|
||||
assert rows, "expected AB 28 to be linked to a committee"
|
||||
assert rows[0][1] == "emergency-management"
|
||||
assert _count_table(db, "committees") >= 1
|
||||
|
||||
|
||||
def test_rebuild_clears_then_backfills(tmp_path):
|
||||
db = str(tmp_path / "rel_rc.sqlite")
|
||||
build_flat_db(db)
|
||||
relational.correlate(db, quiet=True)
|
||||
first = _count_table(db, "bill_committees")
|
||||
c = relational.connect(db)
|
||||
c.execute("INSERT INTO bill_committees (bill_id, committee, session, "
|
||||
"chamber) VALUES ('X999','bogus','','unknown')")
|
||||
c.commit()
|
||||
c.close()
|
||||
assert _count_table(db, "bill_committees") == first + 1
|
||||
relational.correlate(db, rebuild=True, quiet=True)
|
||||
assert _count_table(db, "bill_committees") == first
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# (c) committee<->member + (full 3-way join)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_committee_member_appears_in_join(tmp_path):
|
||||
db = str(tmp_path / "rel_cm.sqlite")
|
||||
build_flat_db(db)
|
||||
roster = write_roster_file(str(tmp_path / "rosters.json"))
|
||||
relational.correlate(db, rosters=roster, quiet=True)
|
||||
rows = _fetch_rows(db,
|
||||
"SELECT member, role FROM committee_members "
|
||||
"WHERE committee='emergency-management'")
|
||||
names = {r[0] for r in rows}
|
||||
assert names == {"smith", "jones"}, names
|
||||
joined = _fetch_rows(db,
|
||||
"SELECT bc.bill_id, c.name_raw, m.full_name, cm.role "
|
||||
"FROM bill_committees bc "
|
||||
"JOIN committees c "
|
||||
"ON c.name_key=bc.committee AND c.chamber=bc.chamber "
|
||||
"AND c.session=bc.session "
|
||||
"JOIN committee_members cm "
|
||||
"ON cm.committee=c.name_key AND cm.chamber=c.chamber "
|
||||
"AND cm.session=c.session "
|
||||
"JOIN members m ON m.name_norm=cm.member")
|
||||
got = {(r[0], r[2]) for r in joined}
|
||||
assert ("202520260AB28", "Smith") in got
|
||||
assert ("202520260AB28", "Jones") in got
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# never-fabricate rule
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_committee_without_roster_has_no_members(tmp_path, logcap):
|
||||
db = str(tmp_path / "rel_nf.sqlite")
|
||||
build_flat_db(db)
|
||||
cap = logcap("scraper.relate")
|
||||
roster = write_roster_file(str(tmp_path / "rosters.json"))
|
||||
counts = relational.correlate(db, rosters=roster, quiet=True)
|
||||
cap.remove()
|
||||
members_for = _fetch_rows(db,
|
||||
"SELECT COUNT(*) FROM committee_members cm "
|
||||
"JOIN committees c ON c.name_key=cm.committee "
|
||||
"WHERE c.name_raw='Budget and Fiscal Review'")
|
||||
assert members_for[0][0] == 0, "members must NOT be fabricated"
|
||||
assert counts["committees_missing_roster"] >= 1
|
||||
joined_cap = " ".join(cap.warnings).lower()
|
||||
assert ("no roster" in joined_cap) or ("fabricat" in joined_cap), \
|
||||
cap.warnings
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# (d) flat-view contract preserved
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_flat_view_matches_table(tmp_path):
|
||||
db = str(tmp_path / "rel_view.sqlite")
|
||||
build_flat_db(db)
|
||||
relational.migrate(db)
|
||||
t_cols = _columns(db, relational.FLAT_TABLE)
|
||||
v_cols = _columns(db, "v_daily_file_records")
|
||||
assert t_cols == v_cols, (t_cols, v_cols)
|
||||
t_rows = _fetch_rows(db,
|
||||
"SELECT * FROM " + relational.FLAT_TABLE + " ORDER BY 1")
|
||||
v_rows = _fetch_rows(db, "SELECT * FROM v_daily_file_records ORDER BY 1")
|
||||
assert t_rows == v_rows, "view row-set must equal the flat table"
|
||||
assert _is_view(db, "v_daily_file_records")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI / wiring: additive registration must not clobber siblings
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_cli_registers_migrate_and_correlate_without_clobbering():
|
||||
from scraper import cli
|
||||
parser = cli.build_parser()
|
||||
a = parser.parse_args(["migrate", "--db", "x.sqlite"])
|
||||
assert a.command == "migrate"
|
||||
b = parser.parse_args(["correlate", "--db", "x.sqlite", "--roster", "r.json"])
|
||||
assert b.command == "correlate"
|
||||
assert parser.parse_args(["parse", "--input", "f.html"]).command == "parse"
|
||||
assert parser.parse_args(["ingest", "--date", "2026-09-01"]).command == "ingest"
|
||||
assert parser.parse_args(["list"]).command == "list"
|
||||
|
||||
|
||||
def test_migrate_then_correlate_cli_end_to_end(tmp_path):
|
||||
import argparse
|
||||
db = str(tmp_path / "rel_cli.sqlite")
|
||||
build_flat_db(db)
|
||||
roster = write_roster_file(str(tmp_path / "rosters.json"))
|
||||
ns = argparse.Namespace(db=db, check=False)
|
||||
assert relate_cli.handle_migrate(ns) == 0
|
||||
ns2 = argparse.Namespace(db=db, roster=roster, rebuild=False)
|
||||
assert relate_cli.handle_correlate(ns2) == 0
|
||||
assert _count_table(db, "committee_members") == 2
|
||||
@@ -0,0 +1,178 @@
|
||||
"""Unit tests for scraper.schema (BillAction) and its JSONL contract.
|
||||
|
||||
The normalised record schema is spec section 7. This file pins the field set,
|
||||
the content-hash id, the null-stripping serialisation, and the round-trip
|
||||
through JSON Lines -- the output contract the downstream RDB / UI lanes depend
|
||||
on, so it must be stable and asserted explicitly.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from conftest import read_jsonl
|
||||
from scraper.schema import (
|
||||
ASSEMBLY_DAILY_FILE,
|
||||
BillAction,
|
||||
HOUSE,
|
||||
SENATE_DAILY_FILE_PDF,
|
||||
)
|
||||
|
||||
|
||||
# A realistic bill record, mirroring the real AB 28 output.
|
||||
def sample_bill() -> BillAction:
|
||||
return BillAction(
|
||||
date="2026-08-31",
|
||||
house="assembly",
|
||||
source=ASSEMBLY_DAILY_FILE,
|
||||
record_type="bill",
|
||||
bill_number="AB 28",
|
||||
bill_id="202520260AB28",
|
||||
session="2025-2026",
|
||||
chamber_origin="ASSEMBLY",
|
||||
author="Schiavo et al.",
|
||||
short_title="An act relating to solid waste.",
|
||||
description="An act relating to solid waste.",
|
||||
location="Unfinished Business",
|
||||
item_number=32,
|
||||
supplement=2,
|
||||
)
|
||||
|
||||
|
||||
class TestFieldSet:
|
||||
def test_field_names_match_schema(self):
|
||||
names = set(BillAction.field_names())
|
||||
# The core fields every downstream consumer expects (spec section 7).
|
||||
for expected in [
|
||||
"id", "date", "house", "source", "record_type",
|
||||
"bill_number", "bill_id", "session", "chamber_origin",
|
||||
"author", "short_title", "description", "committee",
|
||||
"hearing_time", "location", "item_number", "supplement",
|
||||
"ics_url", "agenda_id", "raw_selector",
|
||||
]:
|
||||
assert expected in names, f"missing field {expected!r}"
|
||||
|
||||
def test_default_house_is_assembly(self):
|
||||
assert HOUSE == "assembly"
|
||||
assert BillAction().house == "assembly"
|
||||
|
||||
def test_module_source_constants(self):
|
||||
assert {ASSEMBLY_DAILY_FILE, SENATE_DAILY_FILE_PDF} <= {
|
||||
ASSEMBLY_DAILY_FILE, SENATE_DAILY_FILE_PDF, "LEGINFO_BILL_RESULTS",
|
||||
}
|
||||
|
||||
|
||||
class TestWithId:
|
||||
def test_id_is_deterministic_content_hash(self):
|
||||
a = sample_bill()
|
||||
b = sample_bill()
|
||||
a.with_id()
|
||||
b.with_id()
|
||||
assert a.id == b.id
|
||||
assert a.id == hashlib.sha1(
|
||||
"|".join([a.house, a.bill_number, a.date, a.committee or ""])
|
||||
.encode("utf-8")).hexdigest()
|
||||
|
||||
def test_missing_components_render_empty_in_hash(self):
|
||||
rec = sample_bill()
|
||||
rec.committee = None
|
||||
rec.date = None
|
||||
rec.with_id()
|
||||
expected = hashlib.sha1(
|
||||
"|".join([rec.house or "", rec.bill_number or "",
|
||||
rec.date or "", rec.committee or ""]
|
||||
).encode("utf-8")).hexdigest()
|
||||
assert rec.id == expected
|
||||
|
||||
def test_explicit_content_used_when_supplied(self):
|
||||
rec = BillAction()
|
||||
rec.with_id("explicit-key")
|
||||
assert rec.id == hashlib.sha1("explicit-key".encode()).hexdigest()
|
||||
|
||||
def test_with_id_returns_self_for_chaining(self):
|
||||
rec = sample_bill()
|
||||
assert rec.with_id() is rec
|
||||
|
||||
|
||||
class TestToDictToJson:
|
||||
def test_to_dict_strips_nulls_by_default(self):
|
||||
rec = sample_bill().with_id()
|
||||
data = rec.to_dict()
|
||||
assert None not in data.values()
|
||||
assert data["bill_number"] == "AB 28"
|
||||
assert data["item_number"] == 32
|
||||
|
||||
def test_to_dict_include_nulls(self):
|
||||
rec = sample_bill().with_id()
|
||||
data = rec.to_dict(include_nulls=True)
|
||||
# A field that is None in this record must appear when requested.
|
||||
assert "committee" in data
|
||||
assert data["committee"] is None
|
||||
|
||||
def test_to_json_is_single_line(self):
|
||||
rec = sample_bill()
|
||||
line = rec.to_json()
|
||||
assert "\n" not in line
|
||||
# Re-parse and confirm it is a well-formed object.
|
||||
parsed = json.loads(line)
|
||||
assert parsed["bill_number"] == "AB 28"
|
||||
|
||||
def test_to_json_sorts_keys(self):
|
||||
rec = sample_bill()
|
||||
line = rec.to_json()
|
||||
parsed = json.loads(line)
|
||||
assert list(parsed.keys()) == sorted(parsed.keys())
|
||||
|
||||
def test_to_json_fills_id_when_absent(self):
|
||||
rec = sample_bill() # id is None
|
||||
parsed = json.loads(rec.to_json())
|
||||
assert "id" in parsed
|
||||
assert parsed["id"]
|
||||
|
||||
def test_to_json_omits_nulls(self):
|
||||
rec = BillAction(record_type="bill", bill_number="AB 1")
|
||||
parsed = json.loads(rec.to_json())
|
||||
# null-only fields must not appear
|
||||
assert "committee" not in parsed
|
||||
assert "hearing_time" not in parsed
|
||||
|
||||
|
||||
class TestFromDict:
|
||||
def test_from_dict_ignores_unknown_keys(self):
|
||||
data = {"bill_number": "AB 1", "surprise": "extra", "date": "2026-01-01"}
|
||||
rec = BillAction.from_dict(data)
|
||||
assert rec.bill_number == "AB 1"
|
||||
assert rec.date == "2026-01-01"
|
||||
assert not hasattr(rec, "surprise")
|
||||
|
||||
def test_from_dict_empty(self):
|
||||
rec = BillAction.from_dict({})
|
||||
assert rec.record_type == "bill" # default
|
||||
assert rec.house == "assembly" # default
|
||||
|
||||
|
||||
class TestJsonlRoundTrip:
|
||||
def test_write_read_roundtrip(self, tmp_path):
|
||||
import scraper.io as io
|
||||
recs = [sample_bill(), BillAction(record_type="hearing",
|
||||
committee="Budget",
|
||||
hearing_time="9:30am")]
|
||||
out = tmp_path / "out.jsonl"
|
||||
written = io.write_jsonl(recs, str(out))
|
||||
assert written == 2
|
||||
rows = read_jsonl(str(out))
|
||||
assert len(rows) == 2
|
||||
assert rows[0]["bill_number"] == "AB 28"
|
||||
assert rows[1]["record_type"] == "hearing"
|
||||
|
||||
def test_jsonl_lines_are_valid_json(self, tmp_path):
|
||||
import scraper.io as io
|
||||
recs = [sample_bill()] * 5
|
||||
out = tmp_path / "out.jsonl"
|
||||
io.write_jsonl(recs, str(out))
|
||||
with open(str(out), encoding="utf-8") as fh:
|
||||
for line in fh:
|
||||
assert line.strip()
|
||||
json.loads(line) # raises if any line is malformed
|
||||
Reference in new issue
Block a user