306 lines
12 KiB
Python
306 lines
12 KiB
Python
"""Tests for the Ollama / LLM analysis lane (spec 4, task t_950780e0).
|
|
|
|
Covers the acceptance criteria:
|
|
|
|
(a) fallback path with ``enabled=false`` -> ZERO network (built-in tagger);
|
|
(b) a mocked Ollama response is parsed + stored (generated_by='ollama:m');
|
|
(c) endpoint unreachable -> fallback, no crash;
|
|
plus the tag-cloud aggregation + additive-table contract and the CLI wiring
|
|
(analyze registers without clobbering sibling handlers).
|
|
|
|
The ollama path is never touched against the real network: a canned response is
|
|
fed through the injectable ``post`` callable, and the fallback path is proven to
|
|
need no network by asserting ``urllib.request.urlopen`` is never called.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
from scraper import llm
|
|
from scraper import relational
|
|
from scraper import analyze_cli
|
|
from scraper.schema import BillAction
|
|
from scraper.io import write_sqlite
|
|
|
|
|
|
# ---------------------------------------------------------------- helpers ---
|
|
|
|
def build_flat_db(path):
|
|
recs = [
|
|
BillAction(date="2026-08-31", house="assembly",
|
|
record_type="hearing", committee="Emergency Management",
|
|
hearing_time="9:30am", location="Room 444",
|
|
source="ASSEMBLY_DAILY_FILE"),
|
|
BillAction(date="2026-08-31", house="assembly", record_type="bill",
|
|
bill_number="AB 28", bill_id="202520260AB28",
|
|
session="2025-2026", chamber_origin="assembly",
|
|
author="Macedo et al.",
|
|
short_title=("An act relating to solid waste, and making "
|
|
"an appropriation therefor."),
|
|
committee="Emergency Management",
|
|
source="ASSEMBLY_DAILY_FILE"),
|
|
BillAction(date="2026-08-31", house="assembly", record_type="bill",
|
|
bill_number="AB 568", bill_id="202520260AB568",
|
|
session="2025-2026", chamber_origin="assembly",
|
|
author="Macedo", short_title="An act relating to water "
|
|
"districts.", source="ASSEMBLY_DAILY_FILE"),
|
|
BillAction(date="2026-08-31", house="assembly", record_type="bill",
|
|
bill_number="AB 7", bill_id="202520260AB7",
|
|
session="2025-2026", chamber_origin="assembly",
|
|
author="Ortega",
|
|
short_title="An act relating to housing.",
|
|
source="ASSEMBLY_DAILY_FILE"),
|
|
]
|
|
write_sqlite(recs, path, table=relational.FLAT_TABLE)
|
|
return path
|
|
|
|
|
|
def _fake_ollama_json(tags):
|
|
return {"choices": [{"message": {"content": json.dumps(tags)}}]}
|
|
|
|
|
|
def _fake_post(tags=None, raises=False):
|
|
calls = []
|
|
|
|
def post(url, payload, timeout):
|
|
calls.append(url)
|
|
if raises:
|
|
raise urllib.error.URLError("connection refused")
|
|
return _fake_ollama_json(tags or [])
|
|
|
|
post.calls = calls # type: ignore[attr-defined]
|
|
return post
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# (a) fallback path, enabled=false -> ZERO network
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_fallback_when_disabled_is_zero_network(tmp_path, monkeypatch):
|
|
db = str(tmp_path / "ll_a.sqlite")
|
|
build_flat_db(db)
|
|
|
|
def _deny(*a, **k):
|
|
raise AssertionError("fallback path must not touch the network")
|
|
|
|
monkeypatch.setattr(urllib.request, "urlopen", _deny)
|
|
|
|
results = llm.tag_bills(llm.load_bills(db),
|
|
cfg={"enabled": False, "model": "llama3",
|
|
"endpoint": "http://localhost:11434"})
|
|
assert len(results) == 3, "3 distinct bills expected"
|
|
assert all(r["generated_by"] == llm.FALLBACK for r in results)
|
|
assert all(r["tags"] for r in results), "every bill must get >=1 tag"
|
|
|
|
ab28 = next(r for r in results if r["bill_id"] == "202520260AB28")
|
|
tags = [t for (t, _s) in ab28["tags"]]
|
|
assert "solid waste" in tags, tags
|
|
assert "solid waste & recycling" in tags or "budget" in tags, tags
|
|
|
|
|
|
def test_tag_bill_fallback_never_invents():
|
|
bill = {"bill_id": "X", "short_title": "An act relating to zzzzz."}
|
|
tags = [t for (t, _s) in llm.tag_bill_fallback(bill)]
|
|
assert tags, "must never return an empty tag list"
|
|
|
|
bill2 = {"bill_id": "Y", "short_title": "An act relating to water districts."}
|
|
tags2 = [t for (t, _s) in llm.tag_bill_fallback(bill2)]
|
|
assert "water" in tags2, tags2
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# (b) mocked ollama response parsed + stored
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_mocked_ollama_parsed_and_stored(tmp_path):
|
|
db = str(tmp_path / "ll_b.sqlite")
|
|
build_flat_db(db)
|
|
|
|
post = _fake_post(tags=["housing", "residential rental"])
|
|
results = llm.tag_bills(llm.load_bills(db),
|
|
cfg={"enabled": True, "model": "llama3",
|
|
"endpoint": "http://localhost:11434",
|
|
"default_tags": 8},
|
|
post=post)
|
|
assert results, "expected results"
|
|
assert all(r["generated_by"] == "ollama:llama3" for r in results), \
|
|
[r["generated_by"] for r in results]
|
|
ab568 = next(r for r in results if r["bill_id"] == "202520260AB568")
|
|
tags = {t for (t, _s) in ab568["tags"]}
|
|
assert "housing" in tags and "residential rental" in tags, tags
|
|
assert all(s == "ollama" for (_t, s) in ab568["tags"]), ab568["tags"]
|
|
|
|
written = llm.persist_tags(db, results)
|
|
rows = _fetch(db, "SELECT bill_id, tag, generated_by FROM bill_tags "
|
|
"ORDER BY bill_id, tag")
|
|
assert written > 0
|
|
assert any(gb == "ollama:llama3" for (_b, _t, gb) in rows)
|
|
assert len(rows) >= 2 * 3, "3 bills x >=2 tags each"
|
|
|
|
|
|
def test_ollama_parse_json_and_fenced():
|
|
# JSON array inside a code fence must be parsed (non-filler tags).
|
|
assert llm._parse_tags_for_bill('```json\n["housing","bills"]\n```') == \
|
|
["housing", "bills"]
|
|
# Bare JSON array.
|
|
assert llm._parse_tags_for_bill('[ "housing" , "foreclosure" ]') == \
|
|
["housing", "foreclosure"]
|
|
out = llm._parse_tags_for_bill("housing, foreclosure, education")
|
|
assert "housing" in out and "foreclosure" in out, out
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# (c) endpoint unreachable -> fallback, no crash
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_unreachable_endpoint_falls_back_no_crash(tmp_path):
|
|
db = str(tmp_path / "ll_c.sqlite")
|
|
build_flat_db(db)
|
|
|
|
post = _fake_post(raises=True)
|
|
results = llm.tag_bills(llm.load_bills(db),
|
|
cfg={"enabled": True, "model": "llama3",
|
|
"endpoint": "http://localhost:11434",
|
|
"timeout_seconds": 1},
|
|
post=post)
|
|
assert results, "must return results, not raise"
|
|
assert all(r["generated_by"] == llm.FALLBACK for r in results)
|
|
assert all(r["tags"] for r in results), "fallback must tag every bill"
|
|
assert all("unreachable" in r["reason"] for r in results), \
|
|
[r["reason"] for r in results]
|
|
|
|
|
|
def test_mixed_run_reports_fallback_path(tmp_path):
|
|
db = str(tmp_path / "ll_m.sqlite")
|
|
build_flat_db(db)
|
|
|
|
def flaky(url, payload, timeout):
|
|
return _fake_ollama_json([])
|
|
|
|
results = llm.tag_bills(llm.load_bills(db),
|
|
cfg={"enabled": True, "model": "llama3",
|
|
"endpoint": "http://localhost:11434"},
|
|
post=flaky)
|
|
dom = llm._dominant_generated_by(results)
|
|
assert dom == llm.FALLBACK, "empty ollama answer -> fallback path reported"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# tag-cloud aggregation + additive table contract
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_build_tag_cloud_aggregates(tmp_path):
|
|
db = str(tmp_path / "ll_cloud.sqlite")
|
|
build_flat_db(db)
|
|
llm.analyze(db_path=db, out_path=str(tmp_path / "cloud.json"),
|
|
cfg={"enabled": False}, rebuild=True)
|
|
cloud = llm.build_tag_cloud(db_path=db)
|
|
assert cloud["bill_count"] == 3, cloud
|
|
assert cloud["tag_count"] > 0
|
|
counts = [t["count"] for t in cloud["tags"]]
|
|
assert counts == sorted(counts, reverse=True), counts
|
|
with open(str(tmp_path / "cloud.json"), encoding="utf-8") as fh:
|
|
loaded = json.load(fh)
|
|
assert loaded["bill_count"] == 3
|
|
assert set(loaded) >= {"tag_count", "bill_count", "tags"}
|
|
|
|
|
|
def test_bill_tags_table_is_additive_and_idempotent(tmp_path):
|
|
db = str(tmp_path / "ll_idem.sqlite")
|
|
build_flat_db(db)
|
|
s1 = llm.analyze(db_path=db, cfg={"enabled": False}, rebuild=True)
|
|
s2 = llm.analyze(db_path=db, cfg={"enabled": False}, rebuild=False)
|
|
assert s1["rows_written"] > 0
|
|
assert s2["rows_written"] == 0, "second run must add no rows (idempotent)"
|
|
assert _count(db, relational.FLAT_TABLE) == 4
|
|
|
|
|
|
def test_ensure_tag_table_does_not_touch_flat(tmp_path):
|
|
db = str(tmp_path / "ll_flat.sqlite")
|
|
build_flat_db(db)
|
|
before = _count(db, relational.FLAT_TABLE)
|
|
llm.ensure_tag_table(db)
|
|
assert _count(db, relational.FLAT_TABLE) == before == 4
|
|
assert llm.TAG_TABLE == "bill_tags"
|
|
cols = _cols(db, relational.FLAT_TABLE)
|
|
assert "tag" not in cols and "generated_by" not in cols
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CLI wiring: additive + guarded
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_analyze_registers_without_clobbering():
|
|
from scraper import cli
|
|
parser = cli.build_parser()
|
|
a = parser.parse_args(["analyze", "--top", "5"])
|
|
assert a.command == "analyze"
|
|
assert a.top == 5
|
|
assert parser.parse_args(["parse", "--input", "f.html"]).command == "parse"
|
|
assert parser.parse_args(["migrate"]).command == "migrate"
|
|
assert parser.parse_args(
|
|
["ingest", "--date", "2026-09-01"]).command == "ingest"
|
|
|
|
|
|
def test_handle_analyze_end_to_end(tmp_path):
|
|
import argparse
|
|
from conftest import CONFIG_PATH
|
|
db = str(tmp_path / "ll_cli.sqlite")
|
|
build_flat_db(db)
|
|
out = str(tmp_path / "cloud.json")
|
|
ns = argparse.Namespace(db=db, out_path=out, house=None, top=5,
|
|
rebuild=True, force_llm=False, list_config=False,
|
|
config=CONFIG_PATH)
|
|
rc = analyze_cli.handle_analyze(ns)
|
|
assert rc == 0
|
|
with open(out, encoding="utf-8") as fh:
|
|
cloud = json.load(fh)
|
|
assert cloud["bill_count"] == 3
|
|
assert cloud["tag_count"] > 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# config wiring
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_config_llm_section_is_additive():
|
|
from scraper.config import load_config
|
|
from conftest import CONFIG_PATH
|
|
cfg = load_config(CONFIG_PATH)
|
|
llm_cfg = cfg.llm
|
|
assert llm_cfg["enabled"] is False
|
|
assert llm_cfg["backend"] == "ollama"
|
|
assert "http://localhost:11434" in llm_cfg["endpoint"]
|
|
assert "http" in cfg.data and "sources" in cfg.data
|
|
assert llm_cfg["default_tags"] == 8
|
|
assert llm_cfg["timeout_seconds"] == 30
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# test helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _fetch(db, sql):
|
|
c = relational.connect(db, foreign_keys=False)
|
|
try:
|
|
return [tuple(r) for r in c.execute(sql).fetchall()]
|
|
finally:
|
|
c.close()
|
|
|
|
|
|
def _count(db, table):
|
|
return _fetch(db, "SELECT COUNT(*) FROM " + table)[0][0]
|
|
|
|
|
|
def _cols(db, table):
|
|
c = relational.connect(db, foreign_keys=False)
|
|
try:
|
|
cur = c.execute("PRAGMA table_info(" + table + ")")
|
|
return [row[1] for row in cur.fetchall()]
|
|
finally:
|
|
c.close()
|