Initial commit

tests/test_llm.py

@@ -0,0 +1,305 @@
+"""Tests for the Ollama / LLM analysis lane (spec 4, task t_950780e0).
+
+Covers the acceptance criteria:
+
+     (a) fallback path with ``enabled=false`` -> ZERO network (built-in tagger);
+     (b) a mocked Ollama response is parsed + stored (generated_by='ollama:m');
+     (c) endpoint unreachable -> fallback, no crash;
+plus the tag-cloud aggregation + additive-table contract and the CLI wiring
+(analyze registers without clobbering sibling handlers).
+
+The ollama path is never touched against the real network: a canned response is
+fed through the injectable ``post`` callable, and the fallback path is proven to
+need no network by asserting ``urllib.request.urlopen`` is never called.
+"""
+
+from __future__ import annotations
+
+import json
+
+import urllib.error
+import urllib.request
+
+from scraper import llm
+from scraper import relational
+from scraper import analyze_cli
+from scraper.schema import BillAction
+from scraper.io import write_sqlite
+
+
+# ---------------------------------------------------------------- helpers ---
+
+def build_flat_db(path):
+    recs = [
+        BillAction(date="2026-08-31", house="assembly",
+                   record_type="hearing", committee="Emergency Management",
+                   hearing_time="9:30am", location="Room 444",
+                   source="ASSEMBLY_DAILY_FILE"),
+        BillAction(date="2026-08-31", house="assembly", record_type="bill",
+                   bill_number="AB 28", bill_id="202520260AB28",
+                   session="2025-2026", chamber_origin="assembly",
+                   author="Macedo et al.",
+                   short_title=("An act relating to solid waste, and making "
+                                 "an appropriation therefor."),
+                   committee="Emergency Management",
+                   source="ASSEMBLY_DAILY_FILE"),
+        BillAction(date="2026-08-31", house="assembly", record_type="bill",
+                   bill_number="AB 568", bill_id="202520260AB568",
+                   session="2025-2026", chamber_origin="assembly",
+                   author="Macedo", short_title="An act relating to water "
+                    "districts.", source="ASSEMBLY_DAILY_FILE"),
+        BillAction(date="2026-08-31", house="assembly", record_type="bill",
+                   bill_number="AB 7", bill_id="202520260AB7",
+                   session="2025-2026", chamber_origin="assembly",
+                   author="Ortega",
+                   short_title="An act relating to housing.",
+                   source="ASSEMBLY_DAILY_FILE"),
+      ]
+    write_sqlite(recs, path, table=relational.FLAT_TABLE)
+    return path
+
+
+def _fake_ollama_json(tags):
+    return {"choices": [{"message": {"content": json.dumps(tags)}}]}
+
+
+def _fake_post(tags=None, raises=False):
+    calls = []
+
+    def post(url, payload, timeout):
+        calls.append(url)
+        if raises:
+            raise urllib.error.URLError("connection refused")
+        return _fake_ollama_json(tags or [])
+
+    post.calls = calls   # type: ignore[attr-defined]
+    return post
+
+
+# ---------------------------------------------------------------------------
+# (a) fallback path, enabled=false -> ZERO network
+# ---------------------------------------------------------------------------
+
+def test_fallback_when_disabled_is_zero_network(tmp_path, monkeypatch):
+    db = str(tmp_path / "ll_a.sqlite")
+    build_flat_db(db)
+
+    def _deny(*a, **k):
+        raise AssertionError("fallback path must not touch the network")
+
+    monkeypatch.setattr(urllib.request, "urlopen", _deny)
+
+    results = llm.tag_bills(llm.load_bills(db),
+                            cfg={"enabled": False, "model": "llama3",
+                                "endpoint": "http://localhost:11434"})
+    assert len(results) == 3, "3 distinct bills expected"
+    assert all(r["generated_by"] == llm.FALLBACK for r in results)
+    assert all(r["tags"] for r in results), "every bill must get >=1 tag"
+
+    ab28 = next(r for r in results if r["bill_id"] == "202520260AB28")
+    tags = [t for (t, _s) in ab28["tags"]]
+    assert "solid waste" in tags, tags
+    assert "solid waste & recycling" in tags or "budget" in tags, tags
+
+
+def test_tag_bill_fallback_never_invents():
+    bill = {"bill_id": "X", "short_title": "An act relating to zzzzz."}
+    tags = [t for (t, _s) in llm.tag_bill_fallback(bill)]
+    assert tags, "must never return an empty tag list"
+
+    bill2 = {"bill_id": "Y", "short_title": "An act relating to water districts."}
+    tags2 = [t for (t, _s) in llm.tag_bill_fallback(bill2)]
+    assert "water" in tags2, tags2
+
+
+# ---------------------------------------------------------------------------
+# (b) mocked ollama response parsed + stored
+# ---------------------------------------------------------------------------
+
+def test_mocked_ollama_parsed_and_stored(tmp_path):
+    db = str(tmp_path / "ll_b.sqlite")
+    build_flat_db(db)
+
+    post = _fake_post(tags=["housing", "residential rental"])
+    results = llm.tag_bills(llm.load_bills(db),
+                            cfg={"enabled": True, "model": "llama3",
+                                "endpoint": "http://localhost:11434",
+                                "default_tags": 8},
+                            post=post)
+    assert results, "expected results"
+    assert all(r["generated_by"] == "ollama:llama3" for r in results), \
+        [r["generated_by"] for r in results]
+    ab568 = next(r for r in results if r["bill_id"] == "202520260AB568")
+    tags = {t for (t, _s) in ab568["tags"]}
+    assert "housing" in tags and "residential rental" in tags, tags
+    assert all(s == "ollama" for (_t, s) in ab568["tags"]), ab568["tags"]
+
+    written = llm.persist_tags(db, results)
+    rows = _fetch(db, "SELECT bill_id, tag, generated_by FROM bill_tags "
+                        "ORDER BY bill_id, tag")
+    assert written > 0
+    assert any(gb == "ollama:llama3" for (_b, _t, gb) in rows)
+    assert len(rows) >= 2 * 3, "3 bills x >=2 tags each"
+
+
+def test_ollama_parse_json_and_fenced():
+      # JSON array inside a code fence must be parsed (non-filler tags).
+    assert llm._parse_tags_for_bill('```json\n["housing","bills"]\n```') == \
+            ["housing", "bills"]
+      # Bare JSON array.
+    assert llm._parse_tags_for_bill('[ "housing" , "foreclosure" ]') == \
+            ["housing", "foreclosure"]
+    out = llm._parse_tags_for_bill("housing, foreclosure, education")
+    assert "housing" in out and "foreclosure" in out, out
+
+
+# ---------------------------------------------------------------------------
+# (c) endpoint unreachable -> fallback, no crash
+# ---------------------------------------------------------------------------
+
+def test_unreachable_endpoint_falls_back_no_crash(tmp_path):
+    db = str(tmp_path / "ll_c.sqlite")
+    build_flat_db(db)
+
+    post = _fake_post(raises=True)
+    results = llm.tag_bills(llm.load_bills(db),
+                            cfg={"enabled": True, "model": "llama3",
+                                "endpoint": "http://localhost:11434",
+                                "timeout_seconds": 1},
+                            post=post)
+    assert results, "must return results, not raise"
+    assert all(r["generated_by"] == llm.FALLBACK for r in results)
+    assert all(r["tags"] for r in results), "fallback must tag every bill"
+    assert all("unreachable" in r["reason"] for r in results), \
+        [r["reason"] for r in results]
+
+
+def test_mixed_run_reports_fallback_path(tmp_path):
+    db = str(tmp_path / "ll_m.sqlite")
+    build_flat_db(db)
+
+    def flaky(url, payload, timeout):
+        return _fake_ollama_json([])
+
+    results = llm.tag_bills(llm.load_bills(db),
+                            cfg={"enabled": True, "model": "llama3",
+                                "endpoint": "http://localhost:11434"},
+                            post=flaky)
+    dom = llm._dominant_generated_by(results)
+    assert dom == llm.FALLBACK, "empty ollama answer -> fallback path reported"
+
+
+# ---------------------------------------------------------------------------
+# tag-cloud aggregation + additive table contract
+# ---------------------------------------------------------------------------
+
+def test_build_tag_cloud_aggregates(tmp_path):
+    db = str(tmp_path / "ll_cloud.sqlite")
+    build_flat_db(db)
+    llm.analyze(db_path=db, out_path=str(tmp_path / "cloud.json"),
+                cfg={"enabled": False}, rebuild=True)
+    cloud = llm.build_tag_cloud(db_path=db)
+    assert cloud["bill_count"] == 3, cloud
+    assert cloud["tag_count"] > 0
+    counts = [t["count"] for t in cloud["tags"]]
+    assert counts == sorted(counts, reverse=True), counts
+    with open(str(tmp_path / "cloud.json"), encoding="utf-8") as fh:
+        loaded = json.load(fh)
+    assert loaded["bill_count"] == 3
+    assert set(loaded) >= {"tag_count", "bill_count", "tags"}
+
+
+def test_bill_tags_table_is_additive_and_idempotent(tmp_path):
+    db = str(tmp_path / "ll_idem.sqlite")
+    build_flat_db(db)
+    s1 = llm.analyze(db_path=db, cfg={"enabled": False}, rebuild=True)
+    s2 = llm.analyze(db_path=db, cfg={"enabled": False}, rebuild=False)
+    assert s1["rows_written"] > 0
+    assert s2["rows_written"] == 0, "second run must add no rows (idempotent)"
+    assert _count(db, relational.FLAT_TABLE) == 4
+
+
+def test_ensure_tag_table_does_not_touch_flat(tmp_path):
+    db = str(tmp_path / "ll_flat.sqlite")
+    build_flat_db(db)
+    before = _count(db, relational.FLAT_TABLE)
+    llm.ensure_tag_table(db)
+    assert _count(db, relational.FLAT_TABLE) == before == 4
+    assert llm.TAG_TABLE == "bill_tags"
+    cols = _cols(db, relational.FLAT_TABLE)
+    assert "tag" not in cols and "generated_by" not in cols
+
+
+# ---------------------------------------------------------------------------
+# CLI wiring: additive + guarded
+# ---------------------------------------------------------------------------
+
+def test_analyze_registers_without_clobbering():
+    from scraper import cli
+    parser = cli.build_parser()
+    a = parser.parse_args(["analyze", "--top", "5"])
+    assert a.command == "analyze"
+    assert a.top == 5
+    assert parser.parse_args(["parse", "--input", "f.html"]).command == "parse"
+    assert parser.parse_args(["migrate"]).command == "migrate"
+    assert parser.parse_args(
+        ["ingest", "--date", "2026-09-01"]).command == "ingest"
+
+
+def test_handle_analyze_end_to_end(tmp_path):
+    import argparse
+    from conftest import CONFIG_PATH
+    db = str(tmp_path / "ll_cli.sqlite")
+    build_flat_db(db)
+    out = str(tmp_path / "cloud.json")
+    ns = argparse.Namespace(db=db, out_path=out, house=None, top=5,
+                           rebuild=True, force_llm=False, list_config=False,
+                           config=CONFIG_PATH)
+    rc = analyze_cli.handle_analyze(ns)
+    assert rc == 0
+    with open(out, encoding="utf-8") as fh:
+        cloud = json.load(fh)
+    assert cloud["bill_count"] == 3
+    assert cloud["tag_count"] > 0
+
+
+# ---------------------------------------------------------------------------
+# config wiring
+# ---------------------------------------------------------------------------
+
+def test_config_llm_section_is_additive():
+    from scraper.config import load_config
+    from conftest import CONFIG_PATH
+    cfg = load_config(CONFIG_PATH)
+    llm_cfg = cfg.llm
+    assert llm_cfg["enabled"] is False
+    assert llm_cfg["backend"] == "ollama"
+    assert "http://localhost:11434" in llm_cfg["endpoint"]
+    assert "http" in cfg.data and "sources" in cfg.data
+    assert llm_cfg["default_tags"] == 8
+    assert llm_cfg["timeout_seconds"] == 30
+
+
+# ---------------------------------------------------------------------------
+# test helpers
+# ---------------------------------------------------------------------------
+
+def _fetch(db, sql):
+    c = relational.connect(db, foreign_keys=False)
+    try:
+        return [tuple(r) for r in c.execute(sql).fetchall()]
+    finally:
+        c.close()
+
+
+def _count(db, table):
+    return _fetch(db, "SELECT COUNT(*) FROM " + table)[0][0]
+
+
+def _cols(db, table):
+    c = relational.connect(db, foreign_keys=False)
+    try:
+        cur = c.execute("PRAGMA table_info(" + table + ")")
+        return [row[1] for row in cur.fetchall()]
+    finally:
+        c.close()