133 lines
5.4 KiB
Python
133 lines
5.4 KiB
Python
"""``scraper.analyze`` — spec §4 CLI subcommand for Ollama / fallback bill analysis.
|
|
|
|
Wires into the shared ``scraper/cli.py`` via the guarded ``register()`` pattern
|
|
(see ``scraper/parse_cli.py`` and ``scraper/relate_cli.py``):
|
|
|
|
* The ``analyze`` subcommand reads ``config.yaml``'s ``llm:`` section for the
|
|
endpoint + model knobs.
|
|
* When ``llm.enabled`` is false (or a server is unreachable / model empty) the
|
|
run uses the deterministic built-in tagger (zero network).
|
|
* Tags persist to ``bill_tags`` (``scraper.llm``); the tag cloud is written to
|
|
``out/tag_cloud.json`` (or ``--out``), plus a CLI summary.
|
|
* ``--force-llm`` overrides ``enabled: false`` to force the live server path.
|
|
* ``--list-config`` prints the resolved llm: settings and exits 0.
|
|
* ``--db`` overrides ``relational.DEFAULT_DB``.
|
|
* ``--roster`` is ignored (accepted for CLI compatibility with the correlate lane;
|
|
this lane does no roster matching).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
|
|
from . import llm, relational
|
|
from .config import load_config
|
|
|
|
log = logging.getLogger("scraper.analyze")
|
|
|
|
_DEFAULT_OUT_DIR = os.path.normpath(
|
|
os.path.join(os.path.dirname(relational.DEFAULT_DB), "..", "out"))
|
|
|
|
|
|
def _already_registered(sub, name: str) -> bool:
|
|
actions = getattr(sub, "_choices_actions", [])
|
|
return any(getattr(a, "name", None) == name for a in actions)
|
|
|
|
|
|
def register(sub) -> None:
|
|
"""Register the ``analyze`` subcommand (guarded — skip if a sibling owns it)."""
|
|
if _already_registered(sub, "analyze"):
|
|
return
|
|
|
|
pa = sub.add_parser(
|
|
"analyze",
|
|
help="subject-matter tag bills via Ollama (or deterministic fallback)",
|
|
description=("Tag every bill with subject-matter tags using a configurable "
|
|
"Ollama endpoint + model (from config.llm). Falls back to the "
|
|
"deterministic built-in tagger when llm.enabled=false, the endpoint "
|
|
"is unreachable, or model is empty. Persists to bill_tags; writes "
|
|
"the tag cloud to out/tag_cloud.json"),
|
|
)
|
|
pa.add_argument("--db", default=relational.DEFAULT_DB,
|
|
help="SQLite path (default: db/daily_file.sqlite)")
|
|
pa.add_argument("--out", dest="out_path", default=None,
|
|
help="tag-cloud JSON output path "
|
|
"(default: out/tag_cloud.json)")
|
|
pa.add_argument("--source", dest="house", default=None,
|
|
help="limit to a publish house, e.g. 'assembly' or 'senate' "
|
|
"(matches house or chamber_origin)")
|
|
pa.add_argument("--top", type=int, default=None,
|
|
help="limit the tag cloud / CLI summary to the N most frequent tags")
|
|
pa.add_argument("--rebuild", action="store_true",
|
|
help="clear bill_tags before writing (full re-analysis)")
|
|
pa.add_argument("--force-llm", dest="force_llm", action="store_true",
|
|
help="force the live Ollama path even when llm.enabled=false")
|
|
pa.add_argument("--list-config", action="store_true",
|
|
help="print resolved llm: config and exit 0")
|
|
pa.add_argument("--roster", dest="roster", default=None,
|
|
help="ignored (compat with the correlate lane)")
|
|
pa.set_defaults(func=handle_analyze)
|
|
|
|
|
|
def _resolved_cfg(config_path: str = None, force_llm: bool = False):
|
|
"""Load the ``llm:`` slice from config.yaml (defaults fill missing keys)."""
|
|
cfg = load_config(config_path)
|
|
llm_cfg = dict(cfg.llm)
|
|
if force_llm:
|
|
llm_cfg["enabled"] = True
|
|
return llm_cfg
|
|
|
|
|
|
def _print_llm_config(cfg_llm: dict, config_path: str = "") -> int:
|
|
print("llm:")
|
|
for k in ("enabled", "backend", "endpoint", "model",
|
|
"default_tags", "timeout_seconds"):
|
|
print(" %s: %s" % (k, cfg_llm.get(k)))
|
|
if config_path:
|
|
print(" (from %s)" % config_path)
|
|
return 0
|
|
|
|
|
|
def handle_analyze(args) -> int:
|
|
"""CLI handler for the ``analyze`` subcommand."""
|
|
logging.basicConfig(
|
|
level=logging.WARNING,
|
|
format="%(levelname)s %(name)s: %(message)s")
|
|
|
|
llm_cfg = _resolved_cfg(getattr(args, "config", None),
|
|
force_llm=getattr(args, "force_llm", False))
|
|
|
|
if getattr(args, "list_config", False):
|
|
return _print_llm_config(llm_cfg, getattr(args, "config", ""))
|
|
|
|
out_path = getattr(args, "out_path", None)
|
|
out = out_path or os.path.join(_DEFAULT_OUT_DIR, "tag_cloud.json")
|
|
|
|
db_path = getattr(args, "db", relational.DEFAULT_DB)
|
|
house = getattr(args, "house", None)
|
|
top = getattr(args, "top", None)
|
|
rebuild = getattr(args, "rebuild", False)
|
|
|
|
summary = llm.analyze(
|
|
db_path=db_path,
|
|
out_path=out,
|
|
cfg=llm_cfg,
|
|
house=house,
|
|
rebuild=rebuild,
|
|
top=top,
|
|
)
|
|
|
|
print("=== analyze ===")
|
|
print(" bills_tagged : %d" % summary["bills_tagged"])
|
|
print(" unique_tags : %d" % summary["unique_tags"])
|
|
print(" path : %s" % summary["path"])
|
|
print(" generated_by : %s" % summary["generated_by"])
|
|
print(" reason : %s" % summary["reason"])
|
|
print(" rows_written : %d" % summary["rows_written"])
|
|
print(" out : %s" % summary["cloud_path"])
|
|
print(" top tags :")
|
|
for entry in summary.get("top_tags", []):
|
|
print(" %4d %s" % (entry.get("count", 0), entry.get("tag", "")))
|
|
return 0
|