"""``scraper.analyze`` — spec §4 CLI subcommand for Ollama / fallback bill analysis. Wires into the shared ``scraper/cli.py`` via the guarded ``register()`` pattern (see ``scraper/parse_cli.py`` and ``scraper/relate_cli.py``): * The ``analyze`` subcommand reads ``config.yaml``'s ``llm:`` section for the endpoint + model knobs. * When ``llm.enabled`` is false (or a server is unreachable / model empty) the run uses the deterministic built-in tagger (zero network). * Tags persist to ``bill_tags`` (``scraper.llm``); the tag cloud is written to ``out/tag_cloud.json`` (or ``--out``), plus a CLI summary. * ``--force-llm`` overrides ``enabled: false`` to force the live server path. * ``--list-config`` prints the resolved llm: settings and exits 0. * ``--db`` overrides ``relational.DEFAULT_DB``. * ``--roster`` is ignored (accepted for CLI compatibility with the correlate lane; this lane does no roster matching). """ from __future__ import annotations import logging import os from . import llm, relational from .config import load_config log = logging.getLogger("scraper.analyze") _DEFAULT_OUT_DIR = os.path.normpath( os.path.join(os.path.dirname(relational.DEFAULT_DB), "..", "out")) def _already_registered(sub, name: str) -> bool: actions = getattr(sub, "_choices_actions", []) return any(getattr(a, "name", None) == name for a in actions) def register(sub) -> None: """Register the ``analyze`` subcommand (guarded — skip if a sibling owns it).""" if _already_registered(sub, "analyze"): return pa = sub.add_parser( "analyze", help="subject-matter tag bills via Ollama (or deterministic fallback)", description=("Tag every bill with subject-matter tags using a configurable " "Ollama endpoint + model (from config.llm). Falls back to the " "deterministic built-in tagger when llm.enabled=false, the endpoint " "is unreachable, or model is empty. Persists to bill_tags; writes " "the tag cloud to out/tag_cloud.json"), ) pa.add_argument("--db", default=relational.DEFAULT_DB, help="SQLite path (default: db/daily_file.sqlite)") pa.add_argument("--out", dest="out_path", default=None, help="tag-cloud JSON output path " "(default: out/tag_cloud.json)") pa.add_argument("--source", dest="house", default=None, help="limit to a publish house, e.g. 'assembly' or 'senate' " "(matches house or chamber_origin)") pa.add_argument("--top", type=int, default=None, help="limit the tag cloud / CLI summary to the N most frequent tags") pa.add_argument("--rebuild", action="store_true", help="clear bill_tags before writing (full re-analysis)") pa.add_argument("--force-llm", dest="force_llm", action="store_true", help="force the live Ollama path even when llm.enabled=false") pa.add_argument("--list-config", action="store_true", help="print resolved llm: config and exit 0") pa.add_argument("--roster", dest="roster", default=None, help="ignored (compat with the correlate lane)") pa.set_defaults(func=handle_analyze) def _resolved_cfg(config_path: str = None, force_llm: bool = False): """Load the ``llm:`` slice from config.yaml (defaults fill missing keys).""" cfg = load_config(config_path) llm_cfg = dict(cfg.llm) if force_llm: llm_cfg["enabled"] = True return llm_cfg def _print_llm_config(cfg_llm: dict, config_path: str = "") -> int: print("llm:") for k in ("enabled", "backend", "endpoint", "model", "default_tags", "timeout_seconds"): print(" %s: %s" % (k, cfg_llm.get(k))) if config_path: print(" (from %s)" % config_path) return 0 def handle_analyze(args) -> int: """CLI handler for the ``analyze`` subcommand.""" logging.basicConfig( level=logging.WARNING, format="%(levelname)s %(name)s: %(message)s") llm_cfg = _resolved_cfg(getattr(args, "config", None), force_llm=getattr(args, "force_llm", False)) if getattr(args, "list_config", False): return _print_llm_config(llm_cfg, getattr(args, "config", "")) out_path = getattr(args, "out_path", None) out = out_path or os.path.join(_DEFAULT_OUT_DIR, "tag_cloud.json") db_path = getattr(args, "db", relational.DEFAULT_DB) house = getattr(args, "house", None) top = getattr(args, "top", None) rebuild = getattr(args, "rebuild", False) summary = llm.analyze( db_path=db_path, out_path=out, cfg=llm_cfg, house=house, rebuild=rebuild, top=top, ) print("=== analyze ===") print(" bills_tagged : %d" % summary["bills_tagged"]) print(" unique_tags : %d" % summary["unique_tags"]) print(" path : %s" % summary["path"]) print(" generated_by : %s" % summary["generated_by"]) print(" reason : %s" % summary["reason"]) print(" rows_written : %d" % summary["rows_written"]) print(" out : %s" % summary["cloud_path"]) print(" top tags :") for entry in summary.get("top_tags", []): print(" %4d %s" % (entry.get("count", 0), entry.get("tag", ""))) return 0