# Additive CLI for the correlation lane (task t_2736f40d, spec 2.5). # # The shared entry point `scraper/__main__.py -> scraper/cli.py` is jointly # owned by the ingestion lane (`ingest`/`list`/`check`) and the parser lane # (`parse`, via scraper/parse_cli.py). To avoid clobbering sibling files this # CLI lives in its own module and is wired into the shared dispatcher # ADDITIVELY + DEFENSIVELY: `cli.py` calls `register(sub)` inside a try/except # and adds its two handler entries without touching any sibling handler. The # `migrate`/`correlate` names do not collide with sibling subcommands. # # Direct invocation (no shared dispatcher needed): # python -m scraper.relate_cli migrate --db db/daily_file.sqlite # python -m scraper.relate_cli correlate --roster rosters.json from __future__ import annotations import sys from . import relational log = __import__("logging").getLogger("scraper.relate_cli") def _already_registered(sub, *names: str) -> bool: # Guard: if any of our subcommands is already registered by a sibling, # skip registration rather than shadowing it. actions = getattr(sub, "_choices_actions", []) present = {getattr(a, "name", None) for a in actions} return any(n in present for n in names) def register(sub) -> None: """Register the `migrate` and `correlate` subcommands (guarded). Mirrors scraper/parse_cli.py::register so the pattern stays uniform across lanes and the shared `cli.py` can wire it in without risk. """ if _already_registered(sub, "migrate", "correlate"): return pm = sub.add_parser( "migrate", help="create the relational schema (idempotent; flat table preserved)", description=("Create/upgrade the relational schema; preserves the flat " "daily_file_records table + adds normalized tables & view."), ) pm.add_argument("--db", default=relational.DEFAULT_DB, help="SQLite path (default: db/daily_file.sqlite)") pm.add_argument("--check", action="store_true", help="report schema state only (no writes)") pm.set_defaults(func=handle_migrate) pc = sub.add_parser( "correlate", help="derive bill<->committee<->member relations from the flat table", description=("Backfill normalized relation tables from " "daily_file_records. Committees with no roster source are " "left with empty membership (never fabricated)."), ) pc.add_argument("--db", default=relational.DEFAULT_DB, help="SQLite path (default: db/daily_file.sqlite)") pc.add_argument("--roster", dest="roster", default=None, help="JSON file of committee rosters to backfill members") pc.add_argument("--rebuild", action="store_true", help="clear derived tables first, then backfill cleanly") pc.set_defaults(func=handle_correlate) def _configure_logging(verbose: int) -> None: import logging level = logging.WARNING if verbose == 1: level = logging.INFO elif verbose >= 2: level = logging.DEBUG logging.basicConfig(level=level, format="%(levelname)s %(name)s: %(message)s") def handle_migrate(args) -> int: # No --verbose flag on migrate; keep warnings visible so "no roster" gaps # (which are warnings) still surface by default. _configure_logging(0) db = args.db if getattr(args, "check", False): summary = relational.migrate(db) print("schema migration (check):") print(" db:", summary["db"]) print(" objects:", ", ".join(summary["objects"])) print(" flat_rows:", summary["flat_rows"], "version:", summary["version"]) return 0 summary = relational.migrate(db) print("migrated schema at %s" % summary["db"]) print(" objects:", ", ".join(summary["objects"])) print(" flat_rows preserved:", summary["flat_rows"]) return 0 def handle_correlate(args) -> int: # Verbose default OFF; the per-committee "no roster" warnings that the # never-fabricate rule emits should be visible, so do not suppress log. _configure_logging(0) db = args.db # Ensure the schema exists even if `migrate` was never run on this db. relational.migrate(db) counts = relational.correlate(db, rosters=args.roster, rebuild=args.rebuild, quiet=False) print("correlated %s" % db) print(" counts:", ", ".join("%s=%d" % (k, v) for k, v in sorted(counts.items()))) return 0 def main(argv=None) -> int: import argparse p = argparse.ArgumentParser( prog="scraper.relate", description="Relational correlation: bills <-> committees <-> members.") p.add_argument("-v", "--verbose", action="count", default=0, help="-v INFO, -vv DEBUG") sub = p.add_subparsers(dest="command", required=True) register(sub) args = p.parse_args(argv) return args.func(args) if __name__ == "__main__": sys.exit(main())