Initial commit

scraper/relate_cli.py

@@ -0,0 +1,129 @@
+# Additive CLI for the correlation lane (task t_2736f40d, spec 2.5).
+#
+# The shared entry point `scraper/__main__.py -> scraper/cli.py` is jointly
+# owned by the ingestion lane (`ingest`/`list`/`check`) and the parser lane
+# (`parse`, via scraper/parse_cli.py). To avoid clobbering sibling files this
+# CLI lives in its own module and is wired into the shared dispatcher
+# ADDITIVELY + DEFENSIVELY: `cli.py` calls `register(sub)` inside a try/except
+# and adds its two handler entries without touching any sibling handler. The
+# `migrate`/`correlate` names do not collide with sibling subcommands.
+#
+# Direct invocation (no shared dispatcher needed):
+#     python -m scraper.relate_cli migrate --db db/daily_file.sqlite
+#     python -m scraper.relate_cli correlate --roster rosters.json
+
+from __future__ import annotations
+
+import sys
+
+from . import relational
+
+log = __import__("logging").getLogger("scraper.relate_cli")
+
+
+def _already_registered(sub, *names: str) -> bool:
+    # Guard: if any of our subcommands is already registered by a sibling,
+    # skip registration rather than shadowing it.
+    actions = getattr(sub, "_choices_actions", [])
+    present = {getattr(a, "name", None) for a in actions}
+    return any(n in present for n in names)
+
+
+def register(sub) -> None:
+    """Register the `migrate` and `correlate` subcommands (guarded).
+
+    Mirrors scraper/parse_cli.py::register so the pattern stays uniform across
+    lanes and the shared `cli.py` can wire it in without risk.
+    """
+    if _already_registered(sub, "migrate", "correlate"):
+        return
+
+    pm = sub.add_parser(
+        "migrate",
+        help="create the relational schema (idempotent; flat table preserved)",
+        description=("Create/upgrade the relational schema; preserves the flat "
+                     "daily_file_records table + adds normalized tables & view."),
+    )
+    pm.add_argument("--db", default=relational.DEFAULT_DB,
+                    help="SQLite path (default: db/daily_file.sqlite)")
+    pm.add_argument("--check", action="store_true",
+                    help="report schema state only (no writes)")
+    pm.set_defaults(func=handle_migrate)
+
+    pc = sub.add_parser(
+        "correlate",
+        help="derive bill<->committee<->member relations from the flat table",
+        description=("Backfill normalized relation tables from "
+                     "daily_file_records. Committees with no roster source are "
+                     "left with empty membership (never fabricated)."),
+    )
+    pc.add_argument("--db", default=relational.DEFAULT_DB,
+                    help="SQLite path (default: db/daily_file.sqlite)")
+    pc.add_argument("--roster", dest="roster", default=None,
+                    help="JSON file of committee rosters to backfill members")
+    pc.add_argument("--rebuild", action="store_true",
+                    help="clear derived tables first, then backfill cleanly")
+    pc.set_defaults(func=handle_correlate)
+
+
+def _configure_logging(verbose: int) -> None:
+    import logging
+    level = logging.WARNING
+    if verbose == 1:
+        level = logging.INFO
+    elif verbose >= 2:
+        level = logging.DEBUG
+    logging.basicConfig(level=level,
+                        format="%(levelname)s %(name)s: %(message)s")
+
+
+def handle_migrate(args) -> int:
+    # No --verbose flag on migrate; keep warnings visible so "no roster" gaps
+    # (which are warnings) still surface by default.
+    _configure_logging(0)
+    db = args.db
+    if getattr(args, "check", False):
+        summary = relational.migrate(db)
+        print("schema migration (check):")
+        print("  db:", summary["db"])
+        print("  objects:", ", ".join(summary["objects"]))
+        print("  flat_rows:", summary["flat_rows"], "version:",
+              summary["version"])
+        return 0
+    summary = relational.migrate(db)
+    print("migrated schema at %s" % summary["db"])
+    print("  objects:", ", ".join(summary["objects"]))
+    print("  flat_rows preserved:", summary["flat_rows"])
+    return 0
+
+
+def handle_correlate(args) -> int:
+    # Verbose default OFF; the per-committee "no roster" warnings that the
+    # never-fabricate rule emits should be visible, so do not suppress log.
+    _configure_logging(0)
+    db = args.db
+    # Ensure the schema exists even if `migrate` was never run on this db.
+    relational.migrate(db)
+    counts = relational.correlate(db, rosters=args.roster,
+                                  rebuild=args.rebuild, quiet=False)
+    print("correlated %s" % db)
+    print("  counts:", ", ".join("%s=%d" % (k, v)
+                                  for k, v in sorted(counts.items())))
+    return 0
+
+
+def main(argv=None) -> int:
+    import argparse
+    p = argparse.ArgumentParser(
+        prog="scraper.relate",
+        description="Relational correlation: bills <-> committees <-> members.")
+    p.add_argument("-v", "--verbose", action="count", default=0,
+                   help="-v INFO, -vv DEBUG")
+    sub = p.add_subparsers(dest="command", required=True)
+    register(sub)
+    args = p.parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())