130 lines
5.0 KiB
Python
130 lines
5.0 KiB
Python
# Additive CLI for the correlation lane (task t_2736f40d, spec 2.5).
|
|
#
|
|
# The shared entry point `scraper/__main__.py -> scraper/cli.py` is jointly
|
|
# owned by the ingestion lane (`ingest`/`list`/`check`) and the parser lane
|
|
# (`parse`, via scraper/parse_cli.py). To avoid clobbering sibling files this
|
|
# CLI lives in its own module and is wired into the shared dispatcher
|
|
# ADDITIVELY + DEFENSIVELY: `cli.py` calls `register(sub)` inside a try/except
|
|
# and adds its two handler entries without touching any sibling handler. The
|
|
# `migrate`/`correlate` names do not collide with sibling subcommands.
|
|
#
|
|
# Direct invocation (no shared dispatcher needed):
|
|
# python -m scraper.relate_cli migrate --db db/daily_file.sqlite
|
|
# python -m scraper.relate_cli correlate --roster rosters.json
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
|
|
from . import relational
|
|
|
|
log = __import__("logging").getLogger("scraper.relate_cli")
|
|
|
|
|
|
def _already_registered(sub, *names: str) -> bool:
|
|
# Guard: if any of our subcommands is already registered by a sibling,
|
|
# skip registration rather than shadowing it.
|
|
actions = getattr(sub, "_choices_actions", [])
|
|
present = {getattr(a, "name", None) for a in actions}
|
|
return any(n in present for n in names)
|
|
|
|
|
|
def register(sub) -> None:
|
|
"""Register the `migrate` and `correlate` subcommands (guarded).
|
|
|
|
Mirrors scraper/parse_cli.py::register so the pattern stays uniform across
|
|
lanes and the shared `cli.py` can wire it in without risk.
|
|
"""
|
|
if _already_registered(sub, "migrate", "correlate"):
|
|
return
|
|
|
|
pm = sub.add_parser(
|
|
"migrate",
|
|
help="create the relational schema (idempotent; flat table preserved)",
|
|
description=("Create/upgrade the relational schema; preserves the flat "
|
|
"daily_file_records table + adds normalized tables & view."),
|
|
)
|
|
pm.add_argument("--db", default=relational.DEFAULT_DB,
|
|
help="SQLite path (default: db/daily_file.sqlite)")
|
|
pm.add_argument("--check", action="store_true",
|
|
help="report schema state only (no writes)")
|
|
pm.set_defaults(func=handle_migrate)
|
|
|
|
pc = sub.add_parser(
|
|
"correlate",
|
|
help="derive bill<->committee<->member relations from the flat table",
|
|
description=("Backfill normalized relation tables from "
|
|
"daily_file_records. Committees with no roster source are "
|
|
"left with empty membership (never fabricated)."),
|
|
)
|
|
pc.add_argument("--db", default=relational.DEFAULT_DB,
|
|
help="SQLite path (default: db/daily_file.sqlite)")
|
|
pc.add_argument("--roster", dest="roster", default=None,
|
|
help="JSON file of committee rosters to backfill members")
|
|
pc.add_argument("--rebuild", action="store_true",
|
|
help="clear derived tables first, then backfill cleanly")
|
|
pc.set_defaults(func=handle_correlate)
|
|
|
|
|
|
def _configure_logging(verbose: int) -> None:
|
|
import logging
|
|
level = logging.WARNING
|
|
if verbose == 1:
|
|
level = logging.INFO
|
|
elif verbose >= 2:
|
|
level = logging.DEBUG
|
|
logging.basicConfig(level=level,
|
|
format="%(levelname)s %(name)s: %(message)s")
|
|
|
|
|
|
def handle_migrate(args) -> int:
|
|
# No --verbose flag on migrate; keep warnings visible so "no roster" gaps
|
|
# (which are warnings) still surface by default.
|
|
_configure_logging(0)
|
|
db = args.db
|
|
if getattr(args, "check", False):
|
|
summary = relational.migrate(db)
|
|
print("schema migration (check):")
|
|
print(" db:", summary["db"])
|
|
print(" objects:", ", ".join(summary["objects"]))
|
|
print(" flat_rows:", summary["flat_rows"], "version:",
|
|
summary["version"])
|
|
return 0
|
|
summary = relational.migrate(db)
|
|
print("migrated schema at %s" % summary["db"])
|
|
print(" objects:", ", ".join(summary["objects"]))
|
|
print(" flat_rows preserved:", summary["flat_rows"])
|
|
return 0
|
|
|
|
|
|
def handle_correlate(args) -> int:
|
|
# Verbose default OFF; the per-committee "no roster" warnings that the
|
|
# never-fabricate rule emits should be visible, so do not suppress log.
|
|
_configure_logging(0)
|
|
db = args.db
|
|
# Ensure the schema exists even if `migrate` was never run on this db.
|
|
relational.migrate(db)
|
|
counts = relational.correlate(db, rosters=args.roster,
|
|
rebuild=args.rebuild, quiet=False)
|
|
print("correlated %s" % db)
|
|
print(" counts:", ", ".join("%s=%d" % (k, v)
|
|
for k, v in sorted(counts.items())))
|
|
return 0
|
|
|
|
|
|
def main(argv=None) -> int:
|
|
import argparse
|
|
p = argparse.ArgumentParser(
|
|
prog="scraper.relate",
|
|
description="Relational correlation: bills <-> committees <-> members.")
|
|
p.add_argument("-v", "--verbose", action="count", default=0,
|
|
help="-v INFO, -vv DEBUG")
|
|
sub = p.add_subparsers(dest="command", required=True)
|
|
register(sub)
|
|
args = p.parse_args(argv)
|
|
return args.func(args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|