Initial commit

scraper/parse_cli.py

@@ -0,0 +1,86 @@
+# standalone parse subcommand for the parser lane
+# The shared entry point (`scraper/__main__.py` -> `cli.py`) is jointly owned
+# by the ingestion task (registers `ingest`/`list`/`check`) and this task
+# (`parse`). To avoid clobbering the sibling's files, the parse CLI lives in
+# this module and the shared dispatcher wires it in additively: from
+# .parse_cli import register, handle_parse; then register(sub) and
+# handlers['parse'] = handle_parse. Direct invocation: python -m
+# scraper.parse_main parse --input ...
+
+from __future__ import annotations
+import argparse
+import logging
+import sys
+from . import parsers
+from .io import write_jsonl, write_sqlite
+from .schema import ASSEMBLY_DAILY_FILE, SENATE_DAILY_FILE_PDF
+
+log = logging.getLogger("scraper.parse")
+
+
+def _already_registered(sub, name):
+    actions = getattr(sub, "_choices_actions", [])
+    return any(getattr(a, "name", None) == name for a in actions)
+
+
+def register(sub):
+    if _already_registered(sub, "parse"):
+        return
+    pp = sub.add_parser("parse", help="parse cached Daily File HTML into JSONL / SQLite",
+                         description="Parse cached Daily File HTML into JSONL / SQLite records.")
+    pp.add_argument("--input", "-i", required=True, help="HTML file, directory, or glob")
+    pp.add_argument("--output", "-o", help="JSONL output path; omit for stdout")
+    pp.add_argument("--sqlite", dest="sqlite_path", help="optional SQLite output path")
+    pp.add_argument("--table", default="daily_file_records", help="SQLite table name")
+    pp.add_argument("--house", default="assembly", choices=["assembly", "senate"], help="publishing house")
+    pp.add_argument("--source", default=ASSEMBLY_DAILY_FILE, help="source tag for each record")
+    pp.add_argument("--no-hearings", action="store_true", help="omit hearing records")
+    pp.add_argument("--json-include-nulls", action="store_true", help="include null fields")
+    pp.add_argument("--parser", default="lxml", choices=["lxml", "html.parser"], help="backend")
+    pp.set_defaults(func=handle_parse)
+
+
+def _configure_logging(verbose):
+    level = logging.WARNING
+    if verbose == 1:
+        level = logging.INFO
+    elif verbose >= 2:
+        level = logging.DEBUG
+    fmt = "%(levelname)s %(name)s: %(message)s"
+    logging.basicConfig(level=level, format=fmt)
+
+
+def handle_parse(args):
+    _configure_logging(getattr(args, "verbose", 0))
+    house = args.house.upper()
+    source = args.source
+    if source == ASSEMBLY_DAILY_FILE and args.house == "senate":
+        source = SENATE_DAILY_FILE_PDF
+    records = parsers.parse(args.input, house=house, source=source,
+                            parser=args.parser, include_hearings=not args.no_hearings)
+    out_count = 0
+    if args.output:
+        out_count = write_jsonl(records, args.output, include_nulls=args.json_include_nulls)
+    elif not args.sqlite_path:
+        for rec in records:
+            print(rec.to_json())
+        out_count = len(records)
+    if args.sqlite_path:
+        write_sqlite(records, args.sqlite_path, table=args.table)
+    log.info("parsed %d record(s); jsonL=%s sqlite=%s", len(records),
+             out_count if args.output else "stdout", "on" if args.sqlite_path else "off")
+    return 0
+
+
+def main(argv=None):
+    p = argparse.ArgumentParser(prog="scraper.parse", description="Standalone parse CLI (parser lane).")
+    p.add_argument("-v", "--verbose", action="count", default=0, help="-v INFO, -vv DEBUG")
+    sub = p.add_subparsers(dest="command", required=True)
+    register(sub)
+    args = p.parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())
+