# standalone parse subcommand for the parser lane # The shared entry point (`scraper/__main__.py` -> `cli.py`) is jointly owned # by the ingestion task (registers `ingest`/`list`/`check`) and this task # (`parse`). To avoid clobbering the sibling's files, the parse CLI lives in # this module and the shared dispatcher wires it in additively: from # .parse_cli import register, handle_parse; then register(sub) and # handlers['parse'] = handle_parse. Direct invocation: python -m # scraper.parse_main parse --input ... from __future__ import annotations import argparse import logging import sys from . import parsers from .io import write_jsonl, write_sqlite from .schema import ASSEMBLY_DAILY_FILE, SENATE_DAILY_FILE_PDF log = logging.getLogger("scraper.parse") def _already_registered(sub, name): actions = getattr(sub, "_choices_actions", []) return any(getattr(a, "name", None) == name for a in actions) def register(sub): if _already_registered(sub, "parse"): return pp = sub.add_parser("parse", help="parse cached Daily File HTML into JSONL / SQLite", description="Parse cached Daily File HTML into JSONL / SQLite records.") pp.add_argument("--input", "-i", required=True, help="HTML file, directory, or glob") pp.add_argument("--output", "-o", help="JSONL output path; omit for stdout") pp.add_argument("--sqlite", dest="sqlite_path", help="optional SQLite output path") pp.add_argument("--table", default="daily_file_records", help="SQLite table name") pp.add_argument("--house", default="assembly", choices=["assembly", "senate"], help="publishing house") pp.add_argument("--source", default=ASSEMBLY_DAILY_FILE, help="source tag for each record") pp.add_argument("--no-hearings", action="store_true", help="omit hearing records") pp.add_argument("--json-include-nulls", action="store_true", help="include null fields") pp.add_argument("--parser", default="lxml", choices=["lxml", "html.parser"], help="backend") pp.set_defaults(func=handle_parse) def _configure_logging(verbose): level = logging.WARNING if verbose == 1: level = logging.INFO elif verbose >= 2: level = logging.DEBUG fmt = "%(levelname)s %(name)s: %(message)s" logging.basicConfig(level=level, format=fmt) def handle_parse(args): _configure_logging(getattr(args, "verbose", 0)) house = args.house.upper() source = args.source if source == ASSEMBLY_DAILY_FILE and args.house == "senate": source = SENATE_DAILY_FILE_PDF records = parsers.parse(args.input, house=house, source=source, parser=args.parser, include_hearings=not args.no_hearings) out_count = 0 if args.output: out_count = write_jsonl(records, args.output, include_nulls=args.json_include_nulls) elif not args.sqlite_path: for rec in records: print(rec.to_json()) out_count = len(records) if args.sqlite_path: write_sqlite(records, args.sqlite_path, table=args.table) log.info("parsed %d record(s); jsonL=%s sqlite=%s", len(records), out_count if args.output else "stdout", "on" if args.sqlite_path else "off") return 0 def main(argv=None): p = argparse.ArgumentParser(prog="scraper.parse", description="Standalone parse CLI (parser lane).") p.add_argument("-v", "--verbose", action="count", default=0, help="-v INFO, -vv DEBUG") sub = p.add_subparsers(dest="command", required=True) register(sub) args = p.parse_args(argv) return args.func(args) if __name__ == "__main__": sys.exit(main())