87 lines
3.5 KiB
Python
87 lines
3.5 KiB
Python
# standalone parse subcommand for the parser lane
|
|
# The shared entry point (`scraper/__main__.py` -> `cli.py`) is jointly owned
|
|
# by the ingestion task (registers `ingest`/`list`/`check`) and this task
|
|
# (`parse`). To avoid clobbering the sibling's files, the parse CLI lives in
|
|
# this module and the shared dispatcher wires it in additively: from
|
|
# .parse_cli import register, handle_parse; then register(sub) and
|
|
# handlers['parse'] = handle_parse. Direct invocation: python -m
|
|
# scraper.parse_main parse --input ...
|
|
|
|
from __future__ import annotations
|
|
import argparse
|
|
import logging
|
|
import sys
|
|
from . import parsers
|
|
from .io import write_jsonl, write_sqlite
|
|
from .schema import ASSEMBLY_DAILY_FILE, SENATE_DAILY_FILE_PDF
|
|
|
|
log = logging.getLogger("scraper.parse")
|
|
|
|
|
|
def _already_registered(sub, name):
|
|
actions = getattr(sub, "_choices_actions", [])
|
|
return any(getattr(a, "name", None) == name for a in actions)
|
|
|
|
|
|
def register(sub):
|
|
if _already_registered(sub, "parse"):
|
|
return
|
|
pp = sub.add_parser("parse", help="parse cached Daily File HTML into JSONL / SQLite",
|
|
description="Parse cached Daily File HTML into JSONL / SQLite records.")
|
|
pp.add_argument("--input", "-i", required=True, help="HTML file, directory, or glob")
|
|
pp.add_argument("--output", "-o", help="JSONL output path; omit for stdout")
|
|
pp.add_argument("--sqlite", dest="sqlite_path", help="optional SQLite output path")
|
|
pp.add_argument("--table", default="daily_file_records", help="SQLite table name")
|
|
pp.add_argument("--house", default="assembly", choices=["assembly", "senate"], help="publishing house")
|
|
pp.add_argument("--source", default=ASSEMBLY_DAILY_FILE, help="source tag for each record")
|
|
pp.add_argument("--no-hearings", action="store_true", help="omit hearing records")
|
|
pp.add_argument("--json-include-nulls", action="store_true", help="include null fields")
|
|
pp.add_argument("--parser", default="lxml", choices=["lxml", "html.parser"], help="backend")
|
|
pp.set_defaults(func=handle_parse)
|
|
|
|
|
|
def _configure_logging(verbose):
|
|
level = logging.WARNING
|
|
if verbose == 1:
|
|
level = logging.INFO
|
|
elif verbose >= 2:
|
|
level = logging.DEBUG
|
|
fmt = "%(levelname)s %(name)s: %(message)s"
|
|
logging.basicConfig(level=level, format=fmt)
|
|
|
|
|
|
def handle_parse(args):
|
|
_configure_logging(getattr(args, "verbose", 0))
|
|
house = args.house.upper()
|
|
source = args.source
|
|
if source == ASSEMBLY_DAILY_FILE and args.house == "senate":
|
|
source = SENATE_DAILY_FILE_PDF
|
|
records = parsers.parse(args.input, house=house, source=source,
|
|
parser=args.parser, include_hearings=not args.no_hearings)
|
|
out_count = 0
|
|
if args.output:
|
|
out_count = write_jsonl(records, args.output, include_nulls=args.json_include_nulls)
|
|
elif not args.sqlite_path:
|
|
for rec in records:
|
|
print(rec.to_json())
|
|
out_count = len(records)
|
|
if args.sqlite_path:
|
|
write_sqlite(records, args.sqlite_path, table=args.table)
|
|
log.info("parsed %d record(s); jsonL=%s sqlite=%s", len(records),
|
|
out_count if args.output else "stdout", "on" if args.sqlite_path else "off")
|
|
return 0
|
|
|
|
|
|
def main(argv=None):
|
|
p = argparse.ArgumentParser(prog="scraper.parse", description="Standalone parse CLI (parser lane).")
|
|
p.add_argument("-v", "--verbose", action="count", default=0, help="-v INFO, -vv DEBUG")
|
|
sub = p.add_subparsers(dest="command", required=True)
|
|
register(sub)
|
|
args = p.parse_args(argv)
|
|
return args.func(args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|
|
|