212 lines
6.9 KiB
Python
212 lines
6.9 KiB
Python
# CLI entry point: `python -m scraper <command>`.
|
|
#
|
|
# ingest --date YYYY-MM-DD [--source NAME] [--force] [--json]
|
|
# list
|
|
# check --date YYYY-MM-DD
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
|
|
from .config import load_config
|
|
from .ingest import Ingestor
|
|
from .parse_cli import register, handle_parse
|
|
from .relate_cli import register as relate_register
|
|
from .relate_cli import handle_migrate, handle_correlate
|
|
from . import analyze_cli as _ac
|
|
from .analyze_cli import handle_analyze
|
|
from .dashboard import dashboard_cli as _dcli
|
|
from .dashboard.dashboard_cli import handle_dashboard
|
|
|
|
|
|
def cmd_ingest(args) -> int:
|
|
# --date is required by the task contract; if omitted, default to today
|
|
# so the cron wrapper can run with no arguments.
|
|
date = args.date or _today()
|
|
if getattr(args, "validate", False):
|
|
date = _validate_date(date)
|
|
|
|
exit_code = 0
|
|
try:
|
|
ing = Ingestor(config_path=args.config, force=args.force)
|
|
sources = _sources_arg(args)
|
|
result = ing.ingest(date, sources=sources)
|
|
ing.close()
|
|
except SystemExit as exc:
|
|
# config / argparse errors surface here
|
|
print("ERROR: %s" % exc, file=sys.stderr)
|
|
return int(exc.code or 2)
|
|
|
|
if args.json:
|
|
print(json.dumps(result.to_dict(), indent=2))
|
|
else:
|
|
_print_summary(result)
|
|
|
|
# Non-zero exit only on a *real* failure (antibot/transient/unexpected
|
|
# 404). An expected "no publication" 404 is a legitimate outcome for a
|
|
# non-legislative day; the cron job must not treat it as a failure.
|
|
for c in result.chambers:
|
|
if c.status in ("antibot", "transient"):
|
|
exit_code = 1
|
|
return exit_code
|
|
|
|
|
|
def _print_summary(result) -> None:
|
|
print("=== Ingest %s ===" % result.date)
|
|
for c in result.chambers:
|
|
tag = {
|
|
"ok": "OK",
|
|
"cached": "CACHE",
|
|
"not_found": "MISS",
|
|
"antibot": "ANTIBOT",
|
|
"transient": "FAIL",
|
|
}.get(c.status, "-----")
|
|
path = c.path
|
|
if path is not None:
|
|
path = path.name if hasattr(path, "name") else str(path)
|
|
print(" [%s] %-18s %-10s %8dB attempts=%d %s"
|
|
% (tag, c.source, c.status, c.bytes, c.attempts, c.note))
|
|
if getattr(c, "as_of", None):
|
|
print(" as_of=%s url=%s" % (c.as_of, c.url))
|
|
print(" all_ok=%s" % result.all_ok)
|
|
|
|
|
|
def cmd_list(args) -> int:
|
|
cfg = load_config(args.config)
|
|
print("Configured sources (from %s):" % cfg.path)
|
|
for name, s in cfg.sources.items():
|
|
print(" * %s (%s)" % (name, s.get("kind")))
|
|
for key in ("url", "index_url", "pdf_base_url", "pdf_filename_pattern"):
|
|
if key in s:
|
|
val = s[key]
|
|
if isinstance(val, list):
|
|
print(" %s: %s" % (key, " / ".join(val)))
|
|
else:
|
|
print(" %s: %s" % (key, val))
|
|
return 0
|
|
|
|
|
|
def cmd_check(args) -> int:
|
|
# Report cache status for a date without touching the network.
|
|
from .cache import RawCache
|
|
cfg = load_config(args.config)
|
|
rc = RawCache(cfg.raw_dir)
|
|
date = args.date or _today()
|
|
all_cached = True
|
|
print("=== Cache check %s ===" % date)
|
|
for name in cfg.source_names():
|
|
s = cfg.source(name)
|
|
ext = "pdf" if s.get("kind") == "pdf" else "html"
|
|
hit = rc.exists(name, date, ext)
|
|
all_cached = all_cached and hit
|
|
print(" %-18s %s" % (name, "CACHED" if hit else "MISSING"))
|
|
return 0 if all_cached else 3
|
|
|
|
|
|
def _sources_arg(args):
|
|
# --source accepts a single value or a comma-separated list.
|
|
src = getattr(args, "source", None)
|
|
if not src:
|
|
return None
|
|
if isinstance(src, (list, tuple)):
|
|
return list(src)
|
|
return [p.strip() for p in src.split(",") if p.strip()]
|
|
|
|
|
|
def _today() -> str:
|
|
# Local date so the cron wrapper can run with no args.
|
|
import datetime
|
|
return datetime.date.today().isoformat()
|
|
|
|
|
|
def _validate_date(date: str) -> str:
|
|
# Enforce clean YYYY-MM-DD; a bad date is a clear error, not a crash.
|
|
import datetime
|
|
try:
|
|
datetime.date.fromisoformat(date)
|
|
except ValueError:
|
|
raise SystemExit(
|
|
"Invalid --date '%s': expected YYYY-MM-DD, e.g. 2026-09-02." % date)
|
|
return date
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
p = argparse.ArgumentParser(
|
|
prog="python -m scraper",
|
|
description="Ingest the CA Legislature Daily File pages.")
|
|
p.add_argument("--config", help="Path to config.yaml (default: ./config.yaml)")
|
|
sub = p.add_subparsers(dest="command", required=True)
|
|
|
|
# Parser lane (task t_fb8081e7): additive, guarded so it never shadows a
|
|
# sibling owner of the dispatcher if one registers `parse` later.
|
|
try:
|
|
register(sub)
|
|
except Exception:
|
|
pass
|
|
|
|
# Correlation lane (task t_2736f40d, spec 2.5): additive + guarded, same
|
|
# pattern as `parse`. Registers `migrate` + `correlate`; does not touch the
|
|
# ingest/parse/list/check handlers.
|
|
try:
|
|
relate_register(sub)
|
|
except Exception:
|
|
pass
|
|
|
|
# LLM analysis lane (task t_950780e0, spec §4): additive + guarded, same
|
|
# pattern as parse + correlate. No collision with sibling handlers.
|
|
try:
|
|
_ac.register(sub)
|
|
except Exception:
|
|
pass
|
|
|
|
# Dashboard UI lane (task t_b38e9af2, spec §5): additive + guarded, same
|
|
# pattern. Registers `dashboard`; does not touch sibling handlers.
|
|
try:
|
|
_dcli.register(sub)
|
|
except Exception:
|
|
pass
|
|
|
|
pi = sub.add_parser("ingest", help="Download + cache daily-file page(s)")
|
|
pi.add_argument("--date", help="YYYY-MM-DD (default: today)")
|
|
pi.add_argument("--source",
|
|
help="Chamber name(s), comma-separated "
|
|
"(assembly_daily_file, senate_daily_file). Default: all.")
|
|
pi.add_argument("--force", action="store_true",
|
|
help="Re-fetch even if already cached.")
|
|
pi.add_argument("--json", action="store_true",
|
|
help="Emit the result as JSON.")
|
|
|
|
pl = sub.add_parser("list", help="List configured sources + URLs")
|
|
pc = sub.add_parser("check", help="Cache status for a date (no network)")
|
|
pc.add_argument("--date", help="YYYY-MM-DD (default: today)")
|
|
|
|
return p
|
|
|
|
|
|
def main(argv=None) -> int:
|
|
# argv defaults to sys.argv[1:] when the module is run directly.
|
|
parser = build_parser()
|
|
args = parser.parse_args(argv)
|
|
args.validate = True
|
|
handlers = {
|
|
"ingest": cmd_ingest,
|
|
"list": cmd_list,
|
|
"check": cmd_check,
|
|
"parse": handle_parse,
|
|
"migrate": handle_migrate,
|
|
"correlate": handle_correlate,
|
|
"analyze": handle_analyze,
|
|
"dashboard": handle_dashboard,
|
|
}
|
|
handler = handlers.get(args.command)
|
|
if handler is None:
|
|
parser.print_help()
|
|
return 2
|
|
return handler(args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|