cali_scraper/scraper/cli.py

# CLI entry point: `python -m scraper <command>`.
#
#   ingest --date YYYY-MM-DD [--source NAME] [--force] [--json]
#   list
#   check --date YYYY-MM-DD

from __future__ import annotations

import argparse
import json
import sys

from .config import load_config
from .ingest import Ingestor
from .parse_cli import register, handle_parse
from .relate_cli import register as relate_register
from .relate_cli import handle_migrate, handle_correlate
from . import analyze_cli as _ac
from .analyze_cli import handle_analyze
from .dashboard import dashboard_cli as _dcli
from .dashboard.dashboard_cli import handle_dashboard


def cmd_ingest(args) -> int:
       # --date is required by the task contract; if omitted, default to today
       # so the cron wrapper can run with no arguments.
    date = args.date or _today()
    if getattr(args, "validate", False):
        date = _validate_date(date)

    exit_code = 0
    try:
        ing = Ingestor(config_path=args.config, force=args.force)
        sources = _sources_arg(args)
        if getattr(args, "store_events", False):
              # Forward-looking chamber calendar -> scheduled_events table. Reuses
              # the same cache-first pipeline; a recess (empty calendar) is a
              # no-op, not a failure. Default to the events sources when none are
              # named (the daily-file sources are a different lane).
            if not sources:
                sources = [n for n, s in ing.cfg.sources.items()
                           if s.get("kind") == "events"]
            result = ing.ingest_events(date, sources=sources,
                                       db_path=getattr(args, "db", None))
        else:
            result = ing.ingest(date, sources=sources)
        ing.close()
    except SystemExit as exc:
            # config / argparse errors surface here
        print("ERROR: %s" % exc, file=sys.stderr)
        return int(exc.code or 2)

    if args.json:
        print(json.dumps(result.to_dict(), indent=2))
    else:
         _print_summary(result)

         # Non-zero exit only on a *real* failure (antibot/transient/unexpected
         # 404). An expected "no publication" 404 is a legitimate outcome for a
         # non-legislative day; the cron job must not treat it as a failure.
    for c in result.chambers:
        if c.status in ("antibot", "transient"):
            exit_code = 1
    return exit_code


def _print_summary(result) -> None:
    print("=== Ingest %s ===" % result.date)
    for c in result.chambers:
        tag = {
              "ok": "OK",
              "cached": "CACHE",
              "not_found": "MISS",
              "antibot": "ANTIBOT",
              "transient": "FAIL",
          }.get(c.status, "-----")
        path = c.path
        if path is not None:
            path = path.name if hasattr(path, "name") else str(path)
        print("  [%s] %-18s %-10s %8dB attempts=%d  %s"
                % (tag, c.source, c.status, c.bytes, c.attempts, c.note))
        if getattr(c, "as_of", None):
            print("           as_of=%s  url=%s" % (c.as_of, c.url))
    print("  all_ok=%s" % result.all_ok)


def cmd_list(args) -> int:
    cfg = load_config(args.config)
    print("Configured sources (from %s):" % cfg.path)
    for name, s in cfg.sources.items():
        print("  * %s   (%s)" % (name, s.get("kind")))
        for key in ("url", "index_url", "pdf_base_url", "pdf_filename_pattern"):
            if key in s:
                val = s[key]
                if isinstance(val, list):
                    print("      %s: %s" % (key, " / ".join(val)))
                else:
                    print("      %s: %s" % (key, val))
    return 0


def cmd_check(args) -> int:
          # Report cache status for a date without touching the network.
    from .cache import RawCache
    cfg = load_config(args.config)
    rc = RawCache(cfg.raw_dir)
    date = args.date or _today()
    all_cached = True
    print("=== Cache check %s ===" % date)
    for name in cfg.source_names():
        s = cfg.source(name)
        ext = "pdf" if s.get("kind") == "pdf" else "html"
        hit = rc.exists(name, date, ext)
        all_cached = all_cached and hit
        print("  %-18s %s" % (name, "CACHED" if hit else "MISSING"))
    return 0 if all_cached else 3


def _sources_arg(args):
          # --source accepts a single value or a comma-separated list.
    src = getattr(args, "source", None)
    if not src:
        return None
    if isinstance(src, (list, tuple)):
        return list(src)
    return [p.strip() for p in src.split(",") if p.strip()]


def _today() -> str:
          # Local date so the cron wrapper can run with no args.
    import datetime
    return datetime.date.today().isoformat()


def _validate_date(date: str) -> str:
          # Enforce clean YYYY-MM-DD; a bad date is a clear error, not a crash.
    import datetime
    try:
        datetime.date.fromisoformat(date)
    except ValueError:
        raise SystemExit(
            "Invalid --date '%s': expected YYYY-MM-DD, e.g. 2026-09-02." % date)
    return date


def build_parser() -> argparse.ArgumentParser:
    p = argparse.ArgumentParser(
         prog="python -m scraper",
          description="Ingest the CA Legislature Daily File pages.")
    p.add_argument("--config", help="Path to config.yaml (default: ./config.yaml)")
    sub = p.add_subparsers(dest="command", required=True)

    # Parser lane (task t_fb8081e7): additive, guarded so it never shadows a
    # sibling owner of the dispatcher if one registers `parse` later.
    try:
        register(sub)
    except Exception:
        pass

    # Correlation lane (task t_2736f40d, spec 2.5): additive + guarded, same
    # pattern as `parse`. Registers `migrate` + `correlate`; does not touch the
    # ingest/parse/list/check handlers.
    try:
        relate_register(sub)
    except Exception:
        pass

    # LLM analysis lane (task t_950780e0, spec §4): additive + guarded, same
    # pattern as parse + correlate. No collision with sibling handlers.
    try:
        _ac.register(sub)
    except Exception:
       pass

    # Dashboard UI lane (task t_b38e9af2, spec §5): additive + guarded, same
    # pattern. Registers `dashboard`; does not touch sibling handlers.
    try:
       _dcli.register(sub)
    except Exception:
       pass

    pi = sub.add_parser("ingest", help="Download + cache daily-file page(s)")
    pi.add_argument("--date", help="YYYY-MM-DD (default: today)")
    pi.add_argument("--source",
                     help="Chamber name(s), comma-separated "
                            "(assembly_daily_file, senate_daily_file). Default: all.")
    pi.add_argument("--force", action="store_true",
                     help="Re-fetch even if already cached.")
    pi.add_argument("--json", action="store_true",
                     help="Emit the result as JSON.")
    pi.add_argument("--store-events", dest="store_events", action="store_true",
                     help="Fetch + parse the chamber 'upcoming events' calendar "
                            "and store it in the scheduled_events table "
                            "(forward-looking; the daily-file queue is backward).")
    pi.add_argument("--db", dest="db", default=None,
                     help="SQLite path for --store-events "
                            "(default: the relational lane's db/daily_file.sqlite).")

    pl = sub.add_parser("list", help="List configured sources + URLs")
    pc = sub.add_parser("check", help="Cache status for a date (no network)")
    pc.add_argument("--date", help="YYYY-MM-DD (default: today)")

    return p


def main(argv=None) -> int:
       # argv defaults to sys.argv[1:] when the module is run directly.
    parser = build_parser()
    args = parser.parse_args(argv)
    args.validate = True
    handlers = {
        "ingest": cmd_ingest,
        "list": cmd_list,
        "check": cmd_check,
        "parse": handle_parse,
        "migrate": handle_migrate,
        "correlate": handle_correlate,
        "analyze": handle_analyze,
        "dashboard": handle_dashboard,
        }
    handler = handlers.get(args.command)
    if handler is None:
        parser.print_help()
        return 2
    return handler(args)


if __name__ == "__main__":
    raise SystemExit(main())