Initial commit

scraper/cli.py

@@ -0,0 +1,211 @@
+# CLI entry point: `python -m scraper <command>`.
+#
+#   ingest --date YYYY-MM-DD [--source NAME] [--force] [--json]
+#   list
+#   check --date YYYY-MM-DD
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from .config import load_config
+from .ingest import Ingestor
+from .parse_cli import register, handle_parse
+from .relate_cli import register as relate_register
+from .relate_cli import handle_migrate, handle_correlate
+from . import analyze_cli as _ac
+from .analyze_cli import handle_analyze
+from .dashboard import dashboard_cli as _dcli
+from .dashboard.dashboard_cli import handle_dashboard
+
+
+def cmd_ingest(args) -> int:
+       # --date is required by the task contract; if omitted, default to today
+       # so the cron wrapper can run with no arguments.
+    date = args.date or _today()
+    if getattr(args, "validate", False):
+        date = _validate_date(date)
+
+    exit_code = 0
+    try:
+        ing = Ingestor(config_path=args.config, force=args.force)
+        sources = _sources_arg(args)
+        result = ing.ingest(date, sources=sources)
+        ing.close()
+    except SystemExit as exc:
+              # config / argparse errors surface here
+        print("ERROR: %s" % exc, file=sys.stderr)
+        return int(exc.code or 2)
+
+    if args.json:
+        print(json.dumps(result.to_dict(), indent=2))
+    else:
+         _print_summary(result)
+
+         # Non-zero exit only on a *real* failure (antibot/transient/unexpected
+         # 404). An expected "no publication" 404 is a legitimate outcome for a
+         # non-legislative day; the cron job must not treat it as a failure.
+    for c in result.chambers:
+        if c.status in ("antibot", "transient"):
+            exit_code = 1
+    return exit_code
+
+
+def _print_summary(result) -> None:
+    print("=== Ingest %s ===" % result.date)
+    for c in result.chambers:
+        tag = {
+              "ok": "OK",
+              "cached": "CACHE",
+              "not_found": "MISS",
+              "antibot": "ANTIBOT",
+              "transient": "FAIL",
+          }.get(c.status, "-----")
+        path = c.path
+        if path is not None:
+            path = path.name if hasattr(path, "name") else str(path)
+        print("  [%s] %-18s %-10s %8dB attempts=%d  %s"
+                % (tag, c.source, c.status, c.bytes, c.attempts, c.note))
+        if getattr(c, "as_of", None):
+            print("           as_of=%s  url=%s" % (c.as_of, c.url))
+    print("  all_ok=%s" % result.all_ok)
+
+
+def cmd_list(args) -> int:
+    cfg = load_config(args.config)
+    print("Configured sources (from %s):" % cfg.path)
+    for name, s in cfg.sources.items():
+        print("  * %s   (%s)" % (name, s.get("kind")))
+        for key in ("url", "index_url", "pdf_base_url", "pdf_filename_pattern"):
+            if key in s:
+                val = s[key]
+                if isinstance(val, list):
+                    print("      %s: %s" % (key, " / ".join(val)))
+                else:
+                    print("      %s: %s" % (key, val))
+    return 0
+
+
+def cmd_check(args) -> int:
+          # Report cache status for a date without touching the network.
+    from .cache import RawCache
+    cfg = load_config(args.config)
+    rc = RawCache(cfg.raw_dir)
+    date = args.date or _today()
+    all_cached = True
+    print("=== Cache check %s ===" % date)
+    for name in cfg.source_names():
+        s = cfg.source(name)
+        ext = "pdf" if s.get("kind") == "pdf" else "html"
+        hit = rc.exists(name, date, ext)
+        all_cached = all_cached and hit
+        print("  %-18s %s" % (name, "CACHED" if hit else "MISSING"))
+    return 0 if all_cached else 3
+
+
+def _sources_arg(args):
+          # --source accepts a single value or a comma-separated list.
+    src = getattr(args, "source", None)
+    if not src:
+        return None
+    if isinstance(src, (list, tuple)):
+        return list(src)
+    return [p.strip() for p in src.split(",") if p.strip()]
+
+
+def _today() -> str:
+          # Local date so the cron wrapper can run with no args.
+    import datetime
+    return datetime.date.today().isoformat()
+
+
+def _validate_date(date: str) -> str:
+          # Enforce clean YYYY-MM-DD; a bad date is a clear error, not a crash.
+    import datetime
+    try:
+        datetime.date.fromisoformat(date)
+    except ValueError:
+        raise SystemExit(
+            "Invalid --date '%s': expected YYYY-MM-DD, e.g. 2026-09-02." % date)
+    return date
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+         prog="python -m scraper",
+          description="Ingest the CA Legislature Daily File pages.")
+    p.add_argument("--config", help="Path to config.yaml (default: ./config.yaml)")
+    sub = p.add_subparsers(dest="command", required=True)
+
+    # Parser lane (task t_fb8081e7): additive, guarded so it never shadows a
+    # sibling owner of the dispatcher if one registers `parse` later.
+    try:
+        register(sub)
+    except Exception:
+        pass
+
+    # Correlation lane (task t_2736f40d, spec 2.5): additive + guarded, same
+    # pattern as `parse`. Registers `migrate` + `correlate`; does not touch the
+    # ingest/parse/list/check handlers.
+    try:
+        relate_register(sub)
+    except Exception:
+        pass
+
+    # LLM analysis lane (task t_950780e0, spec §4): additive + guarded, same
+    # pattern as parse + correlate. No collision with sibling handlers.
+    try:
+        _ac.register(sub)
+    except Exception:
+       pass
+
+    # Dashboard UI lane (task t_b38e9af2, spec §5): additive + guarded, same
+    # pattern. Registers `dashboard`; does not touch sibling handlers.
+    try:
+       _dcli.register(sub)
+    except Exception:
+       pass
+
+    pi = sub.add_parser("ingest", help="Download + cache daily-file page(s)")
+    pi.add_argument("--date", help="YYYY-MM-DD (default: today)")
+    pi.add_argument("--source",
+                     help="Chamber name(s), comma-separated "
+                           "(assembly_daily_file, senate_daily_file). Default: all.")
+    pi.add_argument("--force", action="store_true",
+                     help="Re-fetch even if already cached.")
+    pi.add_argument("--json", action="store_true",
+                     help="Emit the result as JSON.")
+
+    pl = sub.add_parser("list", help="List configured sources + URLs")
+    pc = sub.add_parser("check", help="Cache status for a date (no network)")
+    pc.add_argument("--date", help="YYYY-MM-DD (default: today)")
+
+    return p
+
+
+def main(argv=None) -> int:
+       # argv defaults to sys.argv[1:] when the module is run directly.
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    args.validate = True
+    handlers = {
+        "ingest": cmd_ingest,
+        "list": cmd_list,
+        "check": cmd_check,
+        "parse": handle_parse,
+        "migrate": handle_migrate,
+        "correlate": handle_correlate,
+        "analyze": handle_analyze,
+        "dashboard": handle_dashboard,
+        }
+    handler = handlers.get(args.command)
+    if handler is None:
+        parser.print_help()
+        return 2
+    return handler(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())