Initial commit

scripts/run_ingest.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# Cron-compatible wrapper for the California Daily File ingestion layer.
+#
+# Install (nightly at 02:00 PT -- see install notes below) with:
+#   crontab -e   ->
+#     0 2 * * *   /Users/kyle/Documents/cali_scraper/scripts/run_ingest.sh
+#
+# Behavior:
+#   * Runs `python -m scraper ingest` for BOTH chambers for "today" (local date),
+#     unless CALI_SCRAPER_DATE overrides it (YYYY-MM-DD).
+#   * --force is NOT used by default: a date already in the local raw cache is
+#     served from disk with zero network (the "re-parse doesn't re-hit" req).
+#   * Exit code is 0 on ok / cached / expected-404 (no publication that day),
+#     1 on a real failure (antibot / transient / unexpected 404) so cron mail
+#     fires only for problems worth paging on.
+#   * Every request is appended to cache/request_log.jsonl.
+
+set -uo pipefail
+
+# Resolve the repo root regardless of the cron caller's working directory.
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+cd "${ROOT}" || { echo "[run_ingest] cannot cd to ${ROOT}" >&2; exit 2; }
+
+# Pick the Python interpreter (prefer a project venv if present).
+PY="${PYTHON:-python3}"
+
+# Date to ingest: CALI_SCRAPER_DATE or today in the local (PT) timezone.
+DATE="${CALI_SCRAPER_DATE:-$(date +%F)}"
+
+# Force a network refresh even if cached? CALI_SCRAPER_FORCE=1 enables --force.
+FORCE=""
+if [ "${CALI_SCRAPER_FORCE:-0}" = "1" ]; then
+    FORCE="--force"
+fi
+
+echo "==================================================================="
+echo "[run_ingest] ${DATE}  root=${ROOT}  py=${PY}"
+echo "==================================================================="
+
+# -u: unbuffered; the CLI prints the human-readable summary.
+"${PY}" -u -m scraper ingest --date "${DATE}" ${FORCE}
+EXIT=$?
+
+case "${EXIT}" in
+     0) echo "[run_ingest] DONE  ${DATE}  (ok / cached / no publication -- all expected)" ;;
+     1) echo "[run_ingest] FAILED ${DATE}  see cache/request_log.jsonl for the attempt trail" >&2 ;;
+    *)  echo "[run_ingest] ERROR  ${DATE}  (exit ${EXIT})" >&2 ;;
+esac
+
+exit "${EXIT}"