52 lines
2.0 KiB
Bash
Executable File
52 lines
2.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Cron-compatible wrapper for the California Daily File ingestion layer.
|
|
#
|
|
# Install (nightly at 02:00 PT -- see install notes below) with:
|
|
# crontab -e ->
|
|
# 0 2 * * * /Users/kyle/Documents/cali_scraper/scripts/run_ingest.sh
|
|
#
|
|
# Behavior:
|
|
# * Runs `python -m scraper ingest` for BOTH chambers for "today" (local date),
|
|
# unless CALI_SCRAPER_DATE overrides it (YYYY-MM-DD).
|
|
# * --force is NOT used by default: a date already in the local raw cache is
|
|
# served from disk with zero network (the "re-parse doesn't re-hit" req).
|
|
# * Exit code is 0 on ok / cached / expected-404 (no publication that day),
|
|
# 1 on a real failure (antibot / transient / unexpected 404) so cron mail
|
|
# fires only for problems worth paging on.
|
|
# * Every request is appended to cache/request_log.jsonl.
|
|
|
|
set -uo pipefail
|
|
|
|
# Resolve the repo root regardless of the cron caller's working directory.
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
|
cd "${ROOT}" || { echo "[run_ingest] cannot cd to ${ROOT}" >&2; exit 2; }
|
|
|
|
# Pick the Python interpreter (prefer a project venv if present).
|
|
PY="${PYTHON:-python3}"
|
|
|
|
# Date to ingest: CALI_SCRAPER_DATE or today in the local (PT) timezone.
|
|
DATE="${CALI_SCRAPER_DATE:-$(date +%F)}"
|
|
|
|
# Force a network refresh even if cached? CALI_SCRAPER_FORCE=1 enables --force.
|
|
FORCE=""
|
|
if [ "${CALI_SCRAPER_FORCE:-0}" = "1" ]; then
|
|
FORCE="--force"
|
|
fi
|
|
|
|
echo "==================================================================="
|
|
echo "[run_ingest] ${DATE} root=${ROOT} py=${PY}"
|
|
echo "==================================================================="
|
|
|
|
# -u: unbuffered; the CLI prints the human-readable summary.
|
|
"${PY}" -u -m scraper ingest --date "${DATE}" ${FORCE}
|
|
EXIT=$?
|
|
|
|
case "${EXIT}" in
|
|
0) echo "[run_ingest] DONE ${DATE} (ok / cached / no publication -- all expected)" ;;
|
|
1) echo "[run_ingest] FAILED ${DATE} see cache/request_log.jsonl for the attempt trail" >&2 ;;
|
|
*) echo "[run_ingest] ERROR ${DATE} (exit ${EXIT})" >&2 ;;
|
|
esac
|
|
|
|
exit "${EXIT}"
|