From ace59762cd94347b048ccc7d3dc524aae6155ab0 Mon Sep 17 00:00:00 2001 From: kfj001 Date: Sun, 6 Sep 2026 20:09:28 -0700 Subject: [PATCH] Adds readme documentation, and startup script. --- README.md | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ start.sh | 65 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+) create mode 100644 README.md create mode 100755 start.sh diff --git a/README.md b/README.md new file mode 100644 index 0000000..d03cd78 --- /dev/null +++ b/README.md @@ -0,0 +1,81 @@ +# cali_scraper + +A scraper for the California Legislature's Daily File publications — the +Assembly and Senate pages that list bills up for hearing — plus a +relational store and a local web dashboard for browsing the results. + +## What this is + +The pipeline has three lanes: + +1. **Ingest (the scrape)** — `python -m scraper ingest`. Downloads and caches + the Assembly daily-file HTML page and the Senate daily-file PDF for a given + date, plus the chambers' "upcoming events" calendars. Raw pages are cached + on disk (`cache/raw/`), so re-running the same date hits the network zero + times. A legitimate "no publication that day" (weekend / recess) is treated + as a normal outcome, not an error. +2. **Store & analyze** — `parse`, `migrate`, and `correlate` load the cached + pages into a SQLite database (`db/daily_file.sqlite`, schema in + `SCHEMA.md`); `analyze` runs subject-matter tagging on bills (built-in + keyword tagger by default; optionally a local Ollama model, see `llm:` in + `config.yaml`). +3. **Dashboard (web server)** — `python -m scraper dashboard`. Serves an API + plus a static frontend on `http://127.0.0.1:8000/` so you can browse bills, + committees, members, and tags without touching the database directly. + +`start.sh` at the repo root runs the scrapes, then starts the dashboard. + +## How to kick it off + +### Prerequisites + +- Python 3.9+ +- Dependencies: `pip install -r requirements.txt` +- Populate the database at least once so the dashboard has data to show (see + below — `start.sh` scrapes but does not build `db/daily_file.sqlite`). + +### Just run it + +```bash +./start.sh +``` + +That scrapes **today's** daily files (plus the events calendars), then serves +the dashboard at http://127.0.0.1:8000/. Press `Ctrl+C` to stop. + +### Useful variations + +```bash +# scrape a specific date, then serve +DATE=2026-09-02 ./start.sh + +# force a network re-fetch even if a date is already cached +FORCE=1 ./start.sh + +# serve on a different port / bind address +PORT=9000 HOST=0.0.0.0 ./start.sh + +# skip the scrape and just start the dashboard +SKIP_SCRAPE=1 ./start.sh +``` + +### Build the database (one time) + +`start.sh` handles the scrape + server, but to see bills in the dashboard the +SQLite database must be built from the cached pages: + +```bash +python -m scraper ingest --date 2026-09-02 # download + cache +python -m scraper ingest --date 2026-09-02 --store-events \ + --db db/daily_file.sqlite # events calendar +python -m scraper parse --input cache/raw/assembly_daily_file \ + --house assembly --sqlite db/daily_file.sqlite # bills table +python -m scraper migrate --db db/daily_file.sqlite # relational schema +python -m scraper correlate --db db/daily_file.sqlite # bill <-> committee +python -m scraper analyze --db db/daily_file.sqlite # subject tags +``` + +Then `./start.sh` (or `python -m scraper dashboard` directly) will serve it. + +See `INGESTION.md`, `DAILY_FILE_PARSE_SPEC.md`, `CORRELATE.md`, and +`LLM_ANALYSIS.md` for the full command reference. \ No newline at end of file diff --git a/start.sh b/start.sh new file mode 100755 index 0000000..25ad37b --- /dev/null +++ b/start.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Run the daily-file scrape and then serve the dashboard. +# +# Usage: +# ./start.sh # scrape today, then serve on :8000 +# ./start.sh --force # force re-fetch even if cached +# DATE=2026-09-01 ./start.sh # scrape a specific date +# +# Environment overrides: +# DATE - YYYY-MM-DD to scrape (default: today) +# PORT - dashboard port (default: 8000) +# HOST - dashboard bind address (default: 127.0.0.1) +# FORCE - set to 1 to pass --force to ingest +# SKIP_SCRAPE - set to 1 to skip the scrape step entirely + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="${SCRIPT_DIR}" +cd "${ROOT}" || { echo "[start] cannot cd to ${ROOT}" >&2; exit 2; } + +PY="${PYTHON:-python3}" +DATE="${DATE:-$(date +%F)}" +PORT="${PORT:-8000}" +HOST="${HOST:-127.0.0.1}" +FORCE_FLAG="" +if [ "${FORCE:-0}" = "1" ]; then + FORCE_FLAG="--force" +fi + +# ── Step 1: scrape ────────────────────────────────────────────────────── +if [ "${SKIP_SCRAPE:-0}" != "1" ]; then + echo "===================================================================" + echo "[start] scrape date=${DATE} root=${ROOT}" + echo "===================================================================" + + "${PY}" -u -m scraper ingest --date "${DATE}" ${FORCE_FLAG} + INGEST_EXIT=$? + + case "${INGEST_EXIT}" in + 0) echo "[start] scrape OK (ok / cached / no publication)" ;; + 1) echo "[start] scrape FAILED (see cache/request_log.jsonl)" >&2 ;; + *) echo "[start] scrape ERROR (exit ${INGEST_EXIT})" >&2 ;; + esac + + # Also fetch the chamber events calendar (non-fatal if it fails). + if [ "${SKIP_EVENTS:-0}" != "1" ]; then + "${PY}" -u -m scraper ingest --date "${DATE}" ${FORCE_FLAG} \ + --store-events --db db/daily_file.sqlite 2>&1 | head -5 + fi + + # Don't abort on scrape failure — the dashboard can still serve cached data. + if [ "${INGEST_EXIT}" -ne 0 ]; then + echo "[start] continuing to dashboard despite scrape exit ${INGEST_EXIT}" >&2 + fi +else + echo "[start] scrape skipped (SKIP_SCRAPE=1)" +fi + +# ── Step 2: serve ─────────────────────────────────────────────────────── +echo "" +echo "===================================================================" +echo "[start] dashboard http://${HOST}:${PORT}/" +echo "===================================================================" +exec "${PY}" -u -m scraper dashboard --host "${HOST}" --port "${PORT}" --log