Initial commit

config.yaml

@@ -0,0 +1,92 @@
+# California Legislature Daily File - ingestion configuration
+# Consumed by: scraper. Produced from t_a171f4d1 (daily_file_sources.yaml spec).
+# Edit URLs / retry / alert paths here; the code reads everything from this file.
+
+# --- HTTP --------------------------------------------------------------
+http:
+    user_agent: "Mozilla/5.0 (research-scanner/1.0; +https://github.com/your-org/cali_scraper)"
+    timeout_seconds: 20
+    max_retries: 3             # exponential back-off; 3 attempts total, then stop
+    backoff_base_seconds: 1.0 # delay before attempt N (1-based): base * 2**(N-1), capped
+    backoff_cap_seconds: 30.0
+    # Status codes considered transient and therefore retried. 404 is handled
+    # separately as "no file today"; retryable server faults are in this list.
+    retry_status: [500, 502, 503, 504]
+    # A 403 on the Senate PDF path is the documented antibot challenge. It is NOT
+    # a network failure: surface it as an alert and stop, rather than retry-storming.
+    antibot_status: [403]
+
+# --- Raw HTML / byte cache --------------------------------------------
+cache:
+    # Where downloaded raw files land. Re-running ingest for a date already in the
+    # cache is a NO-OP on the network (served straight from disk).
+    raw_dir: "cache/raw"
+    # Append-only log: one JSON object per fetch (url, status, bytes, source, time).
+    request_log: "cache/request_log.jsonl"
+
+# --- Alerting ----------------------------------------------------------
+# Hooks fire on 404 / 5xx / antibot so an operator is pinged instead of the run
+# failing silently. `sink` selects a handler registered in scraper.alerts.
+alerts:
+    enabled: true
+    # "log" always writes to the request log + stderr. Register "email"/"webhook"
+    # later by adding a handler name to ALERT_REGISTRY in scraper/alerts.py.
+    sink: "log"
+    # webhook: "https://hooks.example.com/..."
+    # email: "alerts@your-org.example"
+
+# --- Chamber source definitions (URLs come from the research spec) -----
+# IMPORTANT: neither daily-file page is date-addressable by URL. The Assembly
+# page always renders the CURRENT live file (date is a data value read from the
+# hearing table); the Senate publication is a date-ENCODED PDF filename. The
+# --date value is the cache / manifest key, not a URL query parameter.
+sources:
+    assembly_daily_file:
+        kind: "html"
+        url: "https://www.assembly.ca.gov/schedules-publications/daily-file"
+        aliases:
+            - "https://assembly.ca.gov/schedules-publications/daily-file"
+            - "https://assembly.ca.gov/dailyfile/pdfpublications"
+        antibot: false
+        # The live page is session-cumulative: if the requested date appears in the
+        # hearing table, that date "exists" for the Assembly.
+        resolve_date_from_body: true
+        # date-sort values look like "08/31/26"; strip any trailing "-NNN" suffix.
+        date_field: "hearing_date"
+
+    senate_daily_file:
+        kind: "pdf"
+        # The index page is live HTML; the actual publication is a date-encoded PDF
+        # served behind an antibot challenge (see http.antibot_status).
+        index_url: "https://www.senate.ca.gov/dailyfile"
+        index_canonical: "https://www.senate.ca.gov/publications/senate-daily-file"
+        pdf_base_url: "https://www.senate.ca.gov/dailyfile/publications/download/"
+        antibot: true
+        # SEN_{type}_{year}_R_{YYYYMMDD}.PDF   ->  2026-09-02 => SEN_COM_2025_R_20260902.PDF
+        pdf_filename_pattern: "SEN_{type}_{year}_R_{date}.PDF"
+        default_pdf_type: "COM"
+        session_year_from_date: true
+        # 404 here means "no publication that day" (weekend / recess): EXPECTED, not
+        # an error. The scheduler must distinguish 404-no-file from 5xx-transient.
+        missing_is_expected_404: true
+
+# --- LLM analysis (spec §4, task t_950780e0) -----------------------------
+# Subject-matter tagging of bills via a configurable Ollama endpoint. The
+# endpoint + model are NEVER hardcoded: everything below is a config knob.
+#
+# DETERMINISTIC DEGRADATION: when `enabled` is false, the endpoint is
+# unreachable, or `model` is empty, `scraper analyze` falls back to a built-in
+# deterministic keyword/short_title tagger (zero network) so analysis never
+# blocks the build. The active path is logged and tagged on every bill_tags row.
+#
+# Start disabled: the default `analyze` run uses the built-in fallback tagger,
+# which needs no server. Point `endpoint` at a local Ollama (default
+# http://localhost:11434, OpenAI-compatible: /api/generate or /v1/chat/completions)
+# and flip `enabled: true` to use a real model instead of the fallback.
+llm:
+    enabled: false            # false -> deterministic fallback tagger (no server)
+    backend: ollama           # only "ollama" is supported by this lane
+    endpoint: "http://localhost:11434"   # configurable; default local Ollama
+    model: "llama3"                          # configurable; empty -> fallback
+    default_tags: 8            # max tags emitted per bill
+    timeout_seconds: 30        # per-request HTTP timeout to the Ollama server