93 lines
5.0 KiB
YAML
93 lines
5.0 KiB
YAML
# California Legislature Daily File - ingestion configuration
|
|
# Consumed by: scraper. Produced from t_a171f4d1 (daily_file_sources.yaml spec).
|
|
# Edit URLs / retry / alert paths here; the code reads everything from this file.
|
|
|
|
# --- HTTP --------------------------------------------------------------
|
|
http:
|
|
user_agent: "Mozilla/5.0 (research-scanner/1.0; +https://github.com/your-org/cali_scraper)"
|
|
timeout_seconds: 20
|
|
max_retries: 3 # exponential back-off; 3 attempts total, then stop
|
|
backoff_base_seconds: 1.0 # delay before attempt N (1-based): base * 2**(N-1), capped
|
|
backoff_cap_seconds: 30.0
|
|
# Status codes considered transient and therefore retried. 404 is handled
|
|
# separately as "no file today"; retryable server faults are in this list.
|
|
retry_status: [500, 502, 503, 504]
|
|
# A 403 on the Senate PDF path is the documented antibot challenge. It is NOT
|
|
# a network failure: surface it as an alert and stop, rather than retry-storming.
|
|
antibot_status: [403]
|
|
|
|
# --- Raw HTML / byte cache --------------------------------------------
|
|
cache:
|
|
# Where downloaded raw files land. Re-running ingest for a date already in the
|
|
# cache is a NO-OP on the network (served straight from disk).
|
|
raw_dir: "cache/raw"
|
|
# Append-only log: one JSON object per fetch (url, status, bytes, source, time).
|
|
request_log: "cache/request_log.jsonl"
|
|
|
|
# --- Alerting ----------------------------------------------------------
|
|
# Hooks fire on 404 / 5xx / antibot so an operator is pinged instead of the run
|
|
# failing silently. `sink` selects a handler registered in scraper.alerts.
|
|
alerts:
|
|
enabled: true
|
|
# "log" always writes to the request log + stderr. Register "email"/"webhook"
|
|
# later by adding a handler name to ALERT_REGISTRY in scraper/alerts.py.
|
|
sink: "log"
|
|
# webhook: "https://hooks.example.com/..."
|
|
# email: "alerts@your-org.example"
|
|
|
|
# --- Chamber source definitions (URLs come from the research spec) -----
|
|
# IMPORTANT: neither daily-file page is date-addressable by URL. The Assembly
|
|
# page always renders the CURRENT live file (date is a data value read from the
|
|
# hearing table); the Senate publication is a date-ENCODED PDF filename. The
|
|
# --date value is the cache / manifest key, not a URL query parameter.
|
|
sources:
|
|
assembly_daily_file:
|
|
kind: "html"
|
|
url: "https://www.assembly.ca.gov/schedules-publications/daily-file"
|
|
aliases:
|
|
- "https://assembly.ca.gov/schedules-publications/daily-file"
|
|
- "https://assembly.ca.gov/dailyfile/pdfpublications"
|
|
antibot: false
|
|
# The live page is session-cumulative: if the requested date appears in the
|
|
# hearing table, that date "exists" for the Assembly.
|
|
resolve_date_from_body: true
|
|
# date-sort values look like "08/31/26"; strip any trailing "-NNN" suffix.
|
|
date_field: "hearing_date"
|
|
|
|
senate_daily_file:
|
|
kind: "pdf"
|
|
# The index page is live HTML; the actual publication is a date-encoded PDF
|
|
# served behind an antibot challenge (see http.antibot_status).
|
|
index_url: "https://www.senate.ca.gov/dailyfile"
|
|
index_canonical: "https://www.senate.ca.gov/publications/senate-daily-file"
|
|
pdf_base_url: "https://www.senate.ca.gov/dailyfile/publications/download/"
|
|
antibot: true
|
|
# SEN_{type}_{year}_R_{YYYYMMDD}.PDF -> 2026-09-02 => SEN_COM_2025_R_20260902.PDF
|
|
pdf_filename_pattern: "SEN_{type}_{year}_R_{date}.PDF"
|
|
default_pdf_type: "COM"
|
|
session_year_from_date: true
|
|
# 404 here means "no publication that day" (weekend / recess): EXPECTED, not
|
|
# an error. The scheduler must distinguish 404-no-file from 5xx-transient.
|
|
missing_is_expected_404: true
|
|
|
|
# --- LLM analysis (spec §4, task t_950780e0) -----------------------------
|
|
# Subject-matter tagging of bills via a configurable Ollama endpoint. The
|
|
# endpoint + model are NEVER hardcoded: everything below is a config knob.
|
|
#
|
|
# DETERMINISTIC DEGRADATION: when `enabled` is false, the endpoint is
|
|
# unreachable, or `model` is empty, `scraper analyze` falls back to a built-in
|
|
# deterministic keyword/short_title tagger (zero network) so analysis never
|
|
# blocks the build. The active path is logged and tagged on every bill_tags row.
|
|
#
|
|
# Start disabled: the default `analyze` run uses the built-in fallback tagger,
|
|
# which needs no server. Point `endpoint` at a local Ollama (default
|
|
# http://localhost:11434, OpenAI-compatible: /api/generate or /v1/chat/completions)
|
|
# and flip `enabled: true` to use a real model instead of the fallback.
|
|
llm:
|
|
enabled: false # false -> deterministic fallback tagger (no server)
|
|
backend: ollama # only "ollama" is supported by this lane
|
|
endpoint: "http://localhost:11434" # configurable; default local Ollama
|
|
model: "llama3" # configurable; empty -> fallback
|
|
default_tags: 8 # max tags emitted per bill
|
|
timeout_seconds: 30 # per-request HTTP timeout to the Ollama server
|