# California Legislature Daily File - ingestion configuration
# Consumed by: scraper. Produced from t_a171f4d1 (daily_file_sources.yaml spec).
# Edit URLs / retry / alert paths here; the code reads everything from this file.

# --- HTTP --------------------------------------------------------------
http:
    user_agent: "Mozilla/5.0 (research-scanner/1.0; +https://github.com/your-org/cali_scraper)"
    timeout_seconds: 20
    max_retries: 3             # exponential back-off; 3 attempts total, then stop
    backoff_base_seconds: 1.0 # delay before attempt N (1-based): base * 2**(N-1), capped
    backoff_cap_seconds: 30.0
    # Status codes considered transient and therefore retried. 404 is handled
    # separately as "no file today"; retryable server faults are in this list.
    retry_status: [500, 502, 503, 504]
    # A 403 on the Senate PDF path is the documented antibot challenge. It is NOT
    # a network failure: surface it as an alert and stop, rather than retry-storming.
    antibot_status: [403]

# --- Raw HTML / byte cache --------------------------------------------
cache:
    # Where downloaded raw files land. Re-running ingest for a date already in the
    # cache is a NO-OP on the network (served straight from disk).
    raw_dir: "cache/raw"
    # Append-only log: one JSON object per fetch (url, status, bytes, source, time).
    request_log: "cache/request_log.jsonl"

# --- Alerting ----------------------------------------------------------
# Hooks fire on 404 / 5xx / antibot so an operator is pinged instead of the run
# failing silently. `sink` selects a handler registered in scraper.alerts.
alerts:
    enabled: true
    # "log" always writes to the request log + stderr. Register "email"/"webhook"
    # later by adding a handler name to ALERT_REGISTRY in scraper/alerts.py.
    sink: "log"
    # webhook: "https://hooks.example.com/..."
    # email: "alerts@your-org.example"

# --- Chamber source definitions (URLs come from the research spec) -----
# IMPORTANT: neither daily-file page is date-addressable by URL. The Assembly
# page always renders the CURRENT live file (date is a data value read from the
# hearing table); the Senate publication is a date-ENCODED PDF filename. The
# --date value is the cache / manifest key, not a URL query parameter.
sources:
    assembly_daily_file:
        kind: "html"
        url: "https://www.assembly.ca.gov/schedules-publications/daily-file"
        aliases:
            - "https://assembly.ca.gov/schedules-publications/daily-file"
            - "https://assembly.ca.gov/dailyfile/pdfpublications"
        antibot: false
        # The live page is session-cumulative: if the requested date appears in the
        # hearing table, that date "exists" for the Assembly.
        resolve_date_from_body: true
        # date-sort values look like "08/31/26"; strip any trailing "-NNN" suffix.
        date_field: "hearing_date"

    senate_daily_file:
        kind: "pdf"
        # The index page is live HTML; the actual publication is a date-encoded PDF
        # served behind an antibot challenge (see http.antibot_status).
        index_url: "https://www.senate.ca.gov/dailyfile"
        index_canonical: "https://www.senate.ca.gov/publications/senate-daily-file"
        pdf_base_url: "https://www.senate.ca.gov/dailyfile/publications/download/"
        antibot: true
        # SEN_{type}_{year}_R_{YYYYMMDD}.PDF   ->  2026-09-02 => SEN_COM_2025_R_20260902.PDF
        pdf_filename_pattern: "SEN_{type}_{year}_R_{date}.PDF"
        default_pdf_type: "COM"
        session_year_from_date: true
        # 404 here means "no publication that day" (weekend / recess): EXPECTED, not
        # an error. The scheduler must distinguish 404-no-file from 5xx-transient.
        missing_is_expected_404: true

    # Forward-looking chamber calendar. Unlike the daily file (a backward
    # snapshot) this page lists the *upcoming* hearings/floor sessions -- the
    # only genuine future-dated signal. Empty during recess (a legitimate "no
    # upcoming events" state, DESIGN section 6.1), not an error. Parsed into
    # the scheduled_events table (schema v2) by `ingest --store-events`.
    assembly_events:
        kind: "events"
        url: "https://www.assembly.ca.gov/events/upcoming"
        chamber: "assembly"
        empty_is_ok: true

    senate_events:
        kind: "events"
        url: "https://www.senate.ca.gov/calendar"
        chamber: "senate"
        empty_is_ok: true

# --- LLM analysis (spec §4, task t_950780e0) -----------------------------
# Subject-matter tagging of bills via a configurable Ollama endpoint. The
# endpoint + model are NEVER hardcoded: everything below is a config knob.
#
# DETERMINISTIC DEGRADATION: when `enabled` is false, the endpoint is
# unreachable, or `model` is empty, `scraper analyze` falls back to a built-in
# deterministic keyword/short_title tagger (zero network) so analysis never
# blocks the build. The active path is logged and tagged on every bill_tags row.
#
# Start disabled: the default `analyze` run uses the built-in fallback tagger,
# which needs no server. Point `endpoint` at a local Ollama (default
# http://localhost:11434, OpenAI-compatible: /api/generate or /v1/chat/completions)
# and flip `enabled: true` to use a real model instead of the fallback.
llm:
    enabled: false            # false -> deterministic fallback tagger (no server)
    backend: ollama           # only "ollama" is supported by this lane
    endpoint: "http://localhost:11434"   # configurable; default local Ollama
    model: "llama3"                          # configurable; empty -> fallback
    default_tags: 8            # max tags emitted per bill
    timeout_seconds: 30        # per-request HTTP timeout to the Ollama server