"""Optional live-URL smoke test (scope item 3). Hits the real Assembly Daily File URL once, end-to-end through the real HTTP client, and parses the result. By default this test is SKIPPED so a CI run never depends on the external servers being up. Enable it with:: CALI_SCRAPER_SMOKE=1 python3 -m pytest tests/test_live_smoke.py -m smoke The "smoke" marker is registered in pyproject.toml (--strict-markers). Because the live URL is the only external dependency in the whole suite, keeping it opt-in is what makes the rest of the CI signal deterministic. """ from __future__ import annotations import os import pytest def _live_enabled() -> bool: # "1", "true", "yes", "on" (case-insensitive) all enable the smoke test. return os.environ.get("CALI_SCRAPER_SMOKE", "").strip().lower() in { "1", "true", "yes", "on", } # Skip the whole module unless the operator opts in. pytestmark = pytest.mark.skipif( not _live_enabled(), reason="live smoke test disabled (set CALI_SCRAPER_SMOKE=1 to run)", ) class TestLiveAssembly: """Opt-in: fetch the live Assembly Daily File and parse it.""" def test_live_assembly_daily_file_returns_body(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) from scraper.ingest import Ingestor config_path = os.environ.get("CALI_SCRAPER_CONFIG") \ or os.path.join(os.path.dirname(os.path.dirname(__file__)), "config.yaml") ing = Ingestor(config_path=config_path, force=True) # Use today's date as the cache/manifest key; the Assembly HTML is the # always-current live file (spec section 1). import datetime today = datetime.date.today().isoformat() res = ing.ingest(today, source="assembly_daily_file") cr = res.chambers[0] # The only hard requirement for a smoke test: we got *something* back # that is a real HTML document (not an antibot/transient/404). assert cr.status == "ok", f"unexpected ingest status: {cr.status}" assert cr.bytes > 0, "empty body from live source" assert cr.path is not None and cr.path.exists() # And the parser must handle whatever it is. import scraper.parsers as parsers recs = parsers.parse(str(cr.path)) # The Assembly file is always non-empty in practice, but a transient # skeleton day (recess) could legitimately yield zero records -- so we # only assert the batch did not crash, not a count. assert isinstance(recs, list) def test_live_http_reachable_or_documented_antibot(self): # A second, independent liveness check: the Senate index (which is the # human page, not the antibot PDF) must answer without raising. try: import httpx except Exception as exc: # pragma: no cover - httpx is a hard dep pytest.skip(f"httpx unavailable: {exc}") with httpx.Client(timeout=20, follow_redirects=True, headers={"User-Agent": "research-scanner/1.0"}) as cli: try: resp = cli.get("https://www.senate.ca.gov/dailyfile") except httpx.HTTPError as exc: pytest.skip(f"network unavailable: {exc}") # A 2xx/308/403 (antibot) all mean "the server responded"; anything that # is not a connection failure is a pass for a *reachability* smoke test. assert resp.status_code in (200, 301, 302, 308, 403), resp.status_code