79 lines
3.4 KiB
Python
79 lines
3.4 KiB
Python
"""Optional live-URL smoke test (scope item 3).
|
|
|
|
Hits the real Assembly Daily File URL once, end-to-end through the real HTTP
|
|
client, and parses the result. By default this test is SKIPPED so a CI run never
|
|
depends on the external servers being up. Enable it with::
|
|
|
|
CALI_SCRAPER_SMOKE=1 python3 -m pytest tests/test_live_smoke.py -m smoke
|
|
|
|
The "smoke" marker is registered in pyproject.toml (--strict-markers). Because
|
|
the live URL is the only external dependency in the whole suite, keeping it
|
|
opt-in is what makes the rest of the CI signal deterministic.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
|
|
import pytest
|
|
|
|
|
|
def _live_enabled() -> bool:
|
|
# "1", "true", "yes", "on" (case-insensitive) all enable the smoke test.
|
|
return os.environ.get("CALI_SCRAPER_SMOKE", "").strip().lower() in {
|
|
"1", "true", "yes", "on",
|
|
}
|
|
|
|
|
|
# Skip the whole module unless the operator opts in.
|
|
pytestmark = pytest.mark.skipif(
|
|
not _live_enabled(),
|
|
reason="live smoke test disabled (set CALI_SCRAPER_SMOKE=1 to run)",
|
|
)
|
|
|
|
|
|
class TestLiveAssembly:
|
|
"""Opt-in: fetch the live Assembly Daily File and parse it."""
|
|
|
|
def test_live_assembly_daily_file_returns_body(self, tmp_path, monkeypatch):
|
|
monkeypatch.chdir(tmp_path)
|
|
from scraper.ingest import Ingestor
|
|
config_path = os.environ.get("CALI_SCRAPER_CONFIG") \
|
|
or os.path.join(os.path.dirname(os.path.dirname(__file__)),
|
|
"config.yaml")
|
|
ing = Ingestor(config_path=config_path, force=True)
|
|
# Use today's date as the cache/manifest key; the Assembly HTML is the
|
|
# always-current live file (spec section 1).
|
|
import datetime
|
|
today = datetime.date.today().isoformat()
|
|
res = ing.ingest(today, source="assembly_daily_file")
|
|
cr = res.chambers[0]
|
|
# The only hard requirement for a smoke test: we got *something* back
|
|
# that is a real HTML document (not an antibot/transient/404).
|
|
assert cr.status == "ok", f"unexpected ingest status: {cr.status}"
|
|
assert cr.bytes > 0, "empty body from live source"
|
|
assert cr.path is not None and cr.path.exists()
|
|
# And the parser must handle whatever it is.
|
|
import scraper.parsers as parsers
|
|
recs = parsers.parse(str(cr.path))
|
|
# The Assembly file is always non-empty in practice, but a transient
|
|
# skeleton day (recess) could legitimately yield zero records -- so we
|
|
# only assert the batch did not crash, not a count.
|
|
assert isinstance(recs, list)
|
|
|
|
def test_live_http_reachable_or_documented_antibot(self):
|
|
# A second, independent liveness check: the Senate index (which is the
|
|
# human page, not the antibot PDF) must answer without raising.
|
|
try:
|
|
import httpx
|
|
except Exception as exc: # pragma: no cover - httpx is a hard dep
|
|
pytest.skip(f"httpx unavailable: {exc}")
|
|
with httpx.Client(timeout=20, follow_redirects=True,
|
|
headers={"User-Agent": "research-scanner/1.0"}) as cli:
|
|
try:
|
|
resp = cli.get("https://www.senate.ca.gov/dailyfile")
|
|
except httpx.HTTPError as exc:
|
|
pytest.skip(f"network unavailable: {exc}")
|
|
# A 2xx/308/403 (antibot) all mean "the server responded"; anything that
|
|
# is not a connection failure is a pass for a *reachability* smoke test.
|
|
assert resp.status_code in (200, 301, 302, 308, 403), resp.status_code
|