Initial commit

tests/test_live_smoke.py

@@ -0,0 +1,78 @@
+"""Optional live-URL smoke test (scope item 3).
+
+Hits the real Assembly Daily File URL once, end-to-end through the real HTTP
+client, and parses the result. By default this test is SKIPPED so a CI run never
+depends on the external servers being up. Enable it with::
+
+    CALI_SCRAPER_SMOKE=1 python3 -m pytest tests/test_live_smoke.py -m smoke
+
+The "smoke" marker is registered in pyproject.toml (--strict-markers). Because
+the live URL is the only external dependency in the whole suite, keeping it
+opt-in is what makes the rest of the CI signal deterministic.
+"""
+from __future__ import annotations
+
+import os
+
+import pytest
+
+
+def _live_enabled() -> bool:
+      # "1", "true", "yes", "on" (case-insensitive) all enable the smoke test.
+    return os.environ.get("CALI_SCRAPER_SMOKE", "").strip().lower() in {
+          "1", "true", "yes", "on",
+      }
+
+
+# Skip the whole module unless the operator opts in.
+pytestmark = pytest.mark.skipif(
+      not _live_enabled(),
+      reason="live smoke test disabled (set CALI_SCRAPER_SMOKE=1 to run)",
+)
+
+
+class TestLiveAssembly:
+    """Opt-in: fetch the live Assembly Daily File and parse it."""
+
+    def test_live_assembly_daily_file_returns_body(self, tmp_path, monkeypatch):
+        monkeypatch.chdir(tmp_path)
+        from scraper.ingest import Ingestor
+        config_path = os.environ.get("CALI_SCRAPER_CONFIG") \
+            or os.path.join(os.path.dirname(os.path.dirname(__file__)),
+                               "config.yaml")
+        ing = Ingestor(config_path=config_path, force=True)
+        # Use today's date as the cache/manifest key; the Assembly HTML is the
+        # always-current live file (spec section 1).
+        import datetime
+        today = datetime.date.today().isoformat()
+        res = ing.ingest(today, source="assembly_daily_file")
+        cr = res.chambers[0]
+         # The only hard requirement for a smoke test: we got *something* back
+        # that is a real HTML document (not an antibot/transient/404).
+        assert cr.status == "ok", f"unexpected ingest status: {cr.status}"
+        assert cr.bytes > 0, "empty body from live source"
+        assert cr.path is not None and cr.path.exists()
+         # And the parser must handle whatever it is.
+        import scraper.parsers as parsers
+        recs = parsers.parse(str(cr.path))
+         # The Assembly file is always non-empty in practice, but a transient
+        # skeleton day (recess) could legitimately yield zero records -- so we
+        # only assert the batch did not crash, not a count.
+        assert isinstance(recs, list)
+
+    def test_live_http_reachable_or_documented_antibot(self):
+          # A second, independent liveness check: the Senate index (which is the
+         # human page, not the antibot PDF) must answer without raising.
+        try:
+            import httpx
+        except Exception as exc:   # pragma: no cover - httpx is a hard dep
+            pytest.skip(f"httpx unavailable: {exc}")
+        with httpx.Client(timeout=20, follow_redirects=True,
+                            headers={"User-Agent": "research-scanner/1.0"}) as cli:
+            try:
+                resp = cli.get("https://www.senate.ca.gov/dailyfile")
+            except httpx.HTTPError as exc:
+                pytest.skip(f"network unavailable: {exc}")
+        # A 2xx/308/403 (antibot) all mean "the server responded"; anything that
+         # is not a connection failure is a pass for a *reachability* smoke test.
+        assert resp.status_code in (200, 301, 302, 308, 403), resp.status_code