Initial commit

conftest.py

@@ -0,0 +1,133 @@
+"""Root conftest: make the repo importable and provide shared test fixtures.
+
+Adds the repository root to sys.path so both the ``scraper`` package and the
+``tests`` package import cleanly regardless of the invocation directory, and
+exposes helpers the test-suite shares -- notably a fake HTTP transport that
+drives the REAL retry/backoff loop with no network, and a per-test log capture.
+Nothing here touches the live network unless a test opts in via the
+``CALI_SCRAPER_SMOKE`` env flag.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import os
+import sys
+
+import pytest
+
+# --- make the repo importable from any cwd --------------------------------
+ROOT = os.path.dirname(os.path.abspath(__file__))
+if ROOT not in sys.path:
+    sys.path.insert(0, ROOT)
+
+# An absolute path to the real config.yaml, so tests that chdir() (e.g. to a
+# tmp_path) still find it.
+CONFIG_PATH = os.path.join(ROOT, "config.yaml")
+
+from scraper.cache import RawCache                     # noqa: E402,F401
+from scraper.http_client import (                      # noqa: E402,F401
+    AntibotError,
+    FetchResult,
+    NotFoundError,
+    TransientError,
+)
+from scraper.ingest import Ingestor                    # noqa: E402,F401
+from scraper.request_log import RequestLog             # noqa: E402,F401
+
+
+# ---------------------------------------------------------------- shared fakes
+#
+# A fake transport that mimics httpx.Client.get() so the *real* HttpClient
+# retry loop runs with no network. Routed per-URL so a single test can exercise
+# 200 / 404 / 403 / 5xx behaviour.
+
+class FakeResponse:
+    def __init__(self, status, content, content_type="text/html"):
+        self.status_code = status
+        self.content = content
+        self.headers = {"content-type": content_type}
+
+
+class FakeTransport:
+    def __init__(self, route):
+        self.route = route       # callable(url) -> (status, content_bytes)
+        self.calls = []
+
+    def get(self, url, follow_redirects=False):
+        self.calls.append(url)
+        status, content = self.route(url)
+        return FakeResponse(status, content)
+
+    def close(self):
+        pass
+
+
+def make_ingestor(config_path, raw_dir, log_path, route=None):
+    """Build an Ingestor wired to a temp cache+log and (optionally) a fake
+    transport, with the backoff sleep neutralised so the test is instant.
+
+    Returns (ingestor, transport-or-None).  ``ing._sleeps`` records the backoff
+    delays that WOULD have been slept, so tests can assert exponential timing.
+    """
+    from pathlib import Path
+
+    ing = Ingestor(config_path=config_path, force=True)
+    ing.cache.raw_dir = Path(raw_dir)
+    ing.log.path = Path(log_path)
+    sleeps = []
+    ing.http.sleep = lambda d: sleeps.append(d)
+    ing._sleeps = sleeps
+    transport = None
+    if route is not None:
+        transport = FakeTransport(route)
+        ing.http._client = transport
+    return ing, transport
+
+
+class LogCapture(logging.Handler):
+    def __init__(self, logger):
+        super().__init__()
+        self.records = []
+        self._logger = logger
+        self._prev_level = logger.level
+        logger.addHandler(self)
+        logger.setLevel(logging.DEBUG)
+
+    def emit(self, record):
+         # A no-op emit that records every log message for later inspection.
+        self.records.append(record)
+
+    @property
+    def warnings(self):
+        return [r.getMessage() for r in self.records
+                if r.levelno >= logging.WARNING]
+
+    @property
+    def messages(self):
+        return [r.getMessage() for r in self.records]
+
+    def remove(self):
+        self._logger.removeHandler(self)
+        self._logger.setLevel(self._prev_level)
+
+
+# ---------------------------------------------------------------- pytest fixtures
+
+@pytest.fixture
+def logcap():
+    def _cap(name):
+        h = LogCapture(logging.getLogger(name))
+        return h
+     # No per-call teardown is required: LogCapture installs and uninstalls a
+     # handler on a module logger. Loggers are cached process-wide, so the
+     # handler lingers but is harmless (it just keeps appended records it was
+     # never asked about). For tests that must observe warnings, attach the
+     # capture at the start of the test and read `.warnings` before the next
+     # logcap() call.
+    return _cap
+
+
+def read_jsonl(path):
+    with open(path, encoding="utf-8") as fh:
+        return [json.loads(line) for line in fh if line.strip()]