cali_scraper/scripts/test_parse_acceptance.py

"""Parser-lane acceptance tests for DAILY_FILE_PARSE_SPEC.md (task t_fb8081e7).

Runs the parser core against the archived sample and against a deliberately
malformed fixture, and asserts every acceptance criterion:
   1. clean sample -> expected fields, no exceptions;
   2. malformed input -> a bad row logs a warning and is skipped, batch survives;
   3. bill_id codec correctness;
   4. JSONL + SQLite sinks round-trip;
   5. CLI (handle_parse) writes JSONL and returns 0.

IMPORTANT: the shared ``scraper/__init__.py`` is jointly owned by the ingestion
task (t_f595542f) and currently imports ``.ingest`` / ``.http_client`` which may
be mid-write (transiently broken). To test the parser core in isolation, this
script loads the parser-core modules as a standalone package via importlib, with
a stub ``__init__`` that does NOT execute the sibling's code.
Run a plain ``import scraper`` only once the ingestion layer is stably importable.
"""

from __future__ import annotations

import importlib.util
import json
import logging
import os
import sys
import types
import traceback
from argparse import Namespace

# --- isolated package loader --------------------------------------------
PKG = "scrpkr"
pkgmod = types.ModuleType(PKG)
pkgmod.__path__ = ["scraper"]
sys.modules[PKG] = pkgmod


def load(sub: str):
    name = f"{PKG}.{sub}"
    spec = importlib.util.spec_from_file_location(name, f"scraper/{sub}.py")
    assert spec is not None and spec.loader is not None
    mod = importlib.util.module_from_spec(spec)
    sys.modules[name] = mod
    spec.loader.exec_module(mod)
    return mod


parsers = load("parsers")
io_mod = load("io")
schema = load("schema")
bill_id = load("bill_id")
fixtures = load("test_fixtures")
parse_cli = load("parse_cli")


class _CapHandler(logging.Handler):
    msgs = []

    def emit(self, record):
        self.msgs.append(record.getMessage())


results = []


def check(name, cond, detail=""):
    results.append((name, bool(cond), detail))
    print(("PASS" if cond else "FAIL"), name, "—", detail)


# --- Test 1: clean sample --------------------------------------------------
recs = parsers.parse("samples/assembly_dailyfile_latest.html")
types_ct = {}
for r in recs:
    types_ct[r.record_type] = types_ct.get(r.record_type, 0) + 1
check("clean: total records >= 156", len(recs) >= 156,
      f"got {len(recs)} by type {types_ct}")

bills = [r for r in recs if r.record_type == "bill"]
sample = bills[0] if bills else None
have_fields = sample is not None and all(
    getattr(sample, f) is not None for f in
    ["bill_number", "bill_id", "session", "short_title"])
check("clean: first bill record has bill_number/bill_id/session/short_title",
      have_fields,
      f"bill={getattr(sample,'bill_number',None)!r} id={getattr(sample,'bill_id',None)!r} "
      f"session={getattr(sample,'session',None)!r}")

ab28 = next((r for r in bills if r.bill_number == "AB 28"), None)
check("clean: AB 28 present with item_number 32 + action description",
      ab28 is not None and ab28.item_number == 32 and "third time" in (ab28.description or ""),
      f"found={'yes' if ab28 else 'no'}")

# --- Test 2: malformed input ---------------------------------------------
cap = _CapHandler()
cap.setLevel(logging.WARNING)
parsers.log.addHandler(cap)
n_before = len(cap.msgs)
try:
    mrecs = parsers.parse_html(fixtures.malformed_document())
    aborted = False
except Exception:
    traceback.print_exc()
    aborted = True
    mrecs = []
new_warns = len(cap.msgs) - n_before
check("malformed: no unhandled exception aborts batch", not aborted, "")
check("malformed: bad rows logged as warnings", new_warns >= 2,
      f"{new_warns} warning(s): {cap.msgs[n_before:n_before+3]}")
bns = {r.bill_number for r in mrecs}
check("malformed: valid records survived, bad ones dropped",
      "AB 100" in bns and "AB 300" in bns and "AB 200 (no hyperlink here)" not in bns,
      f"bill_numbers={sorted(bns)}")

# --- Test 3: codec ---------------------------------------------------------
dec = bill_id.parse_bill_id("202520260AB28")
check("codec: 202520260AB28 -> AB 28 / 2025-2026 / assembly",
      bool(dec) and dec["display"] == "AB 28" and dec["session"] == "2025-2026"
      and dec["house"] == "assembly", str(dec))

# --- Test 4: JSONL + SQLite round-trip ------------------------------------
out_dir = "out"
os.makedirs(out_dir, exist_ok=True)
jsonl = os.path.join(out_dir, "records.jsonl")
db = os.path.join(out_dir, "records.db")
try:
    n_jsonl = io_mod.write_jsonl(recs, jsonl)
    n_db = io_mod.write_sqlite(recs, db)
except Exception:
    traceback.print_exc()
    n_jsonl = n_db = -1
check("io: JSONL wrote all records", n_jsonl == len(recs), f"jsonl={n_jsonl}/{len(recs)}")
check("io: SQLite wrote all records", n_db == len(recs), f"db_rows={n_db}/{len(recs)}")
ok_json = True
line_count = 0
with open(jsonl) as fh:
    for line in fh:
        line_count += 1
        try:
            json.loads(line)
        except Exception:
            ok_json = False
check("io: JSONL is one valid JSON object per line",
      ok_json and line_count == len(recs), f"lines={line_count}")

# --- Test 5: CLI end-to-end ------------------------------------------------
cli_args = Namespace(
     input="samples/assembly_dailyfile_latest.html",
     output=os.path.join(out_dir, "cli_records.jsonl"),
     sqlite_path=None,
     table="daily_file_records",
     house="assembly",
     source=schema.ASSEMBLY_DAILY_FILE,
     no_hearings=False,
     json_include_nulls=False,
     parser="lxml",
     verbose=0,
   )
rc = parse_cli.handle_parse(cli_args)
cli_lines = 0
with open(cli_args.output) as fh:
    cli_lines = sum(1 for _ in fh)
check("cli: handle_parse returns 0 and wrote JSONL",
      rc == 0 and cli_lines == len(recs), f"rc={rc} lines={cli_lines}")

# --- verdict ---------------------------------------------------------------
failed = [n for n, ok, _ in results if not ok]
print("\n==== SUMMARY ====")
print(f"passed {len(results) - len(failed)}/{len(results)}")
if failed:
    print("FAILED:", failed)
    sys.exit(1)
print("ALL PARSER ACCEPTANCE TESTS PASSED")