Initial commit

scripts/test_parse_acceptance.py

@@ -0,0 +1,172 @@
+"""Parser-lane acceptance tests for DAILY_FILE_PARSE_SPEC.md (task t_fb8081e7).
+
+Runs the parser core against the archived sample and against a deliberately
+malformed fixture, and asserts every acceptance criterion:
+   1. clean sample -> expected fields, no exceptions;
+   2. malformed input -> a bad row logs a warning and is skipped, batch survives;
+   3. bill_id codec correctness;
+   4. JSONL + SQLite sinks round-trip;
+   5. CLI (handle_parse) writes JSONL and returns 0.
+
+IMPORTANT: the shared ``scraper/__init__.py`` is jointly owned by the ingestion
+task (t_f595542f) and currently imports ``.ingest`` / ``.http_client`` which may
+be mid-write (transiently broken). To test the parser core in isolation, this
+script loads the parser-core modules as a standalone package via importlib, with
+a stub ``__init__`` that does NOT execute the sibling's code.
+Run a plain ``import scraper`` only once the ingestion layer is stably importable.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import logging
+import os
+import sys
+import types
+import traceback
+from argparse import Namespace
+
+# --- isolated package loader --------------------------------------------
+PKG = "scrpkr"
+pkgmod = types.ModuleType(PKG)
+pkgmod.__path__ = ["scraper"]
+sys.modules[PKG] = pkgmod
+
+
+def load(sub: str):
+    name = f"{PKG}.{sub}"
+    spec = importlib.util.spec_from_file_location(name, f"scraper/{sub}.py")
+    assert spec is not None and spec.loader is not None
+    mod = importlib.util.module_from_spec(spec)
+    sys.modules[name] = mod
+    spec.loader.exec_module(mod)
+    return mod
+
+
+parsers = load("parsers")
+io_mod = load("io")
+schema = load("schema")
+bill_id = load("bill_id")
+fixtures = load("test_fixtures")
+parse_cli = load("parse_cli")
+
+
+class _CapHandler(logging.Handler):
+    msgs = []
+
+    def emit(self, record):
+        self.msgs.append(record.getMessage())
+
+
+results = []
+
+
+def check(name, cond, detail=""):
+    results.append((name, bool(cond), detail))
+    print(("PASS" if cond else "FAIL"), name, "—", detail)
+
+
+# --- Test 1: clean sample --------------------------------------------------
+recs = parsers.parse("samples/assembly_dailyfile_latest.html")
+types_ct = {}
+for r in recs:
+    types_ct[r.record_type] = types_ct.get(r.record_type, 0) + 1
+check("clean: total records >= 156", len(recs) >= 156,
+      f"got {len(recs)} by type {types_ct}")
+
+bills = [r for r in recs if r.record_type == "bill"]
+sample = bills[0] if bills else None
+have_fields = sample is not None and all(
+    getattr(sample, f) is not None for f in
+    ["bill_number", "bill_id", "session", "short_title"])
+check("clean: first bill record has bill_number/bill_id/session/short_title",
+      have_fields,
+      f"bill={getattr(sample,'bill_number',None)!r} id={getattr(sample,'bill_id',None)!r} "
+      f"session={getattr(sample,'session',None)!r}")
+
+ab28 = next((r for r in bills if r.bill_number == "AB 28"), None)
+check("clean: AB 28 present with item_number 32 + action description",
+      ab28 is not None and ab28.item_number == 32 and "third time" in (ab28.description or ""),
+      f"found={'yes' if ab28 else 'no'}")
+
+# --- Test 2: malformed input ---------------------------------------------
+cap = _CapHandler()
+cap.setLevel(logging.WARNING)
+parsers.log.addHandler(cap)
+n_before = len(cap.msgs)
+try:
+    mrecs = parsers.parse_html(fixtures.malformed_document())
+    aborted = False
+except Exception:
+    traceback.print_exc()
+    aborted = True
+    mrecs = []
+new_warns = len(cap.msgs) - n_before
+check("malformed: no unhandled exception aborts batch", not aborted, "")
+check("malformed: bad rows logged as warnings", new_warns >= 2,
+      f"{new_warns} warning(s): {cap.msgs[n_before:n_before+3]}")
+bns = {r.bill_number for r in mrecs}
+check("malformed: valid records survived, bad ones dropped",
+      "AB 100" in bns and "AB 300" in bns and "AB 200 (no hyperlink here)" not in bns,
+      f"bill_numbers={sorted(bns)}")
+
+# --- Test 3: codec ---------------------------------------------------------
+dec = bill_id.parse_bill_id("202520260AB28")
+check("codec: 202520260AB28 -> AB 28 / 2025-2026 / assembly",
+      bool(dec) and dec["display"] == "AB 28" and dec["session"] == "2025-2026"
+      and dec["house"] == "assembly", str(dec))
+
+# --- Test 4: JSONL + SQLite round-trip ------------------------------------
+out_dir = "out"
+os.makedirs(out_dir, exist_ok=True)
+jsonl = os.path.join(out_dir, "records.jsonl")
+db = os.path.join(out_dir, "records.db")
+try:
+    n_jsonl = io_mod.write_jsonl(recs, jsonl)
+    n_db = io_mod.write_sqlite(recs, db)
+except Exception:
+    traceback.print_exc()
+    n_jsonl = n_db = -1
+check("io: JSONL wrote all records", n_jsonl == len(recs), f"jsonl={n_jsonl}/{len(recs)}")
+check("io: SQLite wrote all records", n_db == len(recs), f"db_rows={n_db}/{len(recs)}")
+ok_json = True
+line_count = 0
+with open(jsonl) as fh:
+    for line in fh:
+        line_count += 1
+        try:
+            json.loads(line)
+        except Exception:
+            ok_json = False
+check("io: JSONL is one valid JSON object per line",
+      ok_json and line_count == len(recs), f"lines={line_count}")
+
+# --- Test 5: CLI end-to-end ------------------------------------------------
+cli_args = Namespace(
+     input="samples/assembly_dailyfile_latest.html",
+     output=os.path.join(out_dir, "cli_records.jsonl"),
+     sqlite_path=None,
+     table="daily_file_records",
+     house="assembly",
+     source=schema.ASSEMBLY_DAILY_FILE,
+     no_hearings=False,
+     json_include_nulls=False,
+     parser="lxml",
+     verbose=0,
+   )
+rc = parse_cli.handle_parse(cli_args)
+cli_lines = 0
+with open(cli_args.output) as fh:
+    cli_lines = sum(1 for _ in fh)
+check("cli: handle_parse returns 0 and wrote JSONL",
+      rc == 0 and cli_lines == len(recs), f"rc={rc} lines={cli_lines}")
+
+# --- verdict ---------------------------------------------------------------
+failed = [n for n, ok, _ in results if not ok]
+print("\n==== SUMMARY ====")
+print(f"passed {len(results) - len(failed)}/{len(results)}")
+if failed:
+    print("FAILED:", failed)
+    sys.exit(1)
+print("ALL PARSER ACCEPTANCE TESTS PASSED")