Initial commit

scripts/_accept_tests.py

@@ -0,0 +1,143 @@
+"""Run the parser task acceptance tests through an ISOLATED package loader.
+
+The shared scraper/__init__.py currently imports the in-progress ingest layer
+(sibling task t_f595542f) and is transiently broken, so a plain `import scraper`
+fails. We therefore register a stub package `scrpkr` pointing at scraper/ and
+load only the parser-core modules, leaving the sibling's modules untouched.
+"""
+import importlib.util, sys, types, logging, json, os, traceback
+
+PKG = "scrpkr"
+pkgmod = types.ModuleType(PKG)
+pkgmod.__path__ = ["scraper"]
+sys.modules[PKG] = pkgmod
+
+def load(sub):
+    name = f"{PKG}.{sub}"
+    s = importlib.util.spec_from_file_location(name, f"scraper/{sub}.py")
+    m = importlib.util.module_from_spec(s)
+    sys.modules[name] = m
+    s.loader.exec_module(m)
+    return m
+
+parsers = load("parsers")
+io_mod = load("io")
+schema = load("schema")
+bill_id = load("bill_id")
+fixtures = load("test_fixtures")
+parse_cli = load("parse_cli")
+
+# Capture warnings to assert "logged warning".
+class CapHandler(logging.Handler):
+    def __init__(self):
+        super().__init__()
+        self.msgs = []
+    def emit(self, rec):
+        self.msgs.append(rec.getMessage())
+
+results = []
+def check(name, cond, detail=""):
+    results.append((name, bool(cond), detail))
+    print(("PASS" if cond else "FAIL"), name, "—", detail)
+
+# --- Test 1: clean sample -------------------------------------------------
+recs = parsers.parse("samples/assembly_dailyfile_latest.html")
+types_ct = {}
+for r in recs:
+    types_ct[r.record_type] = types_ct.get(r.record_type, 0) + 1
+check("clean: total records >= 156", len(recs) >= 156, f"got {len(recs)} by type {types_ct}")
+
+bills = [r for r in recs if r.record_type == "bill"]
+sample = bills[0]
+have_fields = all(getattr(sample, f) is not None for f in
+                   ["bill_number", "bill_id", "session", "short_title"])
+check("clean: first bill record has bill_number/bill_id/session/short_title",
+      have_fields, f"bill={getattr(sample,'bill_number',None)!r} id={getattr(sample,'bill_id',None)!r} session={getattr(sample,'session',None)!r}")
+# The spec's worked example AB28 / item 32 / "Read third time" should be present.
+ab28 = next((r for r in bills if r.bill_number == "AB 28"), None)
+check("clean: AB 28 present with item_number 32 + action description",
+      ab28 is not None and ab28.item_number == 32 and "third time" in (ab28.description or ""),
+      f"found={'yes' if ab28 else 'no'}")
+
+# --- Test 2: malformed row skip ------------------------------------------
+cap = CapHandler()
+cap.setLevel(logging.WARNING)
+parsers.log.addHandler(cap)
+cap_msgs_before = len(cap.msgs)
+try:
+    mrecs = parsers.parse_html(fixtures.malformed_document())
+    aborted = False
+except Exception:
+    aborted = True
+    mrecs = []
+new_warns = len(cap.msgs) - cap_msgs_before
+check("malformed: no unhandled exception aborts batch", not aborted, "")
+check("malformed: bad rows logged as warnings", new_warns >= 2,
+      f"{new_warns} warning(s): {cap.msgs[-3:]}")
+check("malformed: valid records still produced (batch survived)",
+      any(r.bill_number == "AB 100" for r in mrecs) and any(r.bill_number == "AB 300" for r in mrecs),
+      f"produced bill_numbers={[r.bill_number for r in mrecs]}")
+
+# --- Test 3: codec --------------------------------------------------------
+dec = bill_id.parse_bill_id("202520260AB28")
+check("codec: 202520260AB28 -> AB 28 / 2025-2026 / assembly",
+      dec and dec["display"] == "AB 28" and dec["session"] == "2025-2026" and dec["house"] == "assembly",
+      str(dec))
+
+# --- Test 4: JSONL + SQLite roundtrip, zero unhandled exceptions ----------
+out_dir = "out"
+os.makedirs(out_dir, exist_ok=True)
+jsonl = os.path.join(out_dir, "records.jsonl")
+db = os.path.join(out_dir, "records.db")
+ntests = 0
+try:
+    n_jsonl = io_mod.write_jsonl(recs, jsonl)
+    n_db = io_mod.write_sqlite(recs, db)
+except Exception:
+    traceback.print_exc()
+    n_jsonl = n_db = -1
+check("io: JSONL wrote all records", n_jsonl == len(recs), f"jsonl={n_jsonl}/{len(recs)}")
+check("io: SQLite wrote all records", n_db == len(recs), f"db_rows={n_db}/{len(recs)}")
+# JSONL is one JSON object per line
+ok_json = True
+line_count = 0
+with open(jsonl) as fh:
+    for line in fh:
+        line_count += 1
+        try:
+            json.loads(line)
+        except Exception:
+            ok_json = False
+check("io: JSONL is one valid JSON object per line", ok_json and line_count == len(recs),
+      f"lines={line_count}")
+
+# --- Test 5: CLI end-to-end (parse_cli.handle_parse) ----------------------
+import argparse
+class _A: pass
+cli_args = _A()
+cli_args.input = "samples/assembly_dailyfile_latest.html"
+cli_args.output = os.path.join(out_dir, "cli_records.jsonl")
+cli_args.sqlite_path = None
+cli_args.table = "daily_file_records"
+cli_args.house = "assembly"
+cli_args.source = schema.ASSEMBLY_DAILY_FILE
+cli_args.no_hearings = False
+cli_args.json_include_nulls = False
+cli_args.parser = "lxml"
+rc = parse_cli.handle_parse(cli_args)
+cli_jsonl = os.path.join(out_dir, "cli_records.jsonl")
+cli_lines = 0
+if os.path.exists(cli_jsonl):
+    with open(cli_jsonl) as fh:
+        cli_lines = sum(1 for _ in fh)
+check("cli: handle_parse returns 0 and wrote JSONL", rc == 0 and cli_lines == len(recs),
+      f"rc={rc} lines={cli_lines}")
+
+# --- verdict --------------------------------------------------------------
+failed = [n for n, ok, _ in results if not ok]
+print("\n==== SUMMARY ====")
+print(f"passed {len(results)-len(failed)}/{len(results)}")
+if failed:
+    print("FAILED:", failed)
+    sys.exit(1)
+print("ALL PARSER ACCEPTANCE TESTS PASSED")

scripts/_compile.py

@@ -0,0 +1,16 @@
+import py_compile, glob, sys
+ok = True
+for f in sorted(glob.glob("scraper/*.py")):
+    try:
+        py_compile.compile(f, doraise=True)
+        print("OK   ", f)
+    except py_compile.PyCompileError as e:
+        ok = False
+        print("FAIL", f)
+        print(e)
+print("---IMPORT---")
+import scraper
+print("import scraper OK, version", scraper.__version__)
+from scraper.parsers import parse, parse_html
+print("parse/parse_html callable:", callable(parse), callable(parse_html))
+sys.exit(0 if ok else 1)

scripts/_dbg.py

@@ -0,0 +1,23 @@
+from scraper.cli import build_parser
+# Build and try to invoke parse
+import argparse
+p = build_parser()
+try:
+    p.parse_args(["parse", "--input", "samples/assembly_dailyfile_latest.html", "--output", "out/_dbg.jsonl"])
+    print("parse subcommand EXISTS")
+except SystemExit:
+    pass
+subs = p._subparsers._name_map if hasattr(p, "_subparsers") else None
+print("registered subcommands:", list(subs.keys()) if subs else "none")
+
+# Direct test of register
+from scraper.parse_cli import register, handle_parse
+import argparse as a
+root = a.ArgumentParser()
+sub = root.add_subparsers(dest="command")
+try:
+    register(sub)
+    print("register() OK on fresh sub; name_map:", list(sub._name_map.keys()))
+except Exception as e:
+    import traceback; traceback.print_exc()
+    print("register FAILED:", e)

scripts/_dbg2.py

@@ -0,0 +1,14 @@
+import sys, pkgutil, traceback
+print("cwd sys.path[0]:", sys.path[0])
+print("cwd import path:", [p for p in sys.path])
+try:
+    import scraper
+    print("imported scraper from:", getattr(scraper, "__file__", "namespace"))
+    print("is scraper.cli importable?")
+    import scraper.cli
+    print("scraper.cli OK")
+    from scraper.parse_cli import register, handle_parse
+    print("parse_cli OK")
+except Exception as e:
+    print("IMPORT FAILED:", type(e).__name__, e)
+    traceback.print_exc()

scripts/_diag.py

@@ -0,0 +1,9 @@
+lines = open("scraper/io.py").read().split("\n")
+for i in range(47, 58):
+    if i < len(lines):
+        print(i + 1, repr(lines[i]))
+print("---")
+# also show the _safe_ident and _read_text region
+for i, ln in enumerate(lines, 1):
+    if "def _safe_ident" in ln or "PRAGMA" in ln:
+        print(i, repr(ln))

scripts/_env_probe.py

@@ -0,0 +1,8 @@
+import importlib, sys
+for mod in ["bs4", "lxml", "html5lib", "sqlitedict"]:
+    try:
+        m = importlib.import_module(mod)
+        print(f"OK   {mod} {getattr(m, '__version__', '')}")
+    except Exception as e:
+        print(f"MISS {mod}: {e}")
+print("python", sys.version)

scripts/_final_verify.py

@@ -0,0 +1,27 @@
+import sqlite3, subprocess, sys, os
+os.chdir("/Users/kyle/Documents/cali_scraper")
+
+# 1. SQLite table populated
+conn = sqlite3.connect("out/via_shared_entrypoint.db")
+cur = conn.cursor()
+cur.execute("SELECT count(*) FROM daily_file_records")
+print("sqlite rows in shared-entrypoint db:", cur.fetchone()[0])
+cur.execute("SELECT bill_number, bill_id, session, item_number FROM daily_file_records WHERE item_number=32 LIMIT 1")
+print("AB 28 / item 32 row:", cur.fetchone())
+conn.close()
+
+# 2. Sibling subcommands still work (no regression from my additive edits)
+for args in (["list"], ["check", "--date", "2026-09-02"]):
+    print("\n=== python -m scraper", " ".join(args), "===")
+    r = subprocess.run([sys.executable, "-m", "scraper"] + args,
+                       capture_output=True, text=True)
+    print("rc:", r.returncode)
+    print(r.stdout.strip()[:400])
+    if r.stderr.strip():
+        print("STDERR:", r.stderr.strip()[:200])
+
+# 3. help lists all four subcommands
+r = subprocess.run([sys.executable, "-m", "scraper", "--help"], capture_output=True, text=True)
+print("\n=== subcommands in help ===")
+for tok in ("ingest", "list", "check", "parse"):
+    print(f"  {tok:8} present:", tok in r.stdout)

scripts/_iso.py

@@ -0,0 +1,38 @@
+"""Verify parser-core modules load and work, as a real package but with a
+STUB __init__ (so we do NOT run the sibling's scraper/__init__.py, which
+imports the in-progress ingest layer and is currently broken).
+"""
+import importlib.util, sys, types, traceback
+
+PKG = "scrpkr"  # isolated namespace, does not touch sibling's `scraper`
+spec = importlib.util.spec_from_file_location(PKG, "scraper/__init__.py")
+# Build a stub package (no __init__ body executed) with submodules.
+pkgmod = types.ModuleType(PKG)
+pkgmod.__path__ = ["scraper"]
+sys.modules[PKG] = pkgmod
+
+def load(sub):
+    name = f"{PKG}.{sub}"
+    s = importlib.util.spec_from_file_location(name, f"scraper/{sub}.py")
+    m = importlib.util.module_from_spec(s)
+    sys.modules[name] = m
+    s.loader.exec_module(m)
+    return m
+
+try:
+    schema = load("schema")
+    bill_id = load("bill_id")
+    parsers = load("parsers")
+    io_mod = load("io")
+    cli = load("cli")
+    print("OK schema :", [n for n in dir(schema) if not n.startswith('_')][:6])
+    print("OK bill_id:", [n for n in dir(bill_id) if not n.startswith('_')][:6])
+    print("OK parsers:", [n for n in dir(parsers) if not n.startswith('_') and callable(getattr(parsers,n))][:12])
+    print("OK io     :", [n for n in dir(io_mod) if not n.startswith('_') and callable(getattr(io_mod,n))])
+    print("OK cli    :", callable(cli.main), "argparse subcommands ok")
+    dec = bill_id.parse_bill_id("202520260AB28")
+    print("codec 202520260AB28 ->", dec)
+    print("ALL CORE MODULES OK")
+except Exception:
+    traceback.print_exc()
+    sys.exit(1)

scripts/_probe.py

@@ -0,0 +1,25 @@
+import sys, os
+sys.path.insert(0, os.path.dirname(__file__))
+from bs4 import BeautifulSoup
+HTML = open("samples/assembly_dailyfile_latest.html", encoding="utf-8", errors="replace").read()
+soup = BeautifulSoup(HTML, "lxml")
+
+def count(sel):
+    return len(soup.select(sel))
+
+print("table classes found:")
+for t in soup.select("table"):
+    cls = t.get("class") or []
+    print("  table.", " ".join(cls), "-> rows:", len(t.select("tr")))
+print()
+for sel in [
+    "tr.committee-hearing-details",
+    "tr[class*='committee-hearing-details']",
+    "tr.floor-bill-details",
+    "tr.class='floor-bill-details'",
+    "tr[class*='committee-bills-details']",
+]:
+    try:
+        print(f"  {sel!r}: {count(sel.replace('class=', '') if False else sel)}")
+    except Exception as e:
+        print(f"  {sel!r}: ERROR {e}")

scripts/_probe_indent.py

@@ -0,0 +1,6 @@
+def f():
+    a = 1
+    b = 2
+    if a:
+        c = 3
+    return a + b + c

scripts/_verify_out.py

@@ -0,0 +1,23 @@
+import sqlite3, json, sys
+db = "out/cli_direct.db"
+conn = sqlite3.connect(db)
+cur = conn.cursor()
+cur.execute("SELECT count(*) FROM daily_file_records")
+print("sqlite rows:", cur.fetchone()[0])
+cur.execute("SELECT count(DISTINCT record_type), group_concat(record_type) FROM (SELECT DISTINCT record_type FROM daily_file_records)")
+print("record types:", cur.fetchone())
+cur.execute("SELECT count(*) FROM daily_file_records WHERE committee IS NOT NULL AND committee != ''")
+print("with committee ref:", cur.fetchone()[0])
+cur.execute("SELECT count(*) FROM daily_file_records WHERE record_type='hearing'")
+print("hearings:", cur.fetchone()[0])
+cur.execute("SELECT count(*) FROM daily_file_records WHERE description IS NOT NULL AND description != ''")
+print("non-empty descriptions:", cur.fetchone()[0])
+conn.close()
+
+# Show one hearing record and one committee-bill from the jsonl
+with open("out/cli_direct.jsonl") as fh:
+    hearing = next((json.loads(l) for l in fh if json.loads(l).get("record_type")=="hearing"), None)
+with open("out/cli_direct.jsonl") as fh:
+    cbill = next((json.loads(l) for l in fh if json.loads(l).get("committee") and json.loads(l).get("record_type")=="bill"), None)
+print("sample hearing record:", json.dumps(hearing, sort_keys=True) if hearing else "NONE")
+print("sample committee bill record:", json.dumps(cbill, sort_keys=True) if cbill else "NONE")

scripts/run_ingest.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# Cron-compatible wrapper for the California Daily File ingestion layer.
+#
+# Install (nightly at 02:00 PT -- see install notes below) with:
+#   crontab -e   ->
+#     0 2 * * *   /Users/kyle/Documents/cali_scraper/scripts/run_ingest.sh
+#
+# Behavior:
+#   * Runs `python -m scraper ingest` for BOTH chambers for "today" (local date),
+#     unless CALI_SCRAPER_DATE overrides it (YYYY-MM-DD).
+#   * --force is NOT used by default: a date already in the local raw cache is
+#     served from disk with zero network (the "re-parse doesn't re-hit" req).
+#   * Exit code is 0 on ok / cached / expected-404 (no publication that day),
+#     1 on a real failure (antibot / transient / unexpected 404) so cron mail
+#     fires only for problems worth paging on.
+#   * Every request is appended to cache/request_log.jsonl.
+
+set -uo pipefail
+
+# Resolve the repo root regardless of the cron caller's working directory.
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+cd "${ROOT}" || { echo "[run_ingest] cannot cd to ${ROOT}" >&2; exit 2; }
+
+# Pick the Python interpreter (prefer a project venv if present).
+PY="${PYTHON:-python3}"
+
+# Date to ingest: CALI_SCRAPER_DATE or today in the local (PT) timezone.
+DATE="${CALI_SCRAPER_DATE:-$(date +%F)}"
+
+# Force a network refresh even if cached? CALI_SCRAPER_FORCE=1 enables --force.
+FORCE=""
+if [ "${CALI_SCRAPER_FORCE:-0}" = "1" ]; then
+    FORCE="--force"
+fi
+
+echo "==================================================================="
+echo "[run_ingest] ${DATE}  root=${ROOT}  py=${PY}"
+echo "==================================================================="
+
+# -u: unbuffered; the CLI prints the human-readable summary.
+"${PY}" -u -m scraper ingest --date "${DATE}" ${FORCE}
+EXIT=$?
+
+case "${EXIT}" in
+     0) echo "[run_ingest] DONE  ${DATE}  (ok / cached / no publication -- all expected)" ;;
+     1) echo "[run_ingest] FAILED ${DATE}  see cache/request_log.jsonl for the attempt trail" >&2 ;;
+    *)  echo "[run_ingest] ERROR  ${DATE}  (exit ${EXIT})" >&2 ;;
+esac
+
+exit "${EXIT}"

scripts/test_ingest_offline.py

@@ -0,0 +1,157 @@
+"""Deterministic acceptance test for scraper.ingest -- no real network.
+
+Uses a fake HttpClient and an in-process RawCache (cache/raw) so every path is
+proven without depending on the live servers:
+   1. Happy path: assembly HTML download + cache miss -> ok
+   2. Cache hit on re-run -> no network, cache_hit=True
+   3. 404 EXPECTED (Senate non-legislative day) -> 'not_found', not a crash
+   4. 403 antibot -> 'antibot' status + alert, no crash
+   5. 5xx transient -> retries (3 attempts) -> 'transient' + alert
+   6. Cache-hit on the 404-affected source is NOT persisted (404s are not cached)
+"""
+
+from __future__ import annotations
+
+import time
+import tempfile
+import os
+import sys
+
+# Make the repo importable when run from anywhere.
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, ROOT)
+
+from scraper.ingest import Ingestor  # noqa: E402
+from scraper.http_client import (  # noqa: E402
+    AntibotError,
+    FetchResult,
+    NotFoundError,
+    TransientError,
+)
+from scraper.request_log import RequestLog  # noqa: E402
+
+
+PASS = []
+FAIL = []
+
+
+def check(name, cond, detail=""):
+    (PASS if cond else FAIL).append(name)
+    print(("  PASS  " if cond else "  FAIL  ") + name +
+          (f"   -- {detail}" if detail and not cond else ""))
+
+
+# --- Fake transport: mimics httpx.Client.get() so the REAL retry loop runs ---
+class FakeResponse:
+    def __init__(self, status, content, content_type="text/html"):
+        self.status_code = status
+        self.content = content
+        self.headers = {"content-type": content_type}
+
+
+class FakeTransport:
+         # Injected as Ingestor.http._client. The real HttpClient.get() runs its
+         # own retry/backoff loop on top of this, so transient/404/403 paths all
+         # exercise the real code. route(url) -> (status, content).
+    def __init__(self, route):
+        self.route = route
+        self.calls = []
+
+    def get(self, url, follow_redirects=False):
+        self.calls.append(url)
+        status, content = self.route(url)
+        return FakeResponse(status, content)
+
+    def close(self):
+        pass
+
+
+def inject_transport(ing, route, sleep_spy):
+         # Use the REAL HttpClient (real retry loop) but fake the underlying
+         # transport so no real network happens. No-op the backoff sleep so the
+         # test is instant but still records the *number* of sleeps.
+    ing.http._client = FakeTransport(route)
+    ing.http.sleep = lambda d: sleep_spy.append(d)
+    return ing.http._client
+
+
+def main():
+    from pathlib import Path
+    tmp = tempfile.mkdtemp(prefix="scraper_test_")
+
+    def fresh(cache_sub, log_name):
+          # Build an Ingestor wired to a real cache/log in a temp dir.
+        ing = Ingestor(config_path=os.path.join(ROOT, "config.yaml"))
+        ing.cache.raw_dir = Path(tmp) / cache_sub
+        ing.log.path = Path(tmp) / log_name
+        return ing
+
+    print("== Test 1: assembly happy path (cache miss -> ok) ==")
+    ing1 = fresh("raw1", "log1.jsonl")
+    tp1 = inject_transport(ing1, lambda url: (200, b"<html>ASM BODY</html>"), [])
+    res = ing1.ingest("2026-09-02", source="assembly_daily_file")
+    c = res.chambers[0]
+    check("t1 status ok", c.status == "ok", c.status)
+    check("t1 wrote bytes", c.bytes == len(b"<html>ASM BODY</html>"))
+    check("t1 not cached", c.cache_hit is False)
+    check("t1 one network call", len(tp1.calls) == 1, str(tp1.calls))
+    check("t1 all_ok", res.all_ok is True)
+
+    print("== Test 2: cache hit on re-run (no network) ==")
+    ing2 = fresh("raw1", "log2.jsonl")     # same cache dir as test 1
+    tp2 = inject_transport(ing2, lambda url: (_ for _ in ()).throw(
+         AssertionError("network SHOULD NOT be called on a cache hit")), [])
+    res2 = ing2.ingest("2026-09-02", source="assembly_daily_file")
+    c2 = res2.chambers[0]
+    check("t2 status cached", c2.status == "cached", c2.status)
+    check("t2 cache_hit True", c2.cache_hit is True)
+    check("t2 no network call", len(tp2.calls) == 0, str(tp2.calls))
+
+    print("== Test 3: expected 404 (Senate non-legislative day) ==")
+    ing3 = fresh("raw3", "log3.jsonl")
+    inject_transport(ing3, lambda url: (404, b""), [])
+    res3 = ing3.ingest("2026-09-04", source="senate_daily_file")
+    c3 = res3.chambers[0]
+    check("t3 not a crash", True)
+    check("t3 status not_found", c3.status == "not_found", c3.status)
+    check("t3 expected note", "expected" in c3.note, c3.note)
+
+    print("== Test 4: antibot 403 -> 'antibot' status, no crash ==")
+    ing4 = fresh("raw4", "log4.jsonl")
+    inject_transport(ing4, lambda url: (403, b"<html>antibot</html>"), [])
+    res4 = ing4.ingest("2026-09-02", source="senate_daily_file")
+    c4 = res4.chambers[0]
+    check("t4 status antibot", c4.status == "antibot", c4.status)
+    check("t4 not all_ok", res4.all_ok is False)
+
+    print("== Test 5: 5xx transient retries then gives up ==")
+    sleeps5 = []
+    ing5 = fresh("raw5", "log5.jsonl")
+    tp5 = inject_transport(ing5, lambda url: (503, b"busy"), sleeps5)
+    res5 = ing5.ingest("2026-09-02", source="assembly_daily_file")
+    c5 = res5.chambers[0]
+    check("t5 status transient", c5.status == "transient", c5.status)
+    check("t5 retried all 3 attempts", len(tp5.calls) == 3, str(len(tp5.calls)))
+    check("t5 gave up after max_retries", c5.attempts == 3, str(c5.attempts))
+    check("t5 backed off 2x (3 tries -> 2 sleeps)", len(sleeps5) == 2,
+           str(sleeps5))
+    check("t5 exponential backoff (1s,2s)", sleeps5 == [1.0, 2.0], str(sleeps5))
+
+    print("== Test 6: log records every attempt ==")
+    log5 = RequestLog.read(Path(tmp) / "log5.jsonl")
+    check("t6 attempts logged", len(log5) >= 3, f"{len(log5)} lines")
+    outcomes = [e.get("outcome") for e in log5]
+    check("t6 retry outcomes recorded", "5xx_retrying" in outcomes,
+           str(outcomes))
+
+    print()
+    print(f"PASSED {len(PASS)} / {len(PASS)+len(FAIL)}")
+    if FAIL:
+        print("FAILED:", FAIL)
+        sys.exit(1)
+    print("ALL DETERMINISTIC TESTS PASSED")
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

scripts/test_parse_acceptance.py

@@ -0,0 +1,172 @@
+"""Parser-lane acceptance tests for DAILY_FILE_PARSE_SPEC.md (task t_fb8081e7).
+
+Runs the parser core against the archived sample and against a deliberately
+malformed fixture, and asserts every acceptance criterion:
+   1. clean sample -> expected fields, no exceptions;
+   2. malformed input -> a bad row logs a warning and is skipped, batch survives;
+   3. bill_id codec correctness;
+   4. JSONL + SQLite sinks round-trip;
+   5. CLI (handle_parse) writes JSONL and returns 0.
+
+IMPORTANT: the shared ``scraper/__init__.py`` is jointly owned by the ingestion
+task (t_f595542f) and currently imports ``.ingest`` / ``.http_client`` which may
+be mid-write (transiently broken). To test the parser core in isolation, this
+script loads the parser-core modules as a standalone package via importlib, with
+a stub ``__init__`` that does NOT execute the sibling's code.
+Run a plain ``import scraper`` only once the ingestion layer is stably importable.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import logging
+import os
+import sys
+import types
+import traceback
+from argparse import Namespace
+
+# --- isolated package loader --------------------------------------------
+PKG = "scrpkr"
+pkgmod = types.ModuleType(PKG)
+pkgmod.__path__ = ["scraper"]
+sys.modules[PKG] = pkgmod
+
+
+def load(sub: str):
+    name = f"{PKG}.{sub}"
+    spec = importlib.util.spec_from_file_location(name, f"scraper/{sub}.py")
+    assert spec is not None and spec.loader is not None
+    mod = importlib.util.module_from_spec(spec)
+    sys.modules[name] = mod
+    spec.loader.exec_module(mod)
+    return mod
+
+
+parsers = load("parsers")
+io_mod = load("io")
+schema = load("schema")
+bill_id = load("bill_id")
+fixtures = load("test_fixtures")
+parse_cli = load("parse_cli")
+
+
+class _CapHandler(logging.Handler):
+    msgs = []
+
+    def emit(self, record):
+        self.msgs.append(record.getMessage())
+
+
+results = []
+
+
+def check(name, cond, detail=""):
+    results.append((name, bool(cond), detail))
+    print(("PASS" if cond else "FAIL"), name, "—", detail)
+
+
+# --- Test 1: clean sample --------------------------------------------------
+recs = parsers.parse("samples/assembly_dailyfile_latest.html")
+types_ct = {}
+for r in recs:
+    types_ct[r.record_type] = types_ct.get(r.record_type, 0) + 1
+check("clean: total records >= 156", len(recs) >= 156,
+      f"got {len(recs)} by type {types_ct}")
+
+bills = [r for r in recs if r.record_type == "bill"]
+sample = bills[0] if bills else None
+have_fields = sample is not None and all(
+    getattr(sample, f) is not None for f in
+    ["bill_number", "bill_id", "session", "short_title"])
+check("clean: first bill record has bill_number/bill_id/session/short_title",
+      have_fields,
+      f"bill={getattr(sample,'bill_number',None)!r} id={getattr(sample,'bill_id',None)!r} "
+      f"session={getattr(sample,'session',None)!r}")
+
+ab28 = next((r for r in bills if r.bill_number == "AB 28"), None)
+check("clean: AB 28 present with item_number 32 + action description",
+      ab28 is not None and ab28.item_number == 32 and "third time" in (ab28.description or ""),
+      f"found={'yes' if ab28 else 'no'}")
+
+# --- Test 2: malformed input ---------------------------------------------
+cap = _CapHandler()
+cap.setLevel(logging.WARNING)
+parsers.log.addHandler(cap)
+n_before = len(cap.msgs)
+try:
+    mrecs = parsers.parse_html(fixtures.malformed_document())
+    aborted = False
+except Exception:
+    traceback.print_exc()
+    aborted = True
+    mrecs = []
+new_warns = len(cap.msgs) - n_before
+check("malformed: no unhandled exception aborts batch", not aborted, "")
+check("malformed: bad rows logged as warnings", new_warns >= 2,
+      f"{new_warns} warning(s): {cap.msgs[n_before:n_before+3]}")
+bns = {r.bill_number for r in mrecs}
+check("malformed: valid records survived, bad ones dropped",
+      "AB 100" in bns and "AB 300" in bns and "AB 200 (no hyperlink here)" not in bns,
+      f"bill_numbers={sorted(bns)}")
+
+# --- Test 3: codec ---------------------------------------------------------
+dec = bill_id.parse_bill_id("202520260AB28")
+check("codec: 202520260AB28 -> AB 28 / 2025-2026 / assembly",
+      bool(dec) and dec["display"] == "AB 28" and dec["session"] == "2025-2026"
+      and dec["house"] == "assembly", str(dec))
+
+# --- Test 4: JSONL + SQLite round-trip ------------------------------------
+out_dir = "out"
+os.makedirs(out_dir, exist_ok=True)
+jsonl = os.path.join(out_dir, "records.jsonl")
+db = os.path.join(out_dir, "records.db")
+try:
+    n_jsonl = io_mod.write_jsonl(recs, jsonl)
+    n_db = io_mod.write_sqlite(recs, db)
+except Exception:
+    traceback.print_exc()
+    n_jsonl = n_db = -1
+check("io: JSONL wrote all records", n_jsonl == len(recs), f"jsonl={n_jsonl}/{len(recs)}")
+check("io: SQLite wrote all records", n_db == len(recs), f"db_rows={n_db}/{len(recs)}")
+ok_json = True
+line_count = 0
+with open(jsonl) as fh:
+    for line in fh:
+        line_count += 1
+        try:
+            json.loads(line)
+        except Exception:
+            ok_json = False
+check("io: JSONL is one valid JSON object per line",
+      ok_json and line_count == len(recs), f"lines={line_count}")
+
+# --- Test 5: CLI end-to-end ------------------------------------------------
+cli_args = Namespace(
+     input="samples/assembly_dailyfile_latest.html",
+     output=os.path.join(out_dir, "cli_records.jsonl"),
+     sqlite_path=None,
+     table="daily_file_records",
+     house="assembly",
+     source=schema.ASSEMBLY_DAILY_FILE,
+     no_hearings=False,
+     json_include_nulls=False,
+     parser="lxml",
+     verbose=0,
+   )
+rc = parse_cli.handle_parse(cli_args)
+cli_lines = 0
+with open(cli_args.output) as fh:
+    cli_lines = sum(1 for _ in fh)
+check("cli: handle_parse returns 0 and wrote JSONL",
+      rc == 0 and cli_lines == len(recs), f"rc={rc} lines={cli_lines}")
+
+# --- verdict ---------------------------------------------------------------
+failed = [n for n, ok, _ in results if not ok]
+print("\n==== SUMMARY ====")
+print(f"passed {len(results) - len(failed)}/{len(results)}")
+if failed:
+    print("FAILED:", failed)
+    sys.exit(1)
+print("ALL PARSER ACCEPTANCE TESTS PASSED")

scripts/verify_samples.py

@@ -0,0 +1,74 @@
+# Verification probe: parse the archived sample with the spec's CSS selectors
+# and confirm the selectors actually extract the expected fields.
+import re
+import sys, os
+
+SAMPLES = os.path.join(os.path.dirname(__file__), "..")
+sys.path.insert(0, SAMPLES)
+
+HTML = open("samples/assembly_dailyfile_latest.html").read()
+
+if not HTML:
+    print("ERROR: sample file not found", file=sys.stderr); sys.exit(1)
+
+try:
+    from bs4 import BeautifulSoup
+except ImportError:
+    print("bs4 not available; using regex fallback"); BS4=False
+else:
+    BS4=True
+
+# 1. Committee hearings
+hearings = re.findall(r"<tr[^>]*committee-hearing-details[^>]*>.*?</tr>", HTML, flags=re.S)
+print(f"\n[hearings] found {len(hearings)} rows (expected >=1)")
+if hearings:
+    m = hearings[0]
+    date = re.search(r'committee_hearing-date"[^>]*data-sort="([^"]+)"', m)
+    comm = re.search(r'committee_hearing-name"[^>]*data-sort="([^"]+)"', m)
+    loc  = re.search(r'committee_hearing-location"[^>]*data-sort="([^"]+)"', m)
+    print(f"  date={date.group(1) if date else '?'}, "
+          f"committee={comm.group(1) if comm else '?'}, "
+          f"location={loc.group(1) if loc else '?'}")
+
+# 2. Floor bills
+floor = re.findall(r"<tr[^>]*class=\"floor-bill-details\".*?</tr>", HTML, flags=re.S)
+print(f"\n[floor-bills] found {len(floor)} rows (expected >=10)")
+bills = []
+for row in floor[:5]:
+    h = re.search(r'floor_bill-measure".*?href="([^"]+)"', row)
+    a = re.search(r'floor_bill-author"[^>]*data-sort="([^"]+)"', row)
+    s = re.search(r'floor_bill-subject">([^<]+)<', row)
+    i = re.search(r'floor_bill-item"[^>]*data-sort="([^"]+)"', row)
+    if h:
+        url = h.group(1)
+        m = re.search(r'bill_id=(\d{8})\d([ABSHCRJS]+)(\d+)', url)
+        if m:
+            bills.append({"session":m.group(1), "chamber":m.group(2), "number":m.group(3),
+                          "author":a.group(1) if a else '?',
+                          "item":i.group(1) if i else '?'})
+if bills:
+    print("  first parsed floor bill:", bills[0])
+    # verify bill_id pattern
+    test_id = "202520260AB28"
+    m = re.match(r'^(\d{8})\d([A-Z]{2,4})(\d+)$', test_id)
+    if m:
+        print(f"  bill_id regex test on '{test_id}': session={m.group(1)}, chamber={m.group(2)}, number={m.group(3)}")
+    else:
+        print("  bill_id regex test FAILED on", test_id)
+
+# 3. Committee bills
+cb = re.findall(r"<tr[^>]*committee-bills-details.*?</tr>", HTML, flags=re.S)
+print(f"\n[committee-bills] found {len(cb)} rows (expected >=1)")
+if cb:
+    h = re.search(r'committee_bill-number".*?href="([^"]+)"', cb[0])
+    c = re.search(r'committee_bill-committee"[^>]*data-sort="([^"]+)"', cb[0])
+    print(f"  committee={c.group(1) if c else '?'}, bill_url={h.group(1)[:80]}…" if h else "  committee found, no bill_url")
+
+# 4. Senate index
+SENATA = open("samples/senate_dailyfile_index.html").read()
+srefs = re.findall(r'/dailyfile/publications/download/([^"<> ]+)', SENATA)
+print(f"\n[senate pdf index] found {len(srefs)} pdf refs: {srefs[:3]}…")
+
+# Check data-sort usage
+print(f"\n[data-sort usage] count in assembly: {HTML.count('data-sort')}")
+print("All probes passed.")