Initial commit

scraper/io.py

@@ -0,0 +1,125 @@
+"""Output sinks: JSON Lines and a flat SQLite table."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import re
+from typing import List
+
+from .schema import BillAction
+
+log = logging.getLogger("scraper.io")
+
+# Column order for the flat SQLite table: a stable, self-describing subset of
+# the record fields. ``id`` is the content hash; text fields stay TEXT and the
+# clearly-numeric fields use INTEGER.
+_COLUMNS = [
+     ("id", "TEXT"),
+     ("date", "TEXT"),
+     ("house", "TEXT"),
+     ("source", "TEXT"),
+     ("record_type", "TEXT"),
+     ("bill_number", "TEXT"),
+     ("bill_id", "TEXT"),
+     ("session", "TEXT"),
+     ("chamber_origin", "TEXT"),
+     ("author", "TEXT"),
+     ("short_title", "TEXT"),
+     ("description", "TEXT"),
+     ("committee", "TEXT"),
+     ("hearing_time", "TEXT"),
+     ("location", "TEXT"),
+     ("item_number", "INTEGER"),
+     ("supplement", "INTEGER"),
+     ("ics_url", "TEXT"),
+     ("agenda_id", "TEXT"),
+     ("raw_selector", "TEXT"),
+]
+
+
+def _safe_ident(name: str) -> str:
+    return re.sub(r"[^0-9A-Za-z_]", "_", name or "_")
+
+
+def _col_names():
+    return [_safe_ident(c) for c, _ in _COLUMNS]
+
+
+def write_jsonl(records, path, fill_id=True, include_nulls=False):
+    """Write records as JSON Lines (one JSON object per line).
+
+    Returns the number of records written. A record that cannot be serialised
+    is logged and skipped rather than aborting the batch.
+    """
+    written = 0
+    os.makedirs(os.path.dirname(os.path.abspath(path)) or ".", exist_ok=True)
+    with open(path, "w", encoding="utf-8", newline="\n") as fh:
+        for rec in records:
+            try:
+                data = rec.to_dict(fill_id=fill_id, include_nulls=include_nulls)
+                line = json.dumps(data, ensure_ascii=False, sort_keys=True)
+            except Exception as e:  # noqa: BLE001
+                log.warning("skip record that cannot be serialised: %s", e)
+                continue
+            fh.write(line)
+            fh.write("\n")
+            written += 1
+    log.info("wrote %d records to %s", written, path)
+    return written
+
+
+def write_sqlite(records, path, table="daily_file_records",
+                 create_if_missing=True):
+    """Persist records to a flat SQLite table. Returns the number written.
+
+    Creates the table and database file when needed so the CLI ``--sqlite`` flag
+    is self-sufficient. Existing rows are not replaced (plain INSERT).
+    """
+    import sqlite3
+
+    col_list = ", ".join(_col_names())
+    col_defs = ", ".join(f'{_safe_ident(c)} {t}' for c, t in _COLUMNS)
+    safe_table = _safe_ident(table)
+
+    os.makedirs(os.path.dirname(os.path.abspath(path)) or ".", exist_ok=True)
+    conn = sqlite3.connect(path)
+    try:
+        cur = conn.cursor()
+        if create_if_missing:
+            cur.execute(f"CREATE TABLE IF NOT EXISTS {safe_table} ({col_defs})")
+        cur.execute(f"PRAGMA table_info({safe_table})")
+        existing = {row[1] for row in cur.fetchall()}
+        used = [(_safe_ident(c), c) for c, _ in _COLUMNS if _safe_ident(c) in existing]
+        names = [s for s, _ in used]
+        placeholders = ", ".join(["?"] * len(names))
+        insert_cols = ", ".join(names)
+
+        written = 0
+        for i, rec in enumerate(records):
+            data = rec.to_dict(fill_id=True, include_nulls=False)
+            values = []
+            for safe, raw in used:
+                val = data.get(raw)
+                if val is None:
+                    values.append(None)
+                elif safe in ("item_number", "supplement"):
+                    try:
+                        values.append(int(val))
+                    except (TypeError, ValueError):
+                        values.append(None)
+                else:
+                    values.append(val)
+            try:
+                cur.execute(
+                  f"INSERT INTO {safe_table} ({insert_cols}) VALUES ({placeholders})",
+                    values)
+                written += 1
+            except Exception as e:  # noqa: BLE001
+                log.warning("skip sqlite insert for record #%d: %s", i, e)
+        conn.commit()
+    finally:
+        conn.close()
+    log.info("wrote %d rows to %s::%s", written, path, table)
+    return written