126 lines
4.1 KiB
Python
126 lines
4.1 KiB
Python
"""Output sinks: JSON Lines and a flat SQLite table."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
from typing import List
|
|
|
|
from .schema import BillAction
|
|
|
|
log = logging.getLogger("scraper.io")
|
|
|
|
# Column order for the flat SQLite table: a stable, self-describing subset of
|
|
# the record fields. ``id`` is the content hash; text fields stay TEXT and the
|
|
# clearly-numeric fields use INTEGER.
|
|
_COLUMNS = [
|
|
("id", "TEXT"),
|
|
("date", "TEXT"),
|
|
("house", "TEXT"),
|
|
("source", "TEXT"),
|
|
("record_type", "TEXT"),
|
|
("bill_number", "TEXT"),
|
|
("bill_id", "TEXT"),
|
|
("session", "TEXT"),
|
|
("chamber_origin", "TEXT"),
|
|
("author", "TEXT"),
|
|
("short_title", "TEXT"),
|
|
("description", "TEXT"),
|
|
("committee", "TEXT"),
|
|
("hearing_time", "TEXT"),
|
|
("location", "TEXT"),
|
|
("item_number", "INTEGER"),
|
|
("supplement", "INTEGER"),
|
|
("ics_url", "TEXT"),
|
|
("agenda_id", "TEXT"),
|
|
("raw_selector", "TEXT"),
|
|
]
|
|
|
|
|
|
def _safe_ident(name: str) -> str:
|
|
return re.sub(r"[^0-9A-Za-z_]", "_", name or "_")
|
|
|
|
|
|
def _col_names():
|
|
return [_safe_ident(c) for c, _ in _COLUMNS]
|
|
|
|
|
|
def write_jsonl(records, path, fill_id=True, include_nulls=False):
|
|
"""Write records as JSON Lines (one JSON object per line).
|
|
|
|
Returns the number of records written. A record that cannot be serialised
|
|
is logged and skipped rather than aborting the batch.
|
|
"""
|
|
written = 0
|
|
os.makedirs(os.path.dirname(os.path.abspath(path)) or ".", exist_ok=True)
|
|
with open(path, "w", encoding="utf-8", newline="\n") as fh:
|
|
for rec in records:
|
|
try:
|
|
data = rec.to_dict(fill_id=fill_id, include_nulls=include_nulls)
|
|
line = json.dumps(data, ensure_ascii=False, sort_keys=True)
|
|
except Exception as e: # noqa: BLE001
|
|
log.warning("skip record that cannot be serialised: %s", e)
|
|
continue
|
|
fh.write(line)
|
|
fh.write("\n")
|
|
written += 1
|
|
log.info("wrote %d records to %s", written, path)
|
|
return written
|
|
|
|
|
|
def write_sqlite(records, path, table="daily_file_records",
|
|
create_if_missing=True):
|
|
"""Persist records to a flat SQLite table. Returns the number written.
|
|
|
|
Creates the table and database file when needed so the CLI ``--sqlite`` flag
|
|
is self-sufficient. Existing rows are not replaced (plain INSERT).
|
|
"""
|
|
import sqlite3
|
|
|
|
col_list = ", ".join(_col_names())
|
|
col_defs = ", ".join(f'{_safe_ident(c)} {t}' for c, t in _COLUMNS)
|
|
safe_table = _safe_ident(table)
|
|
|
|
os.makedirs(os.path.dirname(os.path.abspath(path)) or ".", exist_ok=True)
|
|
conn = sqlite3.connect(path)
|
|
try:
|
|
cur = conn.cursor()
|
|
if create_if_missing:
|
|
cur.execute(f"CREATE TABLE IF NOT EXISTS {safe_table} ({col_defs})")
|
|
cur.execute(f"PRAGMA table_info({safe_table})")
|
|
existing = {row[1] for row in cur.fetchall()}
|
|
used = [(_safe_ident(c), c) for c, _ in _COLUMNS if _safe_ident(c) in existing]
|
|
names = [s for s, _ in used]
|
|
placeholders = ", ".join(["?"] * len(names))
|
|
insert_cols = ", ".join(names)
|
|
|
|
written = 0
|
|
for i, rec in enumerate(records):
|
|
data = rec.to_dict(fill_id=True, include_nulls=False)
|
|
values = []
|
|
for safe, raw in used:
|
|
val = data.get(raw)
|
|
if val is None:
|
|
values.append(None)
|
|
elif safe in ("item_number", "supplement"):
|
|
try:
|
|
values.append(int(val))
|
|
except (TypeError, ValueError):
|
|
values.append(None)
|
|
else:
|
|
values.append(val)
|
|
try:
|
|
cur.execute(
|
|
f"INSERT INTO {safe_table} ({insert_cols}) VALUES ({placeholders})",
|
|
values)
|
|
written += 1
|
|
except Exception as e: # noqa: BLE001
|
|
log.warning("skip sqlite insert for record #%d: %s", i, e)
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
log.info("wrote %d rows to %s::%s", written, path, table)
|
|
return written
|