Initial commit

scraper/test_fixtures.py

@@ -0,0 +1,75 @@
+"""Deliberately-malformed daily-file fragment used to prove the parser skips
+bad rows (logged warning, no batch abort) and survives encoding issues.
+
+Three problem rows are embedded:
+  R1  a floor-bill row with NO bill link / no number        -> must skip
+  R2  a floor-bill row whose item cell is non-numeric garbage-> must skip
+  R3  a hearing row with no committee AND no date           -> must skip
+The surrounding rows are valid, so the batch must still yield all valid ones.
+"""
+
+VALID = (
+    '<table class="floor-bills-table">'
+      '<tr class="floor-bill-details">'
+        '<td class="floor_bill-measure" data-sort="AB 00100">'
+        '<a href="https://leginfo.legislature.ca.gov/faces/billNavClient.xhtml?bill_id=202520260AB100">AB 100</a></td>'
+        '<td class="floor_bill-author" data-sort="Smith">Smith</td>'
+        '<td class="floor_bill-subject">An act to do X.</td>'
+        '<td class="floor_bill-location" data-sort="State Capitol">State Capitol</td>'
+        '<td class="floor_bill-item" data-sort="0000001">1</td>'
+        '<td class="floor_bill-view-details">Read third time. </td>'
+      '</tr>'
+    '</table>'
+)
+
+BAD_NO_LINK = (
+    '<table class="floor-bills-table">'
+      '<tr class="floor-bill-details">'
+        '<td class="floor_bill-measure" data-sort="AB 00200">'
+        'AB 200 (no hyperlink here)</td>'
+        '<td class="floor_bill-author" data-sort="Jones">Jones</td>'
+        '<td class="floor_bill-subject">Missing bill link row.</td>'
+        '<td class="floor_bill-item" data-sort="0000002">2</td>'
+      '</tr>'
+    '</table>'
+)
+
+BAD_ITEM = (
+    '<table class="floor-bills-table">'
+      '<tr class="floor-bill-details">'
+        '<td class="floor_bill-measure" data-sort="AB 00300">'
+        '<a href="https://leginfo.legislature.ca.gov/faces/billNavClient.xhtml?bill_id=202520260AB300">AB 300</a></td>'
+        '<td class="floor_bill-author" data-sort="Lee">Lee</td>'
+        '<td class="floor_bill-subject">Non-numeric item row.</td>'
+        '<td class="floor_bill-item">not-a-number</td>'
+      '</tr>'
+    '</table>'
+)
+
+BAD_HEARING = (
+    '<table class="committee-hearing-table">'
+      '<tr class="committee-hearing-details">'
+        '<td class="committee_hearing-time">9:00am</td>'
+        '<td class="committee_hearing-menu">no agenda link</td>'
+      '</tr>'
+    '</table>'
+)
+
+def malformed_document() -> str:
+    """A concatenated fragment: valid, bad, bad, bad rows."""
+    return VALID + BAD_NO_LINK + BAD_ITEM + BAD_HEARING
+
+def write_fixtures(out_dir: str) -> None:
+    import os
+    os.makedirs(out_dir, exist_ok=True)
+    # A clean single-file fixture for the "good" acceptance test.
+    with open(os.path.join(out_dir, "malformed_fragment.html"), "w", encoding="utf-8") as fh:
+        fh.write(malformed_document())
+    # A tiny encoding-edge-case file: UTF-16 with a BOM so the charset path runs.
+    with open(os.path.join(out_dir, "utf16_sample.html"), "w", encoding="utf-16") as fh:
+        fh.write(VALID)
+
+if __name__ == "__main__":
+    write_fixtures(".parser_fixtures")
+    import sys
+    print("wrote fixtures to .parser_fixtures/", file=sys.stderr)