352 lines
18 KiB
YAML
352 lines
18 KiB
YAML
# California Legislature Daily File - source specification
|
|
# Produced by research task t_a171f4d1 (software_engineer_programmer_analyst)
|
|
# Verified live: 2026-09-04 (PDT). Samples archived under ./samples/.
|
|
# Consumed by: t_f595542f (ingestion), t_fb8081e7 (parsing), t_8e5a3821 (RDB schema).
|
|
#
|
|
# IMPORTANT DESIGN NOTE (read first):
|
|
# The Assembly Daily File is a fully server-rendered HTML page with clean,
|
|
# data-attributed tables -> parse with BeautifulSoup/CSS selectors offline.
|
|
# The Senate "Daily File" is NOT HTML. senate.ca.gov/dailyfile is a Vue
|
|
# pdf-viewer wrapper with no parseable table; the publication itself is a
|
|
# PDF, served behind an antibot challenge. Senate parsing must go through
|
|
# a PDF text-extraction path (pdfplumber/PyMuPDF), with antibot mitigation.
|
|
|
|
meta:
|
|
generated: "2026-09-04"
|
|
verified_live: true
|
|
session_current: "2025-2026" # current regular session; used inside bill_id
|
|
notes: >
|
|
Per chamber we expose two independent concerns:
|
|
(a) the human-facing DAILY FILE (hearing schedule + floor items + committee items)
|
|
(b) the per-BILL canonical record (author, title, committee history) on leginfo.
|
|
The parent project (t_21cd58a9 section 2.5) requires correlating bills <-> committees
|
|
<-> committee members <-> legislators. For that we ALSO ingest leginfo bill
|
|
pages and committee rosters (see correlation_sources).
|
|
|
|
sources:
|
|
|
|
assembly_daily_file:
|
|
description: >
|
|
California State Assembly Daily File. Server-rendered HTML, no JS required
|
|
to read the data tables (data lives in static <table> markup with
|
|
data-* attributes). Always renders the CURRENT live daily file; there is
|
|
NO date query parameter that selects a historical daily file.
|
|
url:
|
|
canonical: "https://www.assembly.ca.gov/schedules-publications/daily-file"
|
|
aliases:
|
|
- "https://assembly.ca.gov/schedules-publications/daily-file"
|
|
- "https://assembly.ca.gov/dailyfile/pdfpublications"
|
|
legacy_help: "https://leginfo.ca.gov/help/dayfileh.html"
|
|
fetch:
|
|
method: GET
|
|
auth: none
|
|
requires_js: false
|
|
antibot: false
|
|
suggested_ua: "Mozilla/5.0 (research-scanner/1.0; +https://github.com/<org>)"
|
|
content_type: "text/html; charset=UTF-8"
|
|
sample: "samples/assembly_dailyfile_latest.html"
|
|
parameters:
|
|
# Confirmed by probe: ?date=... and ?dailyfile=... are NOT honored; the
|
|
# page always returns the live file. "Date" is captured FROM the data
|
|
# (hearing dates in the hearing table), not from the URL.
|
|
by_url: []
|
|
historical_retrieval: >
|
|
No date-addressable URL. To obtain a historical daily file, fetch the
|
|
live page and record the as-of date; or pull from the Senate/Assembly
|
|
PDF publication archive (see senate_daily_file + assembly pdfmirror).
|
|
session: "derived from bill_id (leading 8 digits, 'YYYYYYYY')"
|
|
tables:
|
|
- id: committee_hearings
|
|
selector: "table.committee-hearing-table"
|
|
row: "tr[class*='committee-hearing-details']"
|
|
fields:
|
|
date:
|
|
selector: "td.committee_hearing-date"
|
|
best: "attr:data-sort"
|
|
note: "raw 'MM/DD/YY-###'; display text 'MM/DD/YY'"
|
|
time:
|
|
selector: "td.committee_hearing-time"
|
|
best: "text"
|
|
committee:
|
|
selector: "td.committee_hearing-name"
|
|
best: "attr:data-sort"
|
|
location:
|
|
selector: "td.committee_hearing-location"
|
|
best: "attr:data-sort"
|
|
agenda_id:
|
|
selector: "td.committee_hearing-menu a[href^='/api/dailyfile/agenda/']"
|
|
best: "href"
|
|
note: "Drupal AJAX id for detail view"
|
|
ics_url:
|
|
selector: "td.committee_hearing-menu a[href^='/api/ics/generate']"
|
|
best: "href"
|
|
xpath:
|
|
date: "//tr[contains(@class,'committee-hearing-details')]//td[contains(@class,'committee_hearing-date')]/@data-sort"
|
|
committee: "//tr[contains(@class,'committee-hearing-details')]//td[contains(@class,'committee_hearing-name')]/@data-sort"
|
|
agenda_id: "//tr[contains(@class,'committee-hearing-details')]//td[contains(@class,'committee_hearing-menu')]//a[starts-with(@href,'/api/dailyfile/agenda/')]/@href"
|
|
sample_row: |
|
|
<tr class=" committee-hearing-details">
|
|
<td class="committee_hearing-date" data-sort="08/31/26-001">08/31/26</td>
|
|
<td class="committee_hearing-time">Upon Call of the Chair</td>
|
|
<td class="committee_hearing-name" data-sort="Budget">Budget</td>
|
|
<td class="committee_hearing-location" data-sort="1021 O Street, Room 1100">1021 O Street, Room 1100</td>
|
|
<td class="committee_hearing-menu">
|
|
<a href="/api/dailyfile/agenda/20361" role="menuitem">View Agenda</a>
|
|
</td>
|
|
</tr>
|
|
|
|
- id: floor_bills
|
|
selector: "table.floor-bills-table"
|
|
row: "tr.class='floor-bill-details'"
|
|
fields:
|
|
bill_number:
|
|
selector: "td.floor_bill-measure a"
|
|
best: "attr:href"
|
|
parse: "extract bill_id, see bill_id_encoding"
|
|
bill_display:
|
|
selector: "td.floor_bill-measure a"
|
|
best: "text" # e.g. "AB 28"
|
|
supplement:
|
|
selector: "td.floor_bill-measure"
|
|
best: "text-after-br" # e.g. "Supplemental 2"
|
|
author:
|
|
selector: "td.floor_bill-author"
|
|
best: "attr:data-sort"
|
|
title:
|
|
selector: "td.floor_bill-subject"
|
|
best: "text" # short title / description
|
|
location:
|
|
selector: "td.floor_bill-location"
|
|
best: "attr:data-sort"
|
|
item_number:
|
|
selector: "td.floor_bill-item"
|
|
best: "text" # zero-padded; strip leading zeros
|
|
chamber:
|
|
derived: true
|
|
from: "bill_id"
|
|
note: "AB/ACR/HR -> Assembly-origin; SB/SCR/SJR -> Senate-origin"
|
|
date:
|
|
derived: true
|
|
note: "inherit hearing/date context or record as-of; not on this row"
|
|
xpath:
|
|
bill_number: "//tr[contains(@class,'floor-bill-details')]//td[contains(@class,'floor_bill-measure')]/a/@href"
|
|
author: "//tr[contains(@class,'floor-bill-details')]//td[contains(@class,'floor_bill-author')]/@data-sort"
|
|
title: "//tr[contains(@class,'floor-bill-details')]//td[contains(@class,'floor_bill-subject')]/text()"
|
|
item_number: "//tr[contains(@class,'floor-bill-details')]//td[contains(@class,'floor_bill-item')]/text()"
|
|
sample_row: |
|
|
<tr class="floor-bill-details">
|
|
<td class="floor_bill-measure" data-sort="AB 00028">
|
|
<a href="https://leginfo.legislature.ca.gov/faces/billNavClient.xhtml?bill_id=202520260AB28" target="_blank">AB 28</a><br>Supplemental 2</td>
|
|
<td class="floor_bill-author" data-sort="Schiavo et al.">Schiavo et al.</td>
|
|
<td class="floor_bill-subject">An act relating to solid waste, and making an appropriation therefor.</td>
|
|
<td class="floor_bill-location" data-sort="Unfinished Business">Unfinished Business - Concurrence In Senate Amendments</td>
|
|
<td class="floor_bill-item" data-sort="0000032">32</td>
|
|
<td class="floor_bill-view-details"><!-- modal dialog; ignore --></td>
|
|
</tr>
|
|
|
|
- id: committee_bills
|
|
selector: "table.committee-bills-table"
|
|
row: "tr[class*='committee-bills-details']"
|
|
fields:
|
|
bill_number:
|
|
selector: "td.committee_bill-number a"
|
|
best: "attr:href"
|
|
parse: "extract bill_id, see bill_id_encoding"
|
|
author:
|
|
selector: "td.committee_bill-author"
|
|
best: "attr:data-sort"
|
|
title:
|
|
selector: "td.committee_bill-subject"
|
|
best: "text"
|
|
committee:
|
|
selector: "td.committee_bill-committee"
|
|
best: "attr:data-sort"
|
|
xpath:
|
|
bill_number: "//tr[contains(@class,'committee-bills-details')]//td[contains(@class,'committee_bill-number')]/a/@href"
|
|
author: "//tr[contains(@class,'committee-bills-details')]//td[contains(@class,'committee_bill-author')]/@data-sort"
|
|
title: "//tr[contains(@class,'committee-bills-details')]//td[contains(@class,'committee_bill-subject')]/text()"
|
|
committee: "//tr[contains(@class,'committee-bills-details')]//td[contains(@class,'committee_bill-committee')]/@data-sort"
|
|
sample_row: |
|
|
<tr class="committee-bills-details">
|
|
<td class="committee_bill-number" data-sort="SB 00795">
|
|
<a href="https://leginfo.legislature.ca.gov/faces/billNavClient.xhtml?bill_id=202520260SB795" target="_blank">SB 795</a></td>
|
|
<td class="committee_bill-author" data-sort="Archuleta">Archuleta</td>
|
|
<td class="committee_bill-subject">Air pollution: gasoline vapor control systems: ethanol blends.</td>
|
|
<td class="committee_bill-committee" data-sort="Emergency Management">Emergency Management</td>
|
|
</tr>
|
|
|
|
bill_id_encoding:
|
|
template: "https://leginfo.legislature.ca.gov/faces/billNavClient.xhtml?bill_id={bill_id}"
|
|
bill_id_pattern: "^(?P<session>\\d{8})\\d(?P<chamber>AB|SB|ACR|SCR|HR|SJR|ABR)(?P<number>\\d+)$"
|
|
fields:
|
|
session: "leading 8 digits 'YYYYYYYY' = 'YYYY-YYYY' (e.g. 20252026 == 2025-2026)"
|
|
chamber: "AB|SB|ACR|SCR|HR|SJR; ASSEMBLY origin = AB/ACR/HR/ABR, SENATE origin = SB/SCR/SJR"
|
|
number: "digits; zero-pad to 4 when writing back (AB 28 -> '0028')"
|
|
example: "bill_id '202520260AB28' -> session=2025-2026, chamber=AB, number=28 -> 'AB 28'"
|
|
|
|
extraction_notes: >
|
|
PREFER the 'data-sort' attribute over textContent: it holds the already-cleaned
|
|
canonical value (dates without the '-###' suffix, names/committees trimmed).
|
|
Fall back to textContent() when data-sort is absent or null.
|
|
Ignore td.floor_bill-view-details (contains a <dialog> modal widget, not a field).
|
|
'chamber' is a DERIVED field from the bill_id, not a column on the table.
|
|
'date' is NOT per-row: the assembly floor/committee tables are session-wide;
|
|
row-level DATE/COMMITTEE/HEARING-TIME come from the committee_hearings context.
|
|
|
|
senate_daily_file:
|
|
description: >
|
|
California State Senate Daily File. The HTML page is a Vue pdf-viewer
|
|
wrapper with NO static data tables. The publication exists ONLY as a
|
|
PDF.
|
|
url:
|
|
index_html: "https://www.senate.ca.gov/dailyfile"
|
|
index_html_canonical: "https://www.senate.ca.gov/publications/senate-daily-file"
|
|
pdf_index: "https://senate.ca.gov/dailyfile/publications/download/"
|
|
sample_index: "samples/senate_dailyfile_index.html"
|
|
fetch:
|
|
html_index:
|
|
requires_js: true
|
|
antibot: true
|
|
note: "pdf list rendered by JS; parse <canvas data-link> or <a href>"
|
|
pdf:
|
|
antibot: true
|
|
method: GET
|
|
antibot_note: >
|
|
senate.ca.gov wraps the publication download path in a 'form.antibot'
|
|
challenge. Plain HTTP / browser JS fetch returns HTTP 403 with a
|
|
67-byte HTML body (verified 2026-09-04). Mitigate per the
|
|
blocked-page-recovery skill or a pre-warmed headless session.
|
|
parameters:
|
|
pdf_filename_pattern: "SEN_{TYPE}_{YEAR}_R_{YYYYMMDD}.PDF"
|
|
type_values:
|
|
R: "Regular daily file (standard), e.g. SEN_STD_2025_R_20260809.PDF"
|
|
COM: "Committee Daily File, e.g. SEN_COM_2025_R_20260902.PDF (verified 200 when served)"
|
|
ESS: "Special/extraordinary session variant (historical), e.g. SEN_ESS_*"
|
|
verified_fragment: "SEN_COM_2025_R_20260902.PDF"
|
|
session_year_var: "YEAR in filename = legislative session start year (2025 for 2025-2026)"
|
|
date_var: "{YYYYMMDD}"
|
|
not_all_days_have_file: >
|
|
404 on the PDF index for weekends/non-legislative days is EXPECTED,
|
|
NOT an error. A scheduler must distinguish 404-no-file from 5xx-transient.
|
|
parsing:
|
|
engine: "pdfplumber / PyMuPDF (fitz)"
|
|
note: >
|
|
The Senate Daily File PDF has no machine tags. Use text-layout heuristics:
|
|
section headers 'BILLS ON THE FLOOR', 'COMMITTEE HEARINGS',
|
|
'BILLS IN COMMITTEE', 'HELD AT DESK', 'INACTIVE FILE',
|
|
'OFFICERS OF THE SENATE', 'SENATE ORDER OF BUSINESS'; then a
|
|
bill-number regex at line start
|
|
^(AB|SB|ACR|SCR|HR|SJR|ABR)\\s+(\\d+)$.
|
|
Author/subject follow; committee sections read 'COMMITTEE OF <NAME>'.
|
|
Prefer the leginfo HTML alternative below for structured extraction.
|
|
alt_html_source:
|
|
billResultsClient:
|
|
template: "https://leginfo.legislature.ca.gov/faces/billResultsClient.xhtml?agendadate={MM/DD/YY}&description={DESC}&location={LOC}"
|
|
locations: ["AFLOOR", "FLOOR", "SFLO", "COMM"]
|
|
requires_js: true
|
|
note: "For programmatic use prefer the JSON endpoints behind billResultsClient."
|
|
|
|
leginfo_api:
|
|
description: >
|
|
Canonical structured bill + agenda endpoints. Use these when an HTML page
|
|
requires a headless browser. billResultsClient / billHistoryClient are JS
|
|
views; the JSON endpoints behind them are the most reliable for ingestion.
|
|
base: "https://leginfo.legislature.ca.gov"
|
|
endpoints:
|
|
bill_nav:
|
|
template: "https://leginfo.legislature.ca.gov/faces/billNavClient.xhtml?bill_id={bill_id}"
|
|
content: "bill status, author, short title, committee history, votes, analyses"
|
|
sample: "samples/leginfo_assembly_floor_2026-04-16.html"
|
|
structure:
|
|
title_h1: "#bill_nav_title"
|
|
bill_title_div: ".bill_title_yr"
|
|
authors_committees: "in the bill-info block; exact selectors re-verified at parse-time"
|
|
bill_results:
|
|
endpoint: "https://leginfo.legislature.ca.gov/faces/billResultsClient.xhtml"
|
|
params: "session, chamber, billNo/author/keyword"
|
|
note: "Daily-file-style results via location FLOOR / committee name; JSON served behind it."
|
|
|
|
correlation_sources:
|
|
committee_rosters:
|
|
assembly:
|
|
url: "https://www.assembly.ca.gov/committees"
|
|
note: "Lists standing/select committees; each committee has a membership page."
|
|
senate:
|
|
url: "https://www.senate.ca.gov/committees"
|
|
sample: "samples/senate_committees_index.html"
|
|
note: "Each committee = /committees/<slug>, e.g. /committees/selectcommitteeonhydrogenenergy"
|
|
member_rosters:
|
|
assembly: "https://www.assembly.ca.gov/assemblymembers"
|
|
by_address: "https://www.assembly.ca.gov/assemblymembers/find-my-rep"
|
|
find_your_rep: "https://findyourrep.legislature.ca.gov/"
|
|
note: "Zip -> legislator(s); used by the logged-in dashboard (section 5 of parent)."
|
|
bill_detail:
|
|
template: "https://leginfo.legislature.ca.gov/faces/billNavClient.xhtml?bill_id={bill_id}"
|
|
sample: "samples/leginfo_assembly_floor_2026-04-16.html"
|
|
bill_committee_history:
|
|
template: "https://leginfo.legislature.ca.gov/faces/billHistoryClient.xhtml?bill_id={bill_id}"
|
|
note: "Action-level history: which committee acted, when, referral chains."
|
|
|
|
normalized_schema:
|
|
per_record:
|
|
id: "sha1(house|bill_number|date|committee)"
|
|
date: "ISO YYYY-MM-DD (assembly: from hearing table; senate: from PDF filename/header)"
|
|
house: "ASSEMBLY | SENATE"
|
|
source: "ASSEMBLY_DAILY_FILE | SENATE_DAILY_FILE_PDF | LEGINFO_BILL_RESULTS"
|
|
bill_number: "canonical 'AB 28' / 'SB 795' (chamber + 4-digit number)"
|
|
bill_id: "leginfo bill_id, e.g. '202520260AB28' (join to bill detail / committee history)"
|
|
session: "2025-2026"
|
|
chamber_origin: "ASSEMBLY | SENATE"
|
|
author: "lead author string, e.g. 'Schiavo et al.'"
|
|
short_title: "from the subject field"
|
|
description: "same as short_title for daily-file; full title via enrichment"
|
|
committee: "committee name (from committee_bills table or PDF section)"
|
|
hearing_time: "e.g. '9:30am' / 'Upon Call of the Chair'"
|
|
location: "e.g. 'State Capitol, Room 444'"
|
|
item_number: "int from floor table (leading zeros stripped)"
|
|
supplement: "optional edition number from 'Supplemental N'"
|
|
join_keys:
|
|
bill_id: "join to bill detail / committee history / roster membership"
|
|
committee: "join to committee roster -> committee members -> legislators"
|
|
|
|
known_risks_and_pitfalls:
|
|
- "Antibot: senate.ca.gov returns HTTP 403 on pdf download without a browser challenge solver. Use blocked-page-recovery skill or a headless browser for PDF fetch."
|
|
- "Date: neither daily-file page is date-addressable by URL. The assembly page is always current live; the senate PDF filename encodes the date."
|
|
- "Assembly tables are server-rendered and static: BeautifulSoup can parse offline from samples/assembly_dailyfile_latest.html with zero JS."
|
|
- "Senate daily file has NO HTML equivalent with the same structure; PDF parsing is required, or use leginfo billResultsClient + committee history as an enrichment source."
|
|
- "Empty/non-legislative calendar days: 404 on the PDF index is expected, NOT an error. Ingestion must distinguish 404-no-file from 5xx/transient."
|
|
- "Cross-chamber bills: an AB bill can appear on the Senate daily file and vice-versa; chamber_origin means origin, not the house publishing the daily file."
|
|
- "Session boundary: bill_id leading 8 digits ('YYYYYYYY') encode the session; when the session changes in January, the URL pattern stays the same but bills renumber."
|
|
- "data-sort attribute > textContent (cleaner). 'Supplemental 2' on a bill row is the current supplement edition."
|
|
|
|
acceptance_test_harness:
|
|
- id: asm_hearings
|
|
input: "samples/assembly_dailyfile_latest.html"
|
|
selector: "tr[class*='committee-hearing-details']"
|
|
expect_min_rows: 1
|
|
verified: 4
|
|
- id: asm_floor_bills
|
|
input: "samples/assembly_dailyfile_latest.html"
|
|
selector: "tr.class='floor-bill-details'"
|
|
expect_min_rows: 100
|
|
expect_fields: [bill_number, author, short_title, item_number]
|
|
verified: 156
|
|
- id: asm_committee_bills
|
|
input: "samples/assembly_dailyfile_latest.html"
|
|
selector: "tr[class*='committee-bills-details']"
|
|
expect_min_rows: 1
|
|
verified: 1
|
|
- id: bill_id_parse
|
|
regex: "^(?P<session>\\d{8})\\d(?P<chamber>AB|SB|ACR|SCR|HR|SJR|ABR)(?P<number>\\d+)$"
|
|
example_url: "https://leginfo.legislature.ca.gov/faces/billNavClient.xhtml?bill_id=202520260AB28"
|
|
expect:
|
|
session: "2025-2026"
|
|
chamber: "AB"
|
|
number: 28
|
|
display: "AB 28"
|
|
verified: true
|
|
- id: senate_pdf_index
|
|
input: "samples/senate_dailyfile_index.html"
|
|
expect: "at least one /dailyfile/publications/download/*.PDF reference"
|
|
verified: 1
|