feat(dashboard): add forward "what's next" signal + fix upcoming floor
The daily-file bill queue is a backward snapshot, so "upcoming" could only resurface the latest past day (the 08-31 snapshot read as today's queue) and there was no "what is scheduled next." leginfo bill status/history are backward-only, so the genuine future-dated signal is the chambers' "upcoming events" calendars. - api.py: floor the "upcoming" view at *today* (not the DB max date) when no as_of is pinned; surface next_legislative_day + scheduled_events (next 7 days) on /bills. Add _today_iso / _next_legislative_day / _scheduled_events, all degrading to a recess marker / [] (never 5xx). - relational.py: schema v2 adds the scheduled_events table + store_scheduled_events (idempotent per-day refresh), load_scheduled_events, next_legislative_day. - parsers.py: parse_events_page + classify_event_kind for the chamber events calendar; skip empty-title rows (decompose the <time> before the fallback). - ingest.py + cli.py: --store-events lane (cache-first, recess-safe) that parses the calendar HTML into scheduled_events. - sources.py + config.yaml: register assembly_events / senate_events (kind: events, empty_is_ok: true). - run_ingest.sh: run the events lane after the daily-file lane as non-fatal (skippable via CALI_SCRAPER_SKIP_EVENTS=1); daily-file exit still gates cron. - frontend: "Next legislative day" line + "What's next · next 7 days" band with honest recess empty states. - DASHBOARD.md: document the forward signal + known limitations (leginfo backward-only gap, recess empty state, 08-31 snapshot is analysis-only). - tests: 5 new API tests + 18 parse-events tests; 170 pass, 92% coverage.
This commit is contained in:
kfj001 committed
1 parent
1919d43bf0
commit
03eeb6bb62
16 files changed
+941
-70
No files matched your search
+17
@@ -70,6 +70,23 @@ sources:
|
|||||||
# an error. The scheduler must distinguish 404-no-file from 5xx-transient.
|
# an error. The scheduler must distinguish 404-no-file from 5xx-transient.
|
||||||
missing_is_expected_404: true
|
missing_is_expected_404: true
|
||||||
|
|
||||||
|
# Forward-looking chamber calendar. Unlike the daily file (a backward
|
||||||
|
# snapshot) this page lists the *upcoming* hearings/floor sessions -- the
|
||||||
|
# only genuine future-dated signal. Empty during recess (a legitimate "no
|
||||||
|
# upcoming events" state, DESIGN section 6.1), not an error. Parsed into
|
||||||
|
# the scheduled_events table (schema v2) by `ingest --store-events`.
|
||||||
|
assembly_events:
|
||||||
|
kind: "events"
|
||||||
|
url: "https://www.assembly.ca.gov/events/upcoming"
|
||||||
|
chamber: "assembly"
|
||||||
|
empty_is_ok: true
|
||||||
|
|
||||||
|
senate_events:
|
||||||
|
kind: "events"
|
||||||
|
url: "https://www.senate.ca.gov/calendar"
|
||||||
|
chamber: "senate"
|
||||||
|
empty_is_ok: true
|
||||||
|
|
||||||
# --- LLM analysis (spec §4, task t_950780e0) -----------------------------
|
# --- LLM analysis (spec §4, task t_950780e0) -----------------------------
|
||||||
# Subject-matter tagging of bills via a configurable Ollama endpoint. The
|
# Subject-matter tagging of bills via a configurable Ollama endpoint. The
|
||||||
# endpoint + model are NEVER hardcoded: everything below is a config knob.
|
# endpoint + model are NEVER hardcoded: everything below is a config knob.
|
||||||
|
|||||||
@@ -0,0 +1,3 @@
|
|||||||
|
<!DOCTYPE html><html lang="en" dir="ltr"><head><meta charset="utf-8" /><noscript><style>form.antibot > * { display: none !important; } form.antibot noscript { display: block !important; }</style></noscript><script async src="https://www.googletagmanager.com/gtag/js?id=G-4D0PPGX2BH"></script><script>window.dataLayer = window.dataLayer || [];function gtag(){dataLayer.push(arguments)};gtag("js", new Date());gtag("set", "developer_id.dMDhkMT", true);gtag("config", "G-4D0PPGX2BH", {"groups":"default","cookie_domain":"assembly.ca.gov","page_placeholder":"PLACEHOLDER_page_location","link_attribution":true});</script><style id="gutenberg-palette">:root { .has-primary-color{ color:#255752 } .has-primary-background-color{ background-color:#255752 } .has-primary-border-color{ border-color:#255752 } .has-text-color{ color:#292B29 } .has-text-background-color{ background-color:#292B29 } .has-text-border-color{ border-color:#292B29 } .has-link-color{ color:#01579F } .has-link-background-color{ background-color:#01579F } .has-link-border-color{ border-color:#01579F } .has-highlight-color{ color:#595959 } .has-highlight-background-color{ background-color:#595959 } .has-highlight-border-color{ border-color:#595959 } .has-light-color{ color:#a30c1f } .has-light-background-color{ background-color:#a30c1f } .has-light-border-color{ border-color:#a30c1f } }</style><style id="gutenberg-gradients">:root { .has-primary-to-secondary-gradient-background{ background:linear-gradient(135deg, #CEBAFF 0%, #FFC884 100%) } .has-primary-to-contrast-gradient-background{ background:linear-gradient(135deg, #CEBAFF 0%, #3556D1 100%) } .has-secondary-to-light-gradient-background{ background:linear-gradient(to right bottom, #FFC884, #FFFFFF) } }</style><style id="gutenberg-font-sizes">:root { .has-small-font-size{ font-size:14px } .has-normal-font-size{ font-size:16px } .has-large-font-size{ font-size:18px } .has-extra-large-font-size{ font-size:20px } .has-extra-extra-large-font-size{ font-size:24px } }</style><style id="gutenberg-spacing-variables">:root { --wp--preset--spacing--10:8px; --wp--preset--spacing--20:16px; --wp--preset--spacing--30:32px; --wp--preset--spacing--40:64px; --wp--preset--spacing--50:128px; --wp--preset--spacing--60:172px; }</style><style>/* @see https://github.com/aFarkas/lazysizes#broken-image-symbol */.js img.lazyload:not([src]) { visibility: hidden; }/* @see https://github.com/aFarkas/lazysizes#automatically-setting-the-sizes-attribute */.js img.lazyloaded[data-sizes=auto] { display: block; width: 100%; }/* Transition effect. */.js .lazyload, .js .lazyloading { opacity: 0; }.js .lazyloaded { opacity: 1; -webkit-transition: opacity 2000ms; transition: opacity 2000ms; }</style><meta name="MobileOptimized" content="width" /><meta name="HandheldFriendly" content="true" /><meta name="viewport" content="width=device-width, initial-scale=1.0" /><link rel="icon" href="/themes/custom/asm_internet_theme/favicon.ico" type="image/png" /><title>Events | California State Assembly</title><link rel="preload" as="font" href="/themes/custom/asm_internet_theme/assets/fonts/InstrumentSans%5Bwdth%2Cwght%5D.woff2" type="font/woff2" crossorigin="anonymous"><link rel="preload" as="image" type="image/svg+xml" href="/themes/custom/asm_internet_theme/assets/images/Seal.svg"><link rel="preconnect" href="https://webapi.assembly.ca.gov"><link rel="dns-prefetch" href="https://webapi.assembly.ca.gov"><link rel="stylesheet" media="all" href="/sites/assembly.ca.gov/files/css/css_ejSlPE3p5AQVnknFBVL5QBhPgeUULpNo9LmKiD81JM0.css?delta=0&language=en&theme=asm_internet_theme&include=eJx1iVEKwzAMxS4U5iMZN3trszpxiN2F3n5Q2NfojxCSeOXSAqMhODZU0Eslesn7SPI_V7VFNDlk5I2lF5YjLFvtigDd9OSnByot4kifgul08VHteSjSFOf37ConBv3kqtPG7l0ySG0t7QsR4EXh" /><link rel="stylesheet" media="all" href="/sites/assembly.ca.gov/files/css/css_iwL7Sf3hxDGV2hXPq5CgiQnir6U0Y8VHL7HgNPcA6o0.css?delta=1&language=en&theme=asm_internet_theme&include=eJx1iVEKwzAMxS4U5iMZN3trszpxiN2F3n5Q2NfojxCSeOXSAqMhODZU0Eslesn7SPI_V7VFNDlk5I2lF5YjLFvtigDd9OSnByot4kifgul08VHteSjSFOf37ConBv3kqtPG7l0ySG0t7QsR4EXh" /></head><body class="path-events page-events-past"> <a href="#main-content" class="visually-hidden focusable skip-link"> Skip to main content </a><div class="dialog-off-canvas-main-canvas" data-off-canvas-main-canvas><div class="layout-container"><div class="top_bar"><div data-drupal-messages-fallback class="hidden"></div></div> <header class="header"><div class="header__inner"><div class="header__primary"><div class="header__branding"><div class="logo"> <a class="logo__link" href="/"
|
||||||
|
> <img class="logo__image" src="/themes/custom/asm_internet_theme/assets/images/Seal.svg" alt="California State Assembly Home" /> <span class="logo--title"> California State <br> Assembly </span> </a></div></div><div class="header__menu"> <nav role="navigation" aria-labelledby="block-asm-internet-theme-main-menu-menu" id="block-asm-internet-theme-main-menu" class="nav--primary"><h2 class="visually-hidden" id="block-asm-internet-theme-main-menu-menu">Main navigation</h2><div class="nav-wrapper" > <nav id="main-nav" class="main-nav"><div id="mobile-menu--close" class="hide"> <svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg"> <g id="ICO - Close"> <g id="Vector"> <path fill-rule="evenodd" clip-rule="evenodd" d="M25.8839 6.11612C26.372 6.60427 26.372 7.39573 25.8839 7.88388L7.88388 25.8839C7.39573 26.372 6.60427 26.372 6.11612 25.8839C5.62796 25.3957 5.62796 24.6043 6.11612 24.1161L24.1161 6.11612C24.6043 5.62796 25.3957 5.62796 25.8839 6.11612Z" fill="#292B29"/> <path fill-rule="evenodd" clip-rule="evenodd" d="M6.11612 6.11612C6.60427 5.62796 7.39573 5.62796 7.88388 6.11612L25.8839 24.1161C26.372 24.6043 26.372 25.3957 25.8839 25.8839C25.3957 26.372 24.6043 26.372 24.1161 25.8839L6.11612 7.88388C5.62796 7.39573 5.62796 6.60427 6.11612 6.11612Z" fill="#292B29"/> </g> </g> </svg></div><ul class="main-menu"><li class="main-menu__item main-menu__item--og-child main-menu__item--with-sub"> <a href="/assemblymembers" class="main-menu__link main-menu__link--og-child main-menu__link--with-sub" data-drupal-link-system-path="node/86244">Members</a> <span class="expand-sub"></span><ul class="main-menu main-menu--sub main-menu--sub-1"><li class="main-menu__item main-menu__item--sub main-menu__item--sub-1"> <a href="/assemblymembers" class="main-menu__link main-menu__link--sub main-menu__link--sub-1" data-drupal-link-system-path="node/86244">Member Roster</a></li><li class="main-menu__item main-menu__item--sub main-menu__item--sub-1"> <a href="/assemblymembers/leadership" title="Leadership" class="main-menu__link main-menu__link--sub main-menu__link--sub-1" data-drupal-link-system-path="node/8">Leadership</a></li><li class="main-menu__item main-menu__item--sub main-menu__item--sub-1"> <a href="/assemblymembers/member-seating-chart" class="main-menu__link main-menu__link--sub main-menu__link--sub-1" data-drupal-link-system-path="node/86246">Member Seating Chart</a></li><li class="main-menu__item main-menu__item--sub main-menu__item--sub-1"> <a href="/assemblymembers/find-my-rep" class="main-menu__link main-menu__link--sub main-menu__link--sub-1" data-drupal-link-system-path="assemblymembers/find-my-rep">Find Your Representative</a></li><li class="main-menu__item main-menu__item--sub main-menu__item--sub-1"> <a href="/assemblymembers/10th-o-street-offices" class="main-menu__link main-menu__link--sub main-menu__link--sub-1" data-drupal-link-system-path="node/86248">10th & O Street Offices</a></li></ul></li><li class="main-menu__item main-menu__item--og-child main-menu__item--with-sub"> <a href="/committees" class="main-menu__link main-menu__link--og-child main-menu__link--with-sub" data-drupal-link-system-path="node/85503">Committees</a> <span class="expand-sub"></span><ul class="main-menu main-menu--sub main-menu--sub-1"><li class="main-menu__item main-menu__item--sub main-menu__item--sub-1"> <a href="/committees?sessionnumber=0" class="main-menu__link main-menu__link--sub main-menu__link--sub-1" data-drupal-link-query="{"sessionnumber":"0"}" data-drupal-link-system-path="node/85503">Regular Session Committees</a></li><li class="main-menu__item main-menu__item--sub main-menu__item--sub-1"> <a href="/committees?sessionnumber=1" class="main-menu__link main-menu__link--sub main-menu__link--sub-1" data-drupal-link-query="{"sessionnumber":"1"}" data-drupal-link-system-path="node/85503">First Extraordinary Committees</a></li></ul></li><li class="main-menu__item main-menu__item--og-child main-menu__item--with-sub"> <a href="/schedules-publications" title="Schedules & Publications" class="main-menu__link main-menu__link--og-child main-menu__link--with-sub" data-drupal-link-system-path="node/5936">Schedules & Publications</a> <span class="expand-sub"></span><ul class="main-menu main-menu--sub main-menu--sub-1"><li class="main-menu__item main-menu__item--sub main-menu__item--sub-1"> <a href="/schedules-publications" class="main-menu__link main-menu__link--sub main-menu__link--sub-1" data-drupal-link-system-path="node/5936">Overview</a></li><li class="main-menu__item main-menu__item--sub main-menu__item--sub-1"> <a href="/schedules-publications/daily-file" class="main-menu__link main-menu__link--sub main-menu__link--sub-1" data-drupal-link-system-path="node/86247">Daily File</a></li><li class="main-menu__item main-menu__item--sub main-menu__item--sub-1"> <a href="/schedules-publications/salaries-and-expenditures" title="Salaries & ExpendLine truncated
|
||||||
|
> <img class="logo__image" src="/themes/custom/asm_internet_theme/assets/images/Seal.svg" alt="California State Assembly Home" /> <span class="logo--title"> California State <br> Assembly </span> </a></div></div></div></footer><div class="bottom_bar"><div class="bottom_bar__inner"><div id="block-asm-internet-theme-wascopyrightblock" class="block"><div class="block-container"><div><div class="copyright-container"><span class="copyright">Copyright © 2026 State of California</span></div></div></div></div><div id="block-asm-internet-theme-subfooter" class="block"><div class="block-container"><div><div class="copyright-container"><ul><li><a href="/sitemap">Site Map</a></li><li><a href="/accessibility">Accessibility</a></li><li><a href="/privacy-and-conditions-use-policy">Terms & Privacy</a></li><li class="btn-vote"><a href="https://registertovote.ca.gov/">Register to Vote</a></li></ul></div></div></div></div></div></div></div></div><div class="bg-overlay"></div><script type="application/json" data-drupal-selector="drupal-settings-json">{"path":{"baseUrl":"\/","pathPrefix":"","currentPath":"events\/past","currentPathIsAdmin":false,"isFront":false,"currentLanguage":"en"},"pluralDelimiter":"\u0003","suppressDeprecationErrors":true,"ajaxPageState":{"libraries":"eJx1jkFuBCEMBD9EwmdyRx7oZa0YjDDMiLw-2ZFWWiWTSx-62laRlcB1oFeMMO4o8Ak3mjIc_UXYUYddkSy6kVyRQlzfCuq8hLqx4KPTuqKNMvoJChKTpzk0RC1NMBB0R--c8DKIJKiJXm_ODE1oof_85-FQphjf1tM5q2ZBoEqyBkfzvwsn9LX8I5yBerwHahweMk8X_0_vbNlA8RsZ3M44zJ_5XjRNgTvIwqH90xpFeNHM9RvVC57S","theme":"asm_internet_theme","theme_token":null},"ajaxTrustedUrl":{"\/events\/past":true,"\/search":true},"google_analytics":{"account":"G-4D0PPGX2BH","trackOutbound":true,"trackMailto":true,"trackTel":true,"trackDownload":true,"trackDownloadExtensions":"7z|aac|arc|arj|asf|asx|avi|bin|csv|doc(x|m)?|dot(x|m)?|exe|flv|gif|gz|gzip|hqx|jar|jpe?g|js|mp(2|3|4|e?g)|mov(ie)?|msi|msp|pdf|phps|png|ppt(x|m)?|pot(x|m)?|pps(x|m)?|ppam|sld(x|m)?|thmx|qtm?|ra(m|r)?|sea|sit|tar|tgz|torrent|txt|wav|wma|wmv|wpd|xls(x|m|b)?|xlt(x|m)|xlam|xml|z|zip","trackDomainMode":1},"lazy":{"lazysizes":{"lazyClass":"lazyload","loadedClass":"lazyloaded","loadingClass":"lazyloading","preloadClass":"lazypreload","errorClass":"lazyerror","autosizesClass":"lazyautosizes","srcAttr":"data-src","srcsetAttr":"data-srcset","sizesAttr":"data-sizes","minSize":40,"customMedia":[],"init":true,"expFactor":1.5,"hFac":0.8,"loadMode":1,"loadHidden":true,"ricTimeout":0,"throttleDelay":125,"plugins":[]},"placeholderSrc":"data:image\/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==","preferNative":false,"minified":true,"libraryPath":"\/libraries\/lazysizes"},"was_jwplayer":{"config":{"key":"yh5RZuPQQH3OaPRB\/a\/geG6f46hHX1+s7SszOHZIu3T0qSGJ","clean_source":0,"thumbnails":0},"defaults":{"volume":"","height":"","width":"100%","aspectratio":"16:9","poster":{"live":"https:\/\/www.assembly.ca.gov\/sites\/assembly.ca.gov\/files\/jwplayer\/walkthrough_cover_02_1.jpg","archive":"https:\/\/www.assembly.ca.gov\/sites\/assembly.ca.gov\/files\/jwplayer\/walkthrough_cover_02_2.jpg"}},"files":{"images":{"arrow_white":"https:\/\/www.assembly.ca.gov\/modules\/custom\/was_jwplayer\/css\/arrow-white.png"}}},"search_api_autocomplete":{"search":{"auto_submit":true}},"was_workspace":{"toggled_elements":".tabs__nav, .tabs, div:not(.view-offices-menu) .contextual, .edit-mode, .add-docs-form, .contextual, .view-display-id-block_class_session .views-field-edit-node, .unpublished-menu-item, #environment-indicator","edit_mode_key_toggle":"`","toggled_parent_elements":".node--unpublished.node--view-mode-teaser, .unpublished-menu-item","selector_items":[]},"user":{"uid":0,"permissionsHash":"06c739f6dcef12a85d882bf46cf3cf32d176c4d3b071849cbe28f89ad246822c"}}</script><script src="/sites/assembly.ca.gov/files/js/js_HJhQpuSweG7gwE4KjbQL3maEdu-Q6Pf_cgLHvzoRKc4.js?scope=footer&delta=0&language=en&theme=asm_internet_theme&include=eJx1jdsKwjAQRH-okp_xfRmTsV3IjWRbiF-vFYSi9WUe5swFPYlmY8s0sYWJLvCONdqEX8SN2foZSdB8SczrKSw3jbw2jDNaMbO9QWJQOKxWxJdUI41SNramgYeAR2QOOHbeKjVisL321aa5lDlSkBGHqe_u25giHsPtMnWi-UVQVfbzz7f74z8Bx1B6Xg"></script></body></html>
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
<!DOCTYPE html><html lang="en" dir="ltr"><head><meta charset="utf-8" /><noscript><style>form.antibot > * { display: none !important; } form.antibot noscript { display: block !important; }</style></noscript><script async src="https://www.googletagmanager.com/gtag/js?id=G-4D0PPGX2BH"></script><script>window.dataLayer = window.dataLayer || [];function gtag(){dataLayer.push(arguments)};gtag("js", new Date());gtag("set", "developer_id.dMDhkMT", true);gtag("config", "G-4D0PPGX2BH", {"groups":"default","cookie_domain":"assembly.ca.gov","page_placeholder":"PLACEHOLDER_page_location","link_attribution":true});</script><style id="gutenberg-palette">:root { .has-primary-color{ color:#255752 } .has-primary-background-color{ background-color:#255752 } .has-primary-border-color{ border-color:#255752 } .has-text-color{ color:#292B29 } .has-text-background-color{ background-color:#292B29 } .has-text-border-color{ border-color:#292B29 } .has-link-color{ color:#01579F } .has-link-background-color{ background-color:#01579F } .has-link-border-color{ border-color:#01579F } .has-highlight-color{ color:#595959 } .has-highlight-background-color{ background-color:#595959 } .has-highlight-border-color{ border-color:#595959 } .has-light-color{ color:#a30c1f } .has-light-background-color{ background-color:#a30c1f } .has-light-border-color{ border-color:#a30c1f } }</style><style id="gutenberg-gradients">:root { .has-primary-to-secondary-gradient-background{ background:linear-gradient(135deg, #CEBAFF 0%, #FFC884 100%) } .has-primary-to-contrast-gradient-background{ background:linear-gradient(135deg, #CEBAFF 0%, #3556D1 100%) } .has-secondary-to-light-gradient-background{ background:linear-gradient(to right bottom, #FFC884, #FFFFFF) } }</style><style id="gutenberg-font-sizes">:root { .has-small-font-size{ font-size:14px } .has-normal-font-size{ font-size:16px } .has-large-font-size{ font-size:18px } .has-extra-large-font-size{ font-size:20px } .has-extra-extra-large-font-size{ font-size:24px } }</style><style id="gutenberg-spacing-variables">:root { --wp--preset--spacing--10:8px; --wp--preset--spacing--20:16px; --wp--preset--spacing--30:32px; --wp--preset--spacing--40:64px; --wp--preset--spacing--50:128px; --wp--preset--spacing--60:172px; }</style><style>/* @see https://github.com/aFarkas/lazysizes#broken-image-symbol */.js img.lazyload:not([src]) { visibility: hidden; }/* @see https://github.com/aFarkas/lazysizes#automatically-setting-the-sizes-attribute */.js img.lazyloaded[data-sizes=auto] { display: block; width: 100%; }/* Transition effect. */.js .lazyload, .js .lazyloading { opacity: 0; }.js .lazyloaded { opacity: 1; -webkit-transition: opacity 2000ms; transition: opacity 2000ms; }</style><meta name="MobileOptimized" content="width" /><meta name="HandheldFriendly" content="true" /><meta name="viewport" content="width=device-width, initial-scale=1.0" /><link rel="icon" href="/themes/custom/asm_internet_theme/favicon.ico" type="image/png" /><title>Events | California State Assembly</title><link rel="preload" as="font" href="/themes/custom/asm_internet_theme/assets/fonts/InstrumentSans%5Bwdth%2Cwght%5D.woff2" type="font/woff2" crossorigin="anonymous"><link rel="preload" as="image" type="image/svg+xml" href="/themes/custom/asm_internet_theme/assets/images/Seal.svg"><link rel="preconnect" href="https://webapi.assembly.ca.gov"><link rel="dns-prefetch" href="https://webapi.assembly.ca.gov"><link rel="stylesheet" media="all" href="/sites/assembly.ca.gov/files/css/css_ejSlPE3p5AQVnknFBVL5QBhPgeUULpNo9LmKiD81JM0.css?delta=0&language=en&theme=asm_internet_theme&include=eJx1iVEKwzAMxS4U5iMZN3trszpxiN2F3n5Q2NfojxCSeOXSAqMhODZU0Eslesn7SPI_V7VFNDlk5I2lF5YjLFvtigDd9OSnByot4kifgul08VHteSjSFOf37ConBv3kqtPG7l0ySG0t7QsR4EXh" /><link rel="stylesheet" media="all" href="/sites/assembly.ca.gov/files/css/css_iwL7Sf3hxDGV2hXPq5CgiQnir6U0Y8VHL7HgNPcA6o0.css?delta=1&language=en&theme=asm_internet_theme&include=eJx1iVEKwzAMxS4U5iMZN3trszpxiN2F3n5Q2NfojxCSeOXSAqMhODZU0Eslesn7SPI_V7VFNDlk5I2lF5YjLFvtigDd9OSnByot4kifgul08VHteSjSFOf37ConBv3kqtPG7l0ySG0t7QsR4EXh" /></head><body class="path-events page-events-upcoming"> <a href="#main-content" class="visually-hidden focusable skip-link"> Skip to main content </a><div class="dialog-off-canvas-main-canvas" data-off-canvas-main-canvas><div class="layout-container"><div class="top_bar"><div data-drupal-messages-fallback class="hidden"></div></div> <header class="header"><div class="header__inner"><div class="header__primary"><div class="header__branding"><div class="logo"> <a class="logo__link" href="/"
|
||||||
|
> <img class="logo__image" src="/themes/custom/asm_internet_theme/assets/images/Seal.svg" alt="California State Assembly Home" /> <span class="logo--title"> California State <br> Assembly </span> </a></div></div><div class="header__menu"> <nav role="navigation" aria-labelledby="block-asm-internet-theme-main-menu-menu" id="block-asm-internet-theme-main-menu" class="nav--primary"><h2 class="visually-hidden" id="block-asm-internet-theme-main-menu-menu">Main navigation</h2><div class="nav-wrapper" > <nav id="main-nav" class="main-nav"><div id="mobile-menu--close" class="hide"> <svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg"> <g id="ICO - Close"> <g id="Vector"> <path fill-rule="evenodd" clip-rule="evenodd" d="M25.8839 6.11612C26.372 6.60427 26.372 7.39573 25.8839 7.88388L7.88388 25.8839C7.39573 26.372 6.60427 26.372 6.11612 25.8839C5.62796 25.3957 5.62796 24.6043 6.11612 24.1161L24.1161 6.11612C24.6043 5.62796 25.3957 5.62796 25.8839 6.11612Z" fill="#292B29"/> <path fill-rule="evenodd" clip-rule="evenodd" d="M6.11612 6.11612C6.60427 5.62796 7.39573 5.62796 7.88388 6.11612L25.8839 24.1161C26.372 24.6043 26.372 25.3957 25.8839 25.8839C25.3957 26.372 24.6043 26.372 24.1161 25.8839L6.11612 7.88388C5.62796 7.39573 5.62796 6.60427 6.11612 6.11612Z" fill="#292B29"/> </g> </g> </svg></div><ul class="main-menu"><li class="main-menu__item main-menu__item--og-child main-menu__item--with-sub"> <a href="/assemblymembers" class="main-menu__link main-menu__link--og-child main-menu__link--with-sub" data-drupal-link-system-path="node/86244">Members</a> <span class="expand-sub"></span><ul class="main-menu main-menu--sub main-menu--sub-1"><li class="main-menu__item main-menu__item--sub main-menu__item--sub-1"> <a href="/assemblymembers" class="main-menu__link main-menu__link--sub main-menu__link--sub-1" data-drupal-link-system-path="node/86244">Member Roster</a></li><li class="main-menu__item main-menu__item--sub main-menu__item--sub-1"> <a href="/assemblymembers/leadership" title="Leadership" class="main-menu__link main-menu__link--sub main-menu__link--sub-1" data-drupal-link-system-path="node/8">Leadership</a></li><li class="main-menu__item main-menu__item--sub main-menu__item--sub-1"> <a href="/assemblymembers/member-seating-chart" class="main-menu__link main-menu__link--sub main-menu__link--sub-1" data-drupal-link-system-path="node/86246">Member Seating Chart</a></li><li class="main-menu__item main-menu__item--sub main-menu__item--sub-1"> <a href="/assemblymembers/find-my-rep" class="main-menu__link main-menu__link--sub main-menu__link--sub-1" data-drupal-link-system-path="assemblymembers/find-my-rep">Find Your Representative</a></li><li class="main-menu__item main-menu__item--sub main-menu__item--sub-1"> <a href="/assemblymembers/10th-o-street-offices" class="main-menu__link main-menu__link--sub main-menu__link--sub-1" data-drupal-link-system-path="node/86248">10th & O Street Offices</a></li></ul></li><li class="main-menu__item main-menu__item--og-child main-menu__item--with-sub"> <a href="/committees" class="main-menu__link main-menu__link--og-child main-menu__link--with-sub" data-drupal-link-system-path="node/85503">Committees</a> <span class="expand-sub"></span><ul class="main-menu main-menu--sub main-menu--sub-1"><li class="main-menu__item main-menu__item--sub main-menu__item--sub-1"> <a href="/committees?sessionnumber=0" class="main-menu__link main-menu__link--sub main-menu__link--sub-1" data-drupal-link-query="{"sessionnumber":"0"}" data-drupal-link-system-path="node/85503">Regular Session Committees</a></li><li class="main-menu__item main-menu__item--sub main-menu__item--sub-1"> <a href="/committees?sessionnumber=1" class="main-menu__link main-menu__link--sub main-menu__link--sub-1" data-drupal-link-query="{"sessionnumber":"1"}" data-drupal-link-system-path="node/85503">First Extraordinary Committees</a></li></ul></li><li class="main-menu__item main-menu__item--og-child main-menu__item--with-sub"> <a href="/schedules-publications" title="Schedules & Publications" class="main-menu__link main-menu__link--og-child main-menu__link--with-sub" data-drupal-link-system-path="node/5936">Schedules & Publications</a> <span class="expand-sub"></span><ul class="main-menu main-menu--sub main-menu--sub-1"><li class="main-menu__item main-menu__item--sub main-menu__item--sub-1"> <a href="/schedules-publications" class="main-menu__link main-menu__link--sub main-menu__link--sub-1" data-drupal-link-system-path="node/5936">Overview</a></li><li class="main-menu__item main-menu__item--sub main-menu__item--sub-1"> <a href="/schedules-publications/daily-file" class="main-menu__link main-menu__link--sub main-menu__link--sub-1" data-drupal-link-system-path="node/86247">Daily File</a></li><li class="main-menu__item main-menu__item--sub main-menu__item--sub-1"> <a href="/schedules-publications/salaries-and-expenditures" title="Salaries & ExpendLine truncated
|
||||||
|
> <img class="logo__image" src="/themes/custom/asm_internet_theme/assets/images/Seal.svg" alt="California State Assembly Home" /> <span class="logo--title"> California State <br> Assembly </span> </a></div></div></div></footer><div class="bottom_bar"><div class="bottom_bar__inner"><div id="block-asm-internet-theme-wascopyrightblock" class="block"><div class="block-container"><div><div class="copyright-container"><span class="copyright">Copyright © 2026 State of California</span></div></div></div></div><div id="block-asm-internet-theme-subfooter" class="block"><div class="block-container"><div><div class="copyright-container"><ul><li><a href="/sitemap">Site Map</a></li><li><a href="/accessibility">Accessibility</a></li><li><a href="/privacy-and-conditions-use-policy">Terms & Privacy</a></li><li class="btn-vote"><a href="https://registertovote.ca.gov/">Register to Vote</a></li></ul></div></div></div></div></div></div></div></div><div class="bg-overlay"></div><script type="application/json" data-drupal-selector="drupal-settings-json">{"path":{"baseUrl":"\/","pathPrefix":"","currentPath":"events\/upcoming","currentPathIsAdmin":false,"isFront":false,"currentLanguage":"en"},"pluralDelimiter":"\u0003","suppressDeprecationErrors":true,"ajaxPageState":{"libraries":"eJx1kE1uxSAMhC9Ey2W6R5Mw4Vk1EGF4ET1981J1U6WbWcxnj39gOUSIzk2UgU-Wbh6JJSLkGqEOZ4WUzlbYQ38w00duGNrv0E_CHUlal_u4DClvmWXcwrqcm300zItmRoG_NOyKyXY2SHfMQ022-Tsm1ZrOg1Cgs8tq_q_hFF_Tv8QZ0dZHwC4Bo9e15l3Z6f_xnU3rzH6B0T2Fh_lL389_DaU7YOGo7dN2rPRak5RvonyN-w","theme":"asm_internet_theme","theme_token":null},"ajaxTrustedUrl":{"\/search":true},"google_analytics":{"account":"G-4D0PPGX2BH","trackOutbound":true,"trackMailto":true,"trackTel":true,"trackDownload":true,"trackDownloadExtensions":"7z|aac|arc|arj|asf|asx|avi|bin|csv|doc(x|m)?|dot(x|m)?|exe|flv|gif|gz|gzip|hqx|jar|jpe?g|js|mp(2|3|4|e?g)|mov(ie)?|msi|msp|pdf|phps|png|ppt(x|m)?|pot(x|m)?|pps(x|m)?|ppam|sld(x|m)?|thmx|qtm?|ra(m|r)?|sea|sit|tar|tgz|torrent|txt|wav|wma|wmv|wpd|xls(x|m|b)?|xlt(x|m)|xlam|xml|z|zip","trackDomainMode":1},"lazy":{"lazysizes":{"lazyClass":"lazyload","loadedClass":"lazyloaded","loadingClass":"lazyloading","preloadClass":"lazypreload","errorClass":"lazyerror","autosizesClass":"lazyautosizes","srcAttr":"data-src","srcsetAttr":"data-srcset","sizesAttr":"data-sizes","minSize":40,"customMedia":[],"init":true,"expFactor":1.5,"hFac":0.8,"loadMode":1,"loadHidden":true,"ricTimeout":0,"throttleDelay":125,"plugins":[]},"placeholderSrc":"data:image\/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==","preferNative":false,"minified":true,"libraryPath":"\/libraries\/lazysizes"},"was_jwplayer":{"config":{"key":"yh5RZuPQQH3OaPRB\/a\/geG6f46hHX1+s7SszOHZIu3T0qSGJ","clean_source":0,"thumbnails":0},"defaults":{"volume":"","height":"","width":"100%","aspectratio":"16:9","poster":{"live":"https:\/\/www.assembly.ca.gov\/sites\/assembly.ca.gov\/files\/jwplayer\/walkthrough_cover_02_1.jpg","archive":"https:\/\/www.assembly.ca.gov\/sites\/assembly.ca.gov\/files\/jwplayer\/walkthrough_cover_02_2.jpg"}},"files":{"images":{"arrow_white":"https:\/\/www.assembly.ca.gov\/modules\/custom\/was_jwplayer\/css\/arrow-white.png"}}},"search_api_autocomplete":{"search":{"auto_submit":true}},"was_workspace":{"toggled_elements":".tabs__nav, .tabs, div:not(.view-offices-menu) .contextual, .edit-mode, .add-docs-form, .contextual, .view-display-id-block_class_session .views-field-edit-node, .unpublished-menu-item, #environment-indicator","edit_mode_key_toggle":"`","toggled_parent_elements":".node--unpublished.node--view-mode-teaser, .unpublished-menu-item","selector_items":[]},"user":{"uid":0,"permissionsHash":"06c739f6dcef12a85d882bf46cf3cf32d176c4d3b071849cbe28f89ad246822c"}}</script><script src="/sites/assembly.ca.gov/files/js/js_cJF17uhHYne82fqpJY-WNnbu2QR5m7D3XRgV96CJpmk.js?scope=footer&delta=0&language=en&theme=asm_internet_theme&include=eJx1jsEOwyAMQ3-IiZ_ZHXngtpECVJBOYl8_1t2m7uKDn-0EPYcE0bGIMvDJYt1jZUkIuSaow0xIMbZCC7Yx0ycuONSu0HfhimRIuWWW4xLWx7x_bxgnzUwCf2rYFYNtFsTcWus630SBDpPY_a_hFK_hP-I60eIWsEvAYTXWvCuN_o__BrePaYc"></script></body></html>
|
||||||
+21
-3
@@ -32,10 +32,21 @@ def cmd_ingest(args) -> int:
|
|||||||
try:
|
try:
|
||||||
ing = Ingestor(config_path=args.config, force=args.force)
|
ing = Ingestor(config_path=args.config, force=args.force)
|
||||||
sources = _sources_arg(args)
|
sources = _sources_arg(args)
|
||||||
result = ing.ingest(date, sources=sources)
|
if getattr(args, "store_events", False):
|
||||||
|
# Forward-looking chamber calendar -> scheduled_events table. Reuses
|
||||||
|
# the same cache-first pipeline; a recess (empty calendar) is a
|
||||||
|
# no-op, not a failure. Default to the events sources when none are
|
||||||
|
# named (the daily-file sources are a different lane).
|
||||||
|
if not sources:
|
||||||
|
sources = [n for n, s in ing.cfg.sources.items()
|
||||||
|
if s.get("kind") == "events"]
|
||||||
|
result = ing.ingest_events(date, sources=sources,
|
||||||
|
db_path=getattr(args, "db", None))
|
||||||
|
else:
|
||||||
|
result = ing.ingest(date, sources=sources)
|
||||||
ing.close()
|
ing.close()
|
||||||
except SystemExit as exc:
|
except SystemExit as exc:
|
||||||
# config / argparse errors surface here
|
# config / argparse errors surface here
|
||||||
print("ERROR: %s" % exc, file=sys.stderr)
|
print("ERROR: %s" % exc, file=sys.stderr)
|
||||||
return int(exc.code or 2)
|
return int(exc.code or 2)
|
||||||
|
|
||||||
@@ -172,11 +183,18 @@ def build_parser() -> argparse.ArgumentParser:
|
|||||||
pi.add_argument("--date", help="YYYY-MM-DD (default: today)")
|
pi.add_argument("--date", help="YYYY-MM-DD (default: today)")
|
||||||
pi.add_argument("--source",
|
pi.add_argument("--source",
|
||||||
help="Chamber name(s), comma-separated "
|
help="Chamber name(s), comma-separated "
|
||||||
"(assembly_daily_file, senate_daily_file). Default: all.")
|
"(assembly_daily_file, senate_daily_file). Default: all.")
|
||||||
pi.add_argument("--force", action="store_true",
|
pi.add_argument("--force", action="store_true",
|
||||||
help="Re-fetch even if already cached.")
|
help="Re-fetch even if already cached.")
|
||||||
pi.add_argument("--json", action="store_true",
|
pi.add_argument("--json", action="store_true",
|
||||||
help="Emit the result as JSON.")
|
help="Emit the result as JSON.")
|
||||||
|
pi.add_argument("--store-events", dest="store_events", action="store_true",
|
||||||
|
help="Fetch + parse the chamber 'upcoming events' calendar "
|
||||||
|
"and store it in the scheduled_events table "
|
||||||
|
"(forward-looking; the daily-file queue is backward).")
|
||||||
|
pi.add_argument("--db", dest="db", default=None,
|
||||||
|
help="SQLite path for --store-events "
|
||||||
|
"(default: the relational lane's db/daily_file.sqlite).")
|
||||||
|
|
||||||
pl = sub.add_parser("list", help="List configured sources + URLs")
|
pl = sub.add_parser("list", help="List configured sources + URLs")
|
||||||
pc = sub.add_parser("check", help="Cache status for a date (no network)")
|
pc = sub.add_parser("check", help="Cache status for a date (no network)")
|
||||||
|
|||||||
@@ -75,16 +75,42 @@ list:
|
|||||||
marker and a "Floated: ..." reason line
|
marker and a "Floated: ..." reason line
|
||||||
- Zero-scored bills remain in their date-grouped order below
|
- Zero-scored bills remain in their date-grouped order below
|
||||||
|
|
||||||
|
## Forward signal: "what's next" (scheduled_events, schema v2)
|
||||||
|
|
||||||
|
The daily-file bill queue is a *backward* snapshot — leginfo's bill status and
|
||||||
|
bill history are past-action logs, so they cannot answer "what is scheduled
|
||||||
|
next." The only genuine future-dated signal is the chambers' "upcoming events"
|
||||||
|
calendars (`assembly.ca.gov/events/upcoming`, `senate.ca.gov/calendar`). These
|
||||||
|
are fetched (`python3 -m scraper ingest --store-events`), parsed into the
|
||||||
|
`scheduled_events` table (relational schema v2), and surfaced on `/bills` as two
|
||||||
|
new envelope keys:
|
||||||
|
|
||||||
|
- `next_legislative_day` — `{date, kind, chamber, title, source}` (the earliest
|
||||||
|
event on/after `as_of`), or `{date: null, recess: true}` when nothing is
|
||||||
|
scheduled.
|
||||||
|
- `scheduled_events` — the next 7 days of chamber events.
|
||||||
|
|
||||||
|
The "Upcoming" filter is a *forward* view: with `upcoming=1` and no explicit
|
||||||
|
`as_of`, the bill queue floors at **today**, so a stale/older daily-file
|
||||||
|
snapshot can no longer present itself as "upcoming." Omitting `upcoming`
|
||||||
|
(the "All" / history view) still returns every row.
|
||||||
|
|
||||||
|
The dashboard renders a "Next legislative day" line in the freshness band and a
|
||||||
|
"What's next · next 7 days" band. A recess (empty calendar) is an *honest*
|
||||||
|
empty state, not an error.
|
||||||
|
|
||||||
## Tests
|
## Tests
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
python3 -m pytest tests/test_dashboard_ranking.py tests/test_dashboard_api.py
|
python3 -m pytest tests/test_dashboard_ranking.py tests/test_dashboard_api.py \
|
||||||
|
tests/test_parse_events.py
|
||||||
```
|
```
|
||||||
|
|
||||||
32 tests (12 ranking + 20 API), all passing. Pure ranking logic is tested in
|
55 tests (12 ranking + 25 API + 18 parse-events), all passing. Pure ranking
|
||||||
isolation; API handlers are tested with no sockets via the
|
logic is tested in isolation; API handlers are tested with no sockets via the
|
||||||
`handler(req) -> (status, body)` pattern; a live `ThreadingHTTPServer` smoke test
|
`handler(req) -> (status, body)` pattern; a live `ThreadingHTTPServer` smoke test
|
||||||
confirms static + API endpoints serve.
|
confirms static + API endpoints serve. The event parser is tested against
|
||||||
|
inline fixtures plus the saved `samples/assembly_events_*.html` pages.
|
||||||
|
|
||||||
## DASHBOARD.md
|
## DASHBOARD.md
|
||||||
|
|
||||||
@@ -110,3 +136,14 @@ cannot be auto-approved).
|
|||||||
no roster data is supplied.
|
no roster data is supplied.
|
||||||
- Dev auth is a JSON-file local account, not production auth.
|
- Dev auth is a JSON-file local account, not production auth.
|
||||||
- ZIP -> district uses a documented demo dataset until roster ETL lands.
|
- ZIP -> district uses a documented demo dataset until roster ETL lands.
|
||||||
|
- **Forward signal gap:** leginfo's bill status/history are *backward-only*
|
||||||
|
(past actions + status narrative, no future dates), so there is no per-bill
|
||||||
|
"next step" label. The dashboard's forward answer comes solely from the
|
||||||
|
chamber events calendars (`scheduled_events`).
|
||||||
|
- **Recess empty state:** while the Assembly (and at times the Senate) is in
|
||||||
|
final recess, `/events/upcoming` legitimately returns **0 events**. The
|
||||||
|
"What's next" band and `next_legislative_day` then show an honest "recess /
|
||||||
|
unknown" state rather than fabricating a date. This is expected, not a
|
||||||
|
parsing failure.
|
||||||
|
- The 2026-08-31 daily-file snapshot is analysis-only; it is *not* re-surfaced
|
||||||
|
as "upcoming" because the upcoming view floors at today.
|
||||||
+86
-17
@@ -13,6 +13,7 @@ from . import data as _data
|
|||||||
from . import ranking
|
from . import ranking
|
||||||
from . import resolve as _resolve
|
from . import resolve as _resolve
|
||||||
from . import bill_status
|
from . import bill_status
|
||||||
|
from .. import relational
|
||||||
from .data import (load_tag_cloud, list_events,
|
from .data import (load_tag_cloud, list_events,
|
||||||
load_legislators, committee_roster_for)
|
load_legislators, committee_roster_for)
|
||||||
from .ranking import hearing_sort, full_queue, personal_band
|
from .ranking import hearing_sort, full_queue, personal_band
|
||||||
@@ -117,17 +118,25 @@ def _truthy(v):
|
|||||||
|
|
||||||
def handle_bills(req):
|
def handle_bills(req):
|
||||||
q = req.query
|
q = req.query
|
||||||
|
# "Upcoming" is a FORWARD view: when the caller asks for upcoming without
|
||||||
|
# pinning an as_of, floor it at *today*, not the most-recent data date.
|
||||||
|
# Flooring at today is the fix for "upcoming shows the latest PAST day";
|
||||||
|
# a None as_of would fall back to the DB max date and re-introduce it.
|
||||||
|
upcoming = _truthy(q.get("upcoming"))
|
||||||
|
as_of = q.get("as_of")
|
||||||
|
if upcoming and not as_of:
|
||||||
|
as_of = _today_iso()
|
||||||
events = list_events(
|
events = list_events(
|
||||||
db_path=req.svc.db_path,
|
db_path=req.svc.db_path,
|
||||||
upcoming=_truthy(q.get("upcoming")),
|
upcoming=upcoming,
|
||||||
from_date=q.get("from") or q.get("date_from"),
|
from_date=q.get("from") or q.get("date_from"),
|
||||||
to_date=q.get("to") or q.get("date_to"),
|
to_date=q.get("to") or q.get("date_to"),
|
||||||
chamber=q.get("chamber"),
|
chamber=q.get("chamber"),
|
||||||
tags=q_list("tags", q),
|
tags=q_list("tags", q),
|
||||||
q=q.get("q"),
|
q=q.get("q"),
|
||||||
author=q.get("author"),
|
author=q.get("author"),
|
||||||
as_of=q.get("as_of"),
|
as_of=as_of,
|
||||||
)
|
)
|
||||||
if _truthy(q.get("personal")):
|
if _truthy(q.get("personal")):
|
||||||
uid, err = req.require_auth()
|
uid, err = req.require_auth()
|
||||||
if err:
|
if err:
|
||||||
@@ -143,7 +152,7 @@ def handle_bills(req):
|
|||||||
events = full_queue(events)
|
events = full_queue(events)
|
||||||
else:
|
else:
|
||||||
events = personal_band(
|
events = personal_band(
|
||||||
events, int(q.get("band_n") or ranking.PERSONAL_BAND_N))
|
events, int(q.get("band_n") or ranking.PERSONAL_BAND_N))
|
||||||
session = _detect_session(events, req.svc.db_path)
|
session = _detect_session(events, req.svc.db_path)
|
||||||
items = [_client_bill(e) for e in events]
|
items = [_client_bill(e) for e in events]
|
||||||
groups = {}
|
groups = {}
|
||||||
@@ -151,16 +160,76 @@ def handle_bills(req):
|
|||||||
key = e.get("date") or "TBD"
|
key = e.get("date") or "TBD"
|
||||||
groups[key] = groups.get(key, 0) + 1
|
groups[key] = groups.get(key, 0) + 1
|
||||||
chamber_avail = _chamber_availability(items, req.svc.db_path)
|
chamber_avail = _chamber_availability(items, req.svc.db_path)
|
||||||
|
# The bill queue is a backward daily-file snapshot; the chamber "upcoming
|
||||||
|
# events" calendar (scheduled_events, schema v2) is the forward-looking
|
||||||
|
# signal. Surface "next legislative day" + the next-7-days events so a bill
|
||||||
|
# like AB-1099 shows what is scheduled next, not just the past. A None
|
||||||
|
# as_of (the "All"/history view) still answers "next" from today onward.
|
||||||
|
floor = as_of if as_of else _today_iso()
|
||||||
|
next_day = _next_legislative_day(req.svc.db_path, floor, q.get("chamber"))
|
||||||
|
scheduled = _scheduled_events(req.svc.db_path, floor, q.get("chamber"))
|
||||||
body = {
|
body = {
|
||||||
"session": session,
|
"session": session,
|
||||||
"as_of": q.get("as_of") or _now_iso(),
|
"as_of": as_of or floor,
|
||||||
"chamber_availability": chamber_avail,
|
"chamber_availability": chamber_avail,
|
||||||
"items": items,
|
"next_legislative_day": next_day,
|
||||||
"groups": groups,
|
"scheduled_events": scheduled,
|
||||||
}
|
"items": items,
|
||||||
|
"groups": groups,
|
||||||
|
}
|
||||||
return 200, body
|
return 200, body
|
||||||
|
|
||||||
|
|
||||||
|
def _today_iso() -> str:
|
||||||
|
import datetime
|
||||||
|
return datetime.date.today().isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def _next_legislative_day(db_path, as_of, chamber=None):
|
||||||
|
"""Earliest scheduled chamber event on/after as_of (DESIGN section 6.1).
|
||||||
|
|
||||||
|
Degrades gracefully: when the scheduled_events table is absent (pre-v2 DB)
|
||||||
|
or empty for the window (a recess), returns ``{"date": None, "recess":
|
||||||
|
True}`` so the UI can show "next legislative day unknown". Never raises.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
return relational.next_legislative_day(db_path, as_of=as_of,
|
||||||
|
chamber=chamber)
|
||||||
|
except Exception: # noqa: BLE001 - forward source must not break /bills
|
||||||
|
return {"date": None, "recess": True}
|
||||||
|
|
||||||
|
|
||||||
|
def _scheduled_events(db_path, as_of, chamber=None, window_days=7):
|
||||||
|
"""Near-term scheduled chamber events (the "next 7 days" band).
|
||||||
|
|
||||||
|
Returns events from as_of through as_of+window_days, de-duplicated by
|
||||||
|
(date, kind, title). Degrades to [] when the table is absent / empty (a
|
||||||
|
recess). Never raises.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
import datetime
|
||||||
|
start = _today_iso() if not as_of else str(as_of)
|
||||||
|
d0 = datetime.date.fromisoformat(start[:10])
|
||||||
|
end = (d0 + datetime.timedelta(days=window_days)).isoformat()
|
||||||
|
evs = relational.load_scheduled_events(db_path, as_of=start,
|
||||||
|
chamber=chamber)
|
||||||
|
seen = set()
|
||||||
|
out = []
|
||||||
|
for e in evs:
|
||||||
|
if e.get("date") and e["date"] > end:
|
||||||
|
continue
|
||||||
|
k = (e.get("date"), e.get("kind"), e.get("title"))
|
||||||
|
if k in seen:
|
||||||
|
continue
|
||||||
|
seen.add(k)
|
||||||
|
out.append(e)
|
||||||
|
out.sort(key=lambda e: (str(e.get("date") or "z"),
|
||||||
|
str(e.get("kind") or ""),
|
||||||
|
str(e.get("title") or "")))
|
||||||
|
return out
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
return []
|
||||||
|
|
||||||
def _client_bill(ev):
|
def _client_bill(ev):
|
||||||
out = {k: ev.get(k) for k in _data._EVENT_FIELDS}
|
out = {k: ev.get(k) for k in _data._EVENT_FIELDS}
|
||||||
out["hearing_sort"] = hearing_sort(ev.get("hearing_time"))
|
out["hearing_sort"] = hearing_sort(ev.get("hearing_time"))
|
||||||
|
|||||||
@@ -175,16 +175,74 @@
|
|||||||
wireRows(container);
|
wireRows(container);
|
||||||
}
|
}
|
||||||
|
|
||||||
function niceDate(iso) {
|
function niceDate(iso) {
|
||||||
try {
|
try {
|
||||||
var d = new Date(iso + "T00:00:00");
|
var d = new Date(iso + "T00:00:00");
|
||||||
return d.toLocaleDateString(undefined, {
|
return d.toLocaleDateString(undefined, {
|
||||||
weekday: "short", month: "short", day: "numeric",
|
weekday: "short", month: "short", day: "numeric",
|
||||||
year: "numeric" });
|
year: "numeric" });
|
||||||
} catch (e) { return iso; }
|
} catch (e) { return iso; }
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ---------- empty states (DESIGN §6) ---------- */
|
/* ---------- forward "what's next" (scheduled_events, schema v2) -------
|
||||||
|
The daily-file queue is a backward snapshot; the chamber "upcoming
|
||||||
|
events" calendar is the only genuine future-dated signal. A recess
|
||||||
|
(recess: true / empty) is an honest empty state, not an error. */
|
||||||
|
function eventRowHTML(ev) {
|
||||||
|
var url = ev.source_url || "";
|
||||||
|
var kindLabel = ev.kind && ev.kind !== "OTHER"
|
||||||
|
? esc(ev.kind)
|
||||||
|
: (ev.chamber ? ev.chamber : "");
|
||||||
|
return '<div class="ev-row">'
|
||||||
|
+ '<span class="ev-date">' + esc(niceDate(ev.date || ""))
|
||||||
|
+ "</span>"
|
||||||
|
+ '<span class="ev-main">'
|
||||||
|
+ esc(ev.title || "(untitled)")
|
||||||
|
+ (url
|
||||||
|
? " <a href='" + esc(url) + "' target='_blank'"
|
||||||
|
+ " rel='noopener'>details →</a>"
|
||||||
|
: "")
|
||||||
|
+ "</span>"
|
||||||
|
+ (kindLabel ? '<span class="chip ev-kind">'
|
||||||
|
+ kindLabel + "</span>" : "")
|
||||||
|
+ "</div>";
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderNextDay(nd) {
|
||||||
|
var e = el("freshness-next-day");
|
||||||
|
if (!e) return;
|
||||||
|
if (!nd || (nd.recess && !nd.date)) {
|
||||||
|
e.textContent = "Next legislative day: unknown (recess)";
|
||||||
|
e.style.color = "var(--warn)";
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
e.textContent = "Next legislative day: "
|
||||||
|
+ (nd.date ? niceDate(nd.date) : "unknown");
|
||||||
|
e.style.color = "var(--muted)";
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderNextBand(events) {
|
||||||
|
var list = el("next-list");
|
||||||
|
var empty = el("next-empty");
|
||||||
|
var count = el("next-count");
|
||||||
|
if (!list || !empty || !count) return;
|
||||||
|
if (!events.length) {
|
||||||
|
list.innerHTML = "";
|
||||||
|
count.textContent = "";
|
||||||
|
empty.innerHTML =
|
||||||
|
"<h3>Nothing scheduled in the next 7 days</h3>"
|
||||||
|
+ "<p>The chambers are in recess. This band fills in as the "
|
||||||
|
+ "next hearings are posted.</p>";
|
||||||
|
empty.hidden = false;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
empty.hidden = true;
|
||||||
|
count.textContent = events.length
|
||||||
|
+ (events.length === 1 ? " event" : " events");
|
||||||
|
list.innerHTML = events.map(eventRowHTML).join("");
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ---------- empty states (DESIGN §6) ---------- */
|
||||||
function showEmpty(kind, container) {
|
function showEmpty(kind, container) {
|
||||||
var el = el(container);
|
var el = el(container);
|
||||||
var h = "";
|
var h = "";
|
||||||
@@ -250,6 +308,9 @@
|
|||||||
el("freshness-chamber").style.color =
|
el("freshness-chamber").style.color =
|
||||||
c.SENATE === "available" ? "var(--muted)" : "var(--warn)";
|
c.SENATE === "available" ? "var(--muted)" : "var(--warn)";
|
||||||
|
|
||||||
|
renderNextDay(env.next_legislative_day);
|
||||||
|
renderNextBand(env.scheduled_events || []);
|
||||||
|
|
||||||
var items = env.items || [];
|
var items = env.items || [];
|
||||||
state.allItems = items;
|
state.allItems = items;
|
||||||
var band = items.filter(function (i) {
|
var band = items.filter(function (i) {
|
||||||
|
|||||||
@@ -33,10 +33,11 @@
|
|||||||
<main id="app">
|
<main id="app">
|
||||||
<!-- §9 status / context band -->
|
<!-- §9 status / context band -->
|
||||||
<section class="freshness" id="freshness" aria-live="polite">
|
<section class="freshness" id="freshness" aria-live="polite">
|
||||||
<span id="freshness-updated">Loading…</span>
|
<span id="freshness-updated">Loading…</span>
|
||||||
<span id="freshness-session" class="badge"></span>
|
<span id="freshness-session" class="badge"></span>
|
||||||
<span id="freshness-chamber" class="note"></span>
|
<span id="freshness-chamber" class="note"></span>
|
||||||
<button type="button" id="btn-refresh" class="ghost">Refresh</button>
|
<span id="freshness-next-day" class="note"></span>
|
||||||
|
<button type="button" id="btn-refresh" class="ghost">Refresh</button>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
<!-- Controls bar -->
|
<!-- Controls bar -->
|
||||||
@@ -64,9 +65,17 @@
|
|||||||
<input type="checkbox" id="f-reason" title="Show why each
|
<input type="checkbox" id="f-reason" title="Show why each
|
||||||
floated bill is in your view">
|
floated bill is in your view">
|
||||||
</label>
|
</label>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
<!-- Logged-in personal controls (ZIP + watch/opt-in) -->
|
<!-- What's next: forward-looking chamber events calendar (scheduled_events) -->
|
||||||
|
<section class="band next-band" id="next-band" aria-label="Next 7 days">
|
||||||
|
<h2 class="band-title">What's next · next 7 days
|
||||||
|
<span id="next-count" class="count"></span></h2>
|
||||||
|
<div id="next-list" class="list next-list"></div>
|
||||||
|
<div id="next-empty" class="empty-state" hidden></div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<!-- Logged-in personal controls (ZIP + watch/opt-in) -->
|
||||||
<section class="personal-controls" id="personal-controls" hidden>
|
<section class="personal-controls" id="personal-controls" hidden>
|
||||||
<div class="zip-lookup">
|
<div class="zip-lookup">
|
||||||
<label for="zip-input">Where do you live?</label>
|
<label for="zip-input">Where do you live?</label>
|
||||||
|
|||||||
@@ -139,6 +139,19 @@ button.chip-toggle { min-width: 120px; }
|
|||||||
.band-title .count { color: var(--ink); font-weight: 700; }
|
.band-title .count { color: var(--ink); font-weight: 700; }
|
||||||
#personal-band .band-title { color: var(--accent); }
|
#personal-band .band-title { color: var(--accent); }
|
||||||
|
|
||||||
|
/* ---- forward "what's next" band (scheduled_events) ---- */
|
||||||
|
.next-band { background: #fbfcff; }
|
||||||
|
.next-band .band-title { color: var(--accent); }
|
||||||
|
.ev-row {
|
||||||
|
display: grid; grid-template-columns: 120px 1fr auto;
|
||||||
|
gap: 10px; align-items: baseline;
|
||||||
|
padding: 8px 8px 8px 0; border-bottom: 1px solid var(--border);
|
||||||
|
}
|
||||||
|
.ev-date { color: var(--muted); font-size: 13px; white-space: nowrap; }
|
||||||
|
.ev-main { font-weight: 600; }
|
||||||
|
.chip.ev-kind { background: #eef3fb; color: var(--asm); border-color: #cdd9ee; }
|
||||||
|
#freshness-next-day { white-space: nowrap; }
|
||||||
|
|
||||||
.day-heading {
|
.day-heading {
|
||||||
font-size: 13px; color: var(--muted); margin: 14px 0 6px;
|
font-size: 13px; color: var(--muted); margin: 14px 0 6px;
|
||||||
padding-bottom: 4px; border-bottom: 1px solid var(--border); }
|
padding-bottom: 4px; border-bottom: 1px solid var(--border); }
|
||||||
|
|||||||
+60
-4
@@ -97,20 +97,76 @@ class Ingestor:
|
|||||||
timespec="seconds"))
|
timespec="seconds"))
|
||||||
self.log.record(event)
|
self.log.record(event)
|
||||||
|
|
||||||
# -- main API --
|
# -- main API --
|
||||||
def ingest(self, date_iso, source=None, sources=None):
|
def ingest(self, date_iso, source=None, sources=None):
|
||||||
names = sources or ([source] if source else self.cfg.source_names())
|
names = sources or ([source] if source else self.cfg.source_names())
|
||||||
result = IngestResult(date=date_iso)
|
result = IngestResult(date=date_iso)
|
||||||
for name in names:
|
for name in names:
|
||||||
if name not in self.cfg.sources:
|
if name not in self.cfg.sources:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
"Unknown source '%s'. Known: %s"
|
"Unknown source '%s'. Known: %s"
|
||||||
% (name, self.cfg.source_names()))
|
% (name, self.cfg.source_names()))
|
||||||
cr = self._ingest_chamber(name, date_iso)
|
cr = self._ingest_chamber(name, date_iso)
|
||||||
result.chambers.append(cr)
|
result.chambers.append(cr)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
# -- per-chamber --
|
# -- chamber events forward source (scheduled_events, schema v2) --
|
||||||
|
def ingest_events(self, date_iso, source=None, sources=None, db_path=None):
|
||||||
|
"""Fetch + cache + parse + store the chamber "upcoming events" source.
|
||||||
|
|
||||||
|
Like :meth:`ingest`, this is cache-first (a date already on disk is served
|
||||||
|
with no network) and reuses the same source planner (the events source is
|
||||||
|
``kind: events``). It parses the calendar HTML into scheduled-event rows
|
||||||
|
and stores them in the ``scheduled_events`` table.
|
||||||
|
|
||||||
|
``db_path`` defaults to the relational lane DEFAULT_DB so the dashboard
|
||||||
|
can read what this writes. A recess (an empty calendar) is a legitimate
|
||||||
|
no-op, not an error: nothing is cleared and nothing is stored.
|
||||||
|
"""
|
||||||
|
from . import relational
|
||||||
|
from . import parsers
|
||||||
|
|
||||||
|
db = db_path or relational.DEFAULT_DB
|
||||||
|
relational.migrate(db) # ensure the scheduled_events table exists
|
||||||
|
names = sources or ([source] if source else self.cfg.source_names())
|
||||||
|
result = IngestResult(date=date_iso)
|
||||||
|
for name in names:
|
||||||
|
cfg = self.cfg.source(name)
|
||||||
|
if cfg.get("kind") != "events":
|
||||||
|
raise ValueError(
|
||||||
|
"ingest_events needs an events source; %r is kind %r"
|
||||||
|
% (name, cfg.get("kind")))
|
||||||
|
cr = self._ingest_chamber(name, date_iso)
|
||||||
|
|
||||||
|
events = []
|
||||||
|
if cr.status == "ok" and cr.path is not None:
|
||||||
|
try:
|
||||||
|
html = cr.path.read_text(encoding="utf-8",
|
||||||
|
errors="replace")
|
||||||
|
except OSError as e:
|
||||||
|
html = None
|
||||||
|
cr.note += " | cannot read cached page: %s" % e
|
||||||
|
if html:
|
||||||
|
base = getattr(cr, "url", "") or ""
|
||||||
|
events = parsers.parse_events_page(
|
||||||
|
html, chamber=cfg.get("chamber") or cr.source, source=
|
||||||
|
name, base_url=base,)
|
||||||
|
|
||||||
|
# Store when a page was read (fresh or served from cache). A fresh
|
||||||
|
# timestamp lets a re-published calendar refresh its day on the next
|
||||||
|
# run while an unchanged read stays a no-op. An empty parse (recess)
|
||||||
|
# clears no days, so existing scheduled rows are preserved.
|
||||||
|
when = datetime.datetime.now(datetime.timezone.utc).strftime(
|
||||||
|
"%Y-%m-%dT%H:%M:%SZ")
|
||||||
|
summary = relational.store_scheduled_events(events, path=db,
|
||||||
|
fetched_at=when)
|
||||||
|
cr.note += (" | events: %d stored, %d stale cleared, %d total"
|
||||||
|
% (summary["stored"], summary["cleared_stale"],
|
||||||
|
summary["total"]))
|
||||||
|
result.chambers.append(cr)
|
||||||
|
return result
|
||||||
|
|
||||||
|
# -- per-chamber --
|
||||||
def _ingest_chamber(self, name, date_iso):
|
def _ingest_chamber(self, name, date_iso):
|
||||||
cfg = self.cfg.source(name)
|
cfg = self.cfg.source(name)
|
||||||
plan = build_source_plan(name, date_iso, cfg)
|
plan = build_source_plan(name, date_iso, cfg)
|
||||||
|
|||||||
@@ -315,3 +315,109 @@ def parse(input_path: str, *, house: str = "assembly",
|
|||||||
doc_records = [r for r in doc_records if r.record_type != "hearing"]
|
doc_records = [r for r in doc_records if r.record_type != "hearing"]
|
||||||
records.extend(doc_records)
|
records.extend(doc_records)
|
||||||
return records
|
return records
|
||||||
|
|
||||||
|
# -------------------------------------------------------------------
|
||||||
|
# Chamber "upcoming events" calendar (scheduled_events source, v2)
|
||||||
|
#
|
||||||
|
# The chamber calendar (assembly.ca.gov/events/upcoming, senate.ca.gov/
|
||||||
|
# calendar) is a Drupal "views" table (<table class="cols-3"> with Date /
|
||||||
|
# Title / Video columns). Each row is a scheduled hearing / floor session --
|
||||||
|
# the only *forward-looking* signal (the daily file is a backward snapshot).
|
||||||
|
# Rows are stored in scheduled_events (relational.store_scheduled_events).
|
||||||
|
# An empty page (a recess) is a legitimate empty result, not an error.
|
||||||
|
# -------------------------------------------------------------------
|
||||||
|
|
||||||
|
# Title keywords that classify an event's kind (best-effort, never crash).
|
||||||
|
_EV_KIND_PATTERNS = (
|
||||||
|
(re.compile(r"\bfloor\b", re.I), "FLOOR"),
|
||||||
|
(re.compile(r"\bcommittee\b", re.I), "COMMITTEE"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def classify_event_kind(title):
|
||||||
|
"""Map an event title to a kind: FLOOR / COMMITTEE / OTHER."""
|
||||||
|
t = title or ""
|
||||||
|
for pat, kind in _EV_KIND_PATTERNS:
|
||||||
|
if pat.search(t):
|
||||||
|
return kind
|
||||||
|
return "OTHER"
|
||||||
|
|
||||||
|
|
||||||
|
def parse_events_page(html, chamber="assembly", source="assembly_events",
|
||||||
|
base_url="", parser="lxml"):
|
||||||
|
"""Parse a chamber events calendar page into scheduled-event dicts.
|
||||||
|
|
||||||
|
Returns a list of dicts with keys: date (ISO YYYY-MM-DD), chamber
|
||||||
|
(upper), kind (FLOOR|COMMITTEE|OTHER), title, source_url. An empty /
|
||||||
|
recess page yields []. Rows without a resolvable date are skipped
|
||||||
|
(logged). Header rows (<th>) are ignored.
|
||||||
|
"""
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
|
engine = parser if parser in ("lxml", "html.parser") else "html.parser"
|
||||||
|
try:
|
||||||
|
soup = BeautifulSoup(html, engine)
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
log.error("events: failed to parse document (%s): %s",
|
||||||
|
type(e).__name__, e)
|
||||||
|
return []
|
||||||
|
|
||||||
|
out = []
|
||||||
|
# Fall back to any <tr> carrying a <time> cell when table.cols-3 is
|
||||||
|
# absent, so a layout change still yields rows rather than a silent
|
||||||
|
# empty page.
|
||||||
|
base = (base_url or "").rstrip("/")
|
||||||
|
host = base.rsplit("/", 2)[0] if base else "https://www.assembly.ca.gov"
|
||||||
|
tables = soup.select("table.cols-3")
|
||||||
|
rows = [tr for tbl in tables for tr in tbl.find_all("tr")]
|
||||||
|
if not rows:
|
||||||
|
rows = [tr for tr in soup.find_all("tr") if tr.find("time")]
|
||||||
|
for i, tr in enumerate(rows):
|
||||||
|
# Skip header rows (column labels render as <th>).
|
||||||
|
if tr.find("th"):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
time_el = tr.find("time")
|
||||||
|
if time_el is None:
|
||||||
|
raise ValueError("event row has no date cell")
|
||||||
|
# Prefer the datetime attr: "2026-09-01T00:00:00-07:00".
|
||||||
|
dt = time_el.get("datetime") or ""
|
||||||
|
date = dt[:10] if len(dt) >= 10 and dt[4:5] == "-" else None
|
||||||
|
if not date:
|
||||||
|
date = normalise_date(time_el.get_text(strip=True))
|
||||||
|
|
||||||
|
title_td = tr.select_one("td.views-field-title")
|
||||||
|
if title_td is not None:
|
||||||
|
title = title_td.get_text(" ", strip=True)
|
||||||
|
else:
|
||||||
|
t = tr.find("time")
|
||||||
|
if t is not None:
|
||||||
|
t.decompose()
|
||||||
|
title = tr.get_text(" ", strip=True)
|
||||||
|
if not date or not title:
|
||||||
|
raise ValueError("event row has no date or title")
|
||||||
|
|
||||||
|
# source_url: the row's own link (event / video detail) when
|
||||||
|
# present, else the calendar page (base_url) as provenance.
|
||||||
|
a = tr.find("a", href=True)
|
||||||
|
source_url = None
|
||||||
|
if a is not None:
|
||||||
|
href = (a.get("href") or "").strip()
|
||||||
|
if href.startswith("http"):
|
||||||
|
source_url = href
|
||||||
|
elif href.startswith("/"):
|
||||||
|
source_url = host + href
|
||||||
|
if not source_url:
|
||||||
|
source_url = base or None
|
||||||
|
|
||||||
|
out.append({
|
||||||
|
"date": date,
|
||||||
|
"chamber": chamber.strip().upper(),
|
||||||
|
"kind": classify_event_kind(title),
|
||||||
|
"title": title,
|
||||||
|
"source_url": source_url,
|
||||||
|
"source": source,
|
||||||
|
})
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
log.warning("skip malformed event row #%d: %s", i, e)
|
||||||
|
return out
|
||||||
+159
-17
@@ -51,7 +51,11 @@ DEFAULT_DB = os.path.join(_REPO_ROOT, "db", "daily_file.sqlite")
|
|||||||
FLAT_TABLE = "daily_file_records"
|
FLAT_TABLE = "daily_file_records"
|
||||||
|
|
||||||
#: Schema version stamped into ``schema_migrations`` by :func:`migrate`.
|
#: Schema version stamped into ``schema_migrations`` by :func:`migrate`.
|
||||||
SCHEMA_VERSION = 1
|
# v1: bills/committees/members/committee_members/bill_committees + flat view.
|
||||||
|
# v2: adds scheduled_events (the chamber "upcoming events" forward source — the
|
||||||
|
# only genuine future-dated signal; see design/API_CONTRACT.md §5 + the
|
||||||
|
# "what's next" work). Kept OUT of the frozen flat daily_file_records view.
|
||||||
|
SCHEMA_VERSION = 2
|
||||||
|
|
||||||
#: The flat columns QA + downstream rely on. Imported so the flat-preserving
|
#: The flat columns QA + downstream rely on. Imported so the flat-preserving
|
||||||
#: view is byte-for-byte aligned with the contract the rest of the pipeline
|
#: view is byte-for-byte aligned with the contract the rest of the pipeline
|
||||||
@@ -211,9 +215,27 @@ def _create_schema(conn: sqlite3.Connection) -> None:
|
|||||||
applied_at TEXT,
|
applied_at TEXT,
|
||||||
note TEXT,
|
note TEXT,
|
||||||
PRIMARY KEY (version)
|
PRIMARY KEY (version)
|
||||||
);
|
);
|
||||||
"""
|
|
||||||
)
|
-- v2: the chamber "upcoming events" forward source. This is the ONLY
|
||||||
|
-- genuine *future-dated* signal (the daily file is a backward snapshot;
|
||||||
|
-- leginfo bill status/history is backward-only). It holds the next
|
||||||
|
-- scheduled floor/committee events parsed from the chamber events
|
||||||
|
-- calendar (assembly.ca.gov/events, senate.ca.gov/calendar). It is
|
||||||
|
-- deliberately OUT of the frozen flat daily_file_records view.
|
||||||
|
CREATE TABLE IF NOT EXISTS scheduled_events (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
date TEXT NOT NULL, -- ISO YYYY-MM-DD (event day)
|
||||||
|
chamber TEXT NOT NULL, -- ASSEMBLY | SENATE
|
||||||
|
kind TEXT, -- FLOOR | COMMITTEE | OTHER
|
||||||
|
title TEXT NOT NULL, -- human event title
|
||||||
|
source_url TEXT, -- where the event came from
|
||||||
|
fetched_at TEXT, -- when this page was read
|
||||||
|
source TEXT, -- assembly_events | senate_events
|
||||||
|
UNIQUE (date, chamber, kind, title, source)
|
||||||
|
);
|
||||||
|
"""
|
||||||
|
)
|
||||||
# Flat-preserving view: a drop-in mirror of the table QA/UI read. Column
|
# Flat-preserving view: a drop-in mirror of the table QA/UI read. Column
|
||||||
# order and row set are identical to daily_file_records, ordered
|
# order and row set are identical to daily_file_records, ordered
|
||||||
# deterministically so the "flat contract unchanged" test is stable.
|
# deterministically so the "flat contract unchanged" test is stable.
|
||||||
@@ -229,13 +251,13 @@ def _create_schema(conn: sqlite3.Connection) -> None:
|
|||||||
|
|
||||||
def _stamp_migration(conn: sqlite3.Connection, version: int = SCHEMA_VERSION) -> None:
|
def _stamp_migration(conn: sqlite3.Connection, version: int = SCHEMA_VERSION) -> None:
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"INSERT OR IGNORE INTO schema_migrations (version, applied_at, note) "
|
"INSERT OR IGNORE INTO schema_migrations (version, applied_at, note) "
|
||||||
"VALUES (?, ?, ?)",
|
"VALUES (?, ?, ?)",
|
||||||
(version,
|
(version,
|
||||||
__import__("datetime").date.today().isoformat(),
|
__import__("datetime").date.today().isoformat(),
|
||||||
"relational schema: bills/committees/members/committee_members/"
|
"relational schema: bills/committees/members/committee_members/"
|
||||||
"bill_committees + flat view"),
|
"bill_committees + flat view + scheduled_events (v2)"),
|
||||||
)
|
)
|
||||||
conn.commit()
|
conn.commit()
|
||||||
|
|
||||||
|
|
||||||
@@ -514,12 +536,132 @@ def sample_selects(path: str = DEFAULT_DB, limit: int = 5) -> Dict[str, Any]:
|
|||||||
"JOIN committees c ON c.name_key = cm.committee "
|
"JOIN committees c ON c.name_key = cm.committee "
|
||||||
"JOIN members m ON m.name_norm = cm.member "
|
"JOIN members m ON m.name_norm = cm.member "
|
||||||
"ORDER BY c.name_raw, m.full_name LIMIT ?", (limit,)),
|
"ORDER BY c.name_raw, m.full_name LIMIT ?", (limit,)),
|
||||||
"committees_without_roster": q(
|
"committees_without_roster": q(
|
||||||
"SELECT name_raw, chamber, session FROM committees c "
|
"SELECT name_raw, chamber, session FROM committees c "
|
||||||
"WHERE NOT EXISTS (SELECT 1 FROM committee_members cm "
|
"WHERE NOT EXISTS (SELECT 1 FROM committee_members cm "
|
||||||
" WHERE cm.committee = c.name_key "
|
" WHERE cm.committee = c.name_key "
|
||||||
" AND cm.chamber = c.chamber "
|
" AND cm.chamber = c.chamber "
|
||||||
" AND cm.session = c.session)"),
|
" AND cm.session = c.session)"),
|
||||||
}
|
}
|
||||||
finally:
|
finally:
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# scheduled_events: the chamber "upcoming events" forward source (v2)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _scheduled_events_table(conn: sqlite3.Connection, name: str = "scheduled_events") -> bool:
|
||||||
|
cur = conn.execute(
|
||||||
|
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?",
|
||||||
|
(name,))
|
||||||
|
return cur.fetchone() is not None
|
||||||
|
|
||||||
|
|
||||||
|
def store_scheduled_events(events: Iterable[Dict[str, Any]],
|
||||||
|
path: str = DEFAULT_DB,
|
||||||
|
fetched_at: str = "") -> Dict[str, Any]:
|
||||||
|
"""Insert parsed chamber events into ``scheduled_events`` idempotently.
|
||||||
|
|
||||||
|
Each ``event`` is a dict with keys ``date`` (ISO), ``chamber`` (upper),
|
||||||
|
``kind`` (``FLOOR``/``COMMITTEE``/``OTHER``), ``title``, ``source_url``,
|
||||||
|
``source``. The natural key ``(date, chamber, kind, title, source)`` makes
|
||||||
|
re-runs a no-op (``INSERT OR IGNORE``), so the existing-cron behaviour
|
||||||
|
("re-parse doesn't re-hit") holds: a second parse of the same calendar day
|
||||||
|
adds no rows. Ensures the table exists (via :func:`_create_schema`). Returns
|
||||||
|
a small summary ``{stored, cleared_stale, total, fetched_at}``.
|
||||||
|
"""
|
||||||
|
conn = connect(path)
|
||||||
|
try:
|
||||||
|
_create_schema(conn) # CREATE IF NOT EXISTS; cheap + idempotent
|
||||||
|
# Drop a stale day's rows first so a re-published calendar (a moved or
|
||||||
|
# renamed hearing) reflects the latest read rather than a frozen
|
||||||
|
# snapshot. Only that day is cleared; other days are untouched.
|
||||||
|
days = {str(e.get("date")) for e in events if e.get("date")}
|
||||||
|
deleted = 0
|
||||||
|
for d in days:
|
||||||
|
cur = conn.execute(
|
||||||
|
"DELETE FROM scheduled_events WHERE date=? AND fetched_at<>?",
|
||||||
|
(d, fetched_at))
|
||||||
|
deleted += cur.rowcount
|
||||||
|
rows = []
|
||||||
|
for e in events:
|
||||||
|
if not e.get("date") or not e.get("title"):
|
||||||
|
continue
|
||||||
|
rows.append((
|
||||||
|
str(e["date"]).strip(),
|
||||||
|
(e.get("chamber") or "unknown").strip().upper(),
|
||||||
|
(e.get("kind") or "OTHER").strip().upper() or "OTHER",
|
||||||
|
str(e["title"]).strip(),
|
||||||
|
e.get("source_url") or None,
|
||||||
|
fetched_at or None,
|
||||||
|
(e.get("source") or "assembly_events").strip()
|
||||||
|
or "assembly_events",
|
||||||
|
))
|
||||||
|
if rows:
|
||||||
|
cur = conn.executemany(
|
||||||
|
"INSERT OR IGNORE INTO scheduled_events "
|
||||||
|
"(date, chamber, kind, title, source_url, fetched_at, "
|
||||||
|
"source) VALUES (?,?,?,?,?,?,?)",
|
||||||
|
rows)
|
||||||
|
stored = cur.rowcount
|
||||||
|
else:
|
||||||
|
stored = 0
|
||||||
|
conn.commit()
|
||||||
|
total = conn.execute(
|
||||||
|
"SELECT COUNT(*) FROM scheduled_events").fetchone()[0]
|
||||||
|
return {"stored": stored, "cleared_stale": deleted,
|
||||||
|
"total": total, "fetched_at": fetched_at or None}
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def load_scheduled_events(path: str = DEFAULT_DB,
|
||||||
|
as_of: Optional[str] = None,
|
||||||
|
chamber: Optional[str] = None,
|
||||||
|
limit: int = 200) -> List[Dict[str, Any]]:
|
||||||
|
"""Return scheduled (chamber) events, optionally future-dated from ``as_of``.
|
||||||
|
|
||||||
|
A ``None``/absent ``as_of`` keeps every event; a date string keeps
|
||||||
|
``date >= as_of`` (a T-1 event is excluded, matching API_CONTRACT §5).
|
||||||
|
Degrades to ``[]`` when the table does not exist (pre-v2 DBs / fresh schema).
|
||||||
|
"""
|
||||||
|
conn = connect(path, foreign_keys=False)
|
||||||
|
try:
|
||||||
|
if not _scheduled_events_table(conn):
|
||||||
|
return []
|
||||||
|
sql = ("SELECT date, chamber, kind, title, source_url, fetched_at, "
|
||||||
|
"source FROM scheduled_events WHERE 1=1")
|
||||||
|
clauses: list = []
|
||||||
|
args: list = []
|
||||||
|
if as_of:
|
||||||
|
clauses.append("date >= ?")
|
||||||
|
args.append(str(as_of))
|
||||||
|
if chamber:
|
||||||
|
clauses.append("UPPER(chamber) = ?")
|
||||||
|
args.append(chamber.strip().upper())
|
||||||
|
if clauses:
|
||||||
|
sql += " AND " + " AND ".join(clauses)
|
||||||
|
sql += " ORDER BY date, chamber, kind, title LIMIT ?"
|
||||||
|
args.append(int(limit))
|
||||||
|
rows = conn.execute(sql, args).fetchall()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
return [dict(r) for r in rows]
|
||||||
|
|
||||||
|
|
||||||
|
def next_legislative_day(path: str = DEFAULT_DB,
|
||||||
|
as_of: Optional[str] = None,
|
||||||
|
chamber: Optional[str] = None) -> Dict[str, Any]:
|
||||||
|
"""The earliest scheduled event on/after ``as_of`` (DESIGN §6.1 "next
|
||||||
|
legislative day"), or a ``{date: None, recess: True}`` marker when nothing is
|
||||||
|
scheduled (a genuine recess).
|
||||||
|
|
||||||
|
This is the dashboard's "what's next" answer when the daily-file bill queue
|
||||||
|
has no future rows: it surfaces the chamber events calendar. ``recess`` is
|
||||||
|
True when there is no event after ``as_of``.
|
||||||
|
"""
|
||||||
|
events = load_scheduled_events(path, as_of=as_of, chamber=chamber, limit=1)
|
||||||
|
if not events:
|
||||||
|
return {"date": None, "recess": True}
|
||||||
|
return {**events[0], "recess": False}
|
||||||
+21
-6
@@ -16,18 +16,33 @@ SENATE_PDF_RE = re.compile(
|
|||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class SourcePlan:
|
class SourcePlan:
|
||||||
source: str # "assembly_daily_file" | "senate_daily_file"
|
source: str # "assembly_daily_file" | "senate_daily_file"
|
||||||
date: str # requested YYYY-MM-DD
|
date: str # requested YYYY-MM-DD
|
||||||
kind: str # "html" | "pdf"
|
kind: str # "html" | "pdf" | "events"
|
||||||
primary_url: str # URL that is actually fetched
|
primary_url: str # URL that is actually fetched
|
||||||
ext: str # cache extension ("html" | "pdf")
|
ext: str # cache extension ("html" | "pdf")
|
||||||
pdf_url: Optional[str] = None
|
pdf_url: Optional[str] = None
|
||||||
as_of: Optional[str] = None # date the live page itself represents
|
as_of: Optional[str] = None # date the live page itself represents
|
||||||
extra_urls: list = field(default_factory=list)
|
extra_urls: list = field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
def build_source_plan(name: str, date_iso: str, src_cfg: dict) -> SourcePlan:
|
def build_source_plan(name: str, date_iso: str, src_cfg: dict) -> SourcePlan:
|
||||||
kind = src_cfg.get("kind")
|
kind = src_cfg.get("kind")
|
||||||
|
if kind == "events":
|
||||||
|
# The chamber "upcoming events" calendar is a live HTML page (a
|
||||||
|
# forward-looking source, distinct from the daily file). Its URL is
|
||||||
|
# constant; the date only drives the cache key (the page may list
|
||||||
|
# future-dated events, none of which is the --date value itself).
|
||||||
|
url = src_cfg.get("url") or src_cfg.get("index_url")
|
||||||
|
return SourcePlan(
|
||||||
|
source=name,
|
||||||
|
date=date_iso,
|
||||||
|
kind="events",
|
||||||
|
primary_url=url,
|
||||||
|
ext="html",
|
||||||
|
as_of=date_iso,
|
||||||
|
)
|
||||||
|
|
||||||
if kind == "html":
|
if kind == "html":
|
||||||
# Assembly: constant URL, always the current live file.
|
# Assembly: constant URL, always the current live file.
|
||||||
return SourcePlan(
|
return SourcePlan(
|
||||||
|
|||||||
+21
-4
@@ -38,14 +38,31 @@ echo "==================================================================="
|
|||||||
echo "[run_ingest] ${DATE} root=${ROOT} py=${PY}"
|
echo "[run_ingest] ${DATE} root=${ROOT} py=${PY}"
|
||||||
echo "==================================================================="
|
echo "==================================================================="
|
||||||
|
|
||||||
# -u: unbuffered; the CLI prints the human-readable summary.
|
# Lane 1: the daily-file bill queue (backward snapshot). -u: unbuffered; prints
|
||||||
|
# the human-readable summary.
|
||||||
"${PY}" -u -m scraper ingest --date "${DATE}" ${FORCE}
|
"${PY}" -u -m scraper ingest --date "${DATE}" ${FORCE}
|
||||||
EXIT=$?
|
EXIT=$?
|
||||||
|
|
||||||
case "${EXIT}" in
|
case "${EXIT}" in
|
||||||
0) echo "[run_ingest] DONE ${DATE} (ok / cached / no publication -- all expected)" ;;
|
0) echo "[run_ingest] DONE ${DATE} daily-file (ok / cached / no publication -- all expected)" ;;
|
||||||
1) echo "[run_ingest] FAILED ${DATE} see cache/request_log.jsonl for the attempt trail" >&2 ;;
|
1) echo "[run_ingest] FAILED ${DATE} daily-file see cache/request_log.jsonl for the attempt trail" >&2 ;;
|
||||||
*) echo "[run_ingest] ERROR ${DATE} (exit ${EXIT})" >&2 ;;
|
*) echo "[run_ingest] ERROR ${DATE} daily-file (exit ${EXIT})" >&2 ;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
|
# Lane 2: the chamber "upcoming events" calendar -> scheduled_events table
|
||||||
|
# (the forward-looking "what's next" signal; the daily file alone is backward).
|
||||||
|
# A recess (empty calendar) is a no-op, not a failure. CALI_SCRAPER_SKIP_EVENTS=
|
||||||
|
# 1 disables this lane. The daily-file EXIT still governs cron mail: events are
|
||||||
|
# best-effort and degrade to an empty forward band rather than page.
|
||||||
|
if [ "${CALI_SCRAPER_SKIP_EVENTS:-0}" != "1" ]; then
|
||||||
|
"${PY}" -u -m scraper ingest --date "${DATE}" ${FORCE} --store-events \
|
||||||
|
--db "db/daily_file.sqlite"
|
||||||
|
EVT=$?
|
||||||
|
case "${EVT}" in
|
||||||
|
0) echo "[run_ingest] DONE ${DATE} events (ok / recess / cached)" ;;
|
||||||
|
1) echo "[run_ingest] WARNING ${DATE} events ingest hiccup (non-fatal; see request_log)" >&2 ;;
|
||||||
|
*) echo "[run_ingest] WARNING ${DATE} events unexpected exit ${EVT} (non-fatal)" >&2 ;;
|
||||||
|
esac
|
||||||
|
fi
|
||||||
|
|
||||||
exit "${EXIT}"
|
exit "${EXIT}"
|
||||||
@@ -214,6 +214,94 @@ def test_empty_no_tagged_bills_and_chamber_pending(tmp_path, services):
|
|||||||
assert status == 200 and body["items"] == []
|
assert status == 200 and body["items"] == []
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# ---- forward "upcoming" source: scheduled_events + next_legislative_day ----
|
||||||
|
|
||||||
|
|
||||||
|
def _events_db(tmp_path):
|
||||||
|
"""A small DB whose flat snapshot is in the PAST but whose chamber
|
||||||
|
scheduled_events table holds future-dated events (the forward source)."""
|
||||||
|
p = str(tmp_path / "fwd.sqlite")
|
||||||
|
write_sqlite([
|
||||||
|
BillAction(date="2026-08-31", house="assembly", record_type="bill",
|
||||||
|
bill_number="AB 1099", bill_id="202520260AB1099",
|
||||||
|
session="2025-2026", chamber_origin="assembly",
|
||||||
|
author="Bryan", short_title="Developmental services",
|
||||||
|
description="Read third time. Passed. Ordered to the Assembly.",
|
||||||
|
source="ASSEMBLY_DAILY_FILE")], p, table=relational.FLAT_TABLE)
|
||||||
|
relational.migrate(p)
|
||||||
|
relational.store_scheduled_events([
|
||||||
|
{"date": "2026-09-08", "chamber": "ASSEMBLY", "kind": "FLOOR",
|
||||||
|
"title": "Assembly Floor Session",
|
||||||
|
"source_url": "https://www.assembly.ca.gov/events/upcoming",
|
||||||
|
"source": "assembly_events"},
|
||||||
|
{"date": "2026-09-09", "chamber": "ASSEMBLY", "kind": "COMMITTEE",
|
||||||
|
"title": "Judiciary Committee", "source": "assembly_events"},
|
||||||
|
], path=p, fetched_at="2026-09-06T00:00:00Z")
|
||||||
|
return p
|
||||||
|
|
||||||
|
|
||||||
|
def test_upcoming_defaults_to_not_past(tmp_path):
|
||||||
|
"""The bug that motivated this work: with upcoming=1 and no as_of, the
|
||||||
|
bill queue must floor at *today* (the past snapshot drops out), not at the
|
||||||
|
most-recent data date (which would re-surface yesterday as "upcoming")."""
|
||||||
|
fwd = _events_db(tmp_path)
|
||||||
|
svc = Services(fwd, PrefsStore(str(tmp_path / "p.json")),
|
||||||
|
LocalAuth(str(tmp_path / "a.json")))
|
||||||
|
status, body = api.run(_req("GET", "/api/v1/bills", svc,
|
||||||
|
query={"upcoming": "1"}))
|
||||||
|
assert status == 200, body
|
||||||
|
# the 08-31 snapshot is in the past relative to today -> not "upcoming"
|
||||||
|
assert body["items"] == []
|
||||||
|
# ...but the chamber calendar still answers "next":
|
||||||
|
assert body["next_legislative_day"]["date"] is not None
|
||||||
|
assert body["next_legislative_day"]["recess"] is False
|
||||||
|
assert len(body["scheduled_events"]) >= 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_envelope_carries_forward_fields(tmp_path):
|
||||||
|
"""Envelope must carry the forward fields with a 7-day window."""
|
||||||
|
fwd = _events_db(tmp_path)
|
||||||
|
svc = Services(fwd, PrefsStore(str(tmp_path / "p.json")),
|
||||||
|
LocalAuth(str(tmp_path / "a.json")))
|
||||||
|
status, body = api.run(_req("GET", "/api/v1/bills", svc,
|
||||||
|
query={"upcoming": "1", "as_of": "2026-09-06"}))
|
||||||
|
assert status == 200, body
|
||||||
|
for key in ("next_legislative_day", "scheduled_events"):
|
||||||
|
assert key in body
|
||||||
|
nd = body["next_legislative_day"]
|
||||||
|
assert nd["date"] == "2026-09-08"
|
||||||
|
assert nd["kind"] == "FLOOR"
|
||||||
|
# 7-day window from 09-06 reaches 09-08 and 09-09, not further:
|
||||||
|
dates = {e["date"] for e in body["scheduled_events"]}
|
||||||
|
assert "2026-09-08" in dates and "2026-09-09" in dates
|
||||||
|
|
||||||
|
|
||||||
|
def test_next_day_recess_when_no_forward_events(tmp_path):
|
||||||
|
"""A pre-v2-style DB with no scheduled_events degrades to a recess marker
|
||||||
|
(never a 5xx): next_legislative_day is a {date: None, recess: True}."""
|
||||||
|
p = str(tmp_path / "empty.sqlite")
|
||||||
|
relational.migrate(p)
|
||||||
|
svc = Services(p, PrefsStore(str(tmp_path / "p.json")),
|
||||||
|
LocalAuth(str(tmp_path / "a.json")))
|
||||||
|
status, body = api.run(_req("GET", "/api/v1/bills", svc,
|
||||||
|
query={"upcoming": "1"}))
|
||||||
|
assert status == 200, body
|
||||||
|
assert body["next_legislative_day"] == {"date": None, "recess": True}
|
||||||
|
assert body["scheduled_events"] == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_all_view_still_returns_history(tmp_path):
|
||||||
|
"""Omitting upcoming (the "All"/history view) must still return the past
|
||||||
|
snapshot rows -- the forward fix only changes the *upcoming* view."""
|
||||||
|
fwd = _events_db(tmp_path)
|
||||||
|
svc = Services(fwd, PrefsStore(str(tmp_path / "p.json")),
|
||||||
|
LocalAuth(str(tmp_path / "a.json")))
|
||||||
|
status, body = api.run(_req("GET", "/api/v1/bills", svc, query={}))
|
||||||
|
assert status == 200
|
||||||
|
assert len(body["items"]) >= 1
|
||||||
|
assert any(i["bill_number"] == "AB 1099" for i in body["items"])
|
||||||
|
|
||||||
# ---- /tags, /legislators, /districts (§4.1 / §3.2 / §4.2) -----------------
|
# ---- /tags, /legislators, /districts (§4.1 / §3.2 / §4.2) -----------------
|
||||||
|
|
||||||
def test_tags_cloud(services):
|
def test_tags_cloud(services):
|
||||||
|
|||||||
@@ -0,0 +1,217 @@
|
|||||||
|
"""Unit tests for scraper.parsers.parse_events_page (the chamber "upcoming"
|
||||||
|
events calendar parser, schema v2 / scheduled_events forward source).
|
||||||
|
|
||||||
|
The bill queue (daily file) is a backward snapshot, so the chamber calendar is
|
||||||
|
the only forward-looking signal. These tests cover the new parser branches:
|
||||||
|
date-from-<time> attribute, kind classification, source-url resolution, header
|
||||||
|
row skipping, the <tr>-with-<time> fallback, and the empty/recess case.
|
||||||
|
|
||||||
|
The parser is the coverage-gated module, so these aim for complete branch
|
||||||
|
coverage of parse_events_page + classify_event_kind.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
from scraper import parsers
|
||||||
|
from scraper.parsers import parse_events_page, classify_event_kind
|
||||||
|
|
||||||
|
|
||||||
|
# ---- a populated "views" table, matching the live assembly.ca.gov layout ----
|
||||||
|
POPULATED_HTML = """
|
||||||
|
<html><body>
|
||||||
|
<div class="view-content">
|
||||||
|
<table class="cols-3">
|
||||||
|
<thead><tr>
|
||||||
|
<th class="views-field views-field-media-source-pub-date">Date</th>
|
||||||
|
<th class="views-field views-field-title">Title</th>
|
||||||
|
<th class="views-field">Video</th>
|
||||||
|
</tr></thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td class="views-field views-field-media-source-pub-date">
|
||||||
|
<time datetime="2026-09-10T00:00:00-07:00">September 10, 2026</time>
|
||||||
|
</td>
|
||||||
|
<td class="views-field views-field-title">
|
||||||
|
Assembly Floor Session</td>
|
||||||
|
<td class="views-field">
|
||||||
|
<a href="/media/assembly-floor-session-20260910">watch now</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td class="views-field views-field-media-source-pub-date">
|
||||||
|
<time datetime="2026-09-11T00:00:00-07:00">September 11, 2026</time>
|
||||||
|
</td>
|
||||||
|
<td class="views-field views-field-title">
|
||||||
|
Judiciary Committee</td>
|
||||||
|
<td class="views-field"></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td class="views-field views-field-media-source-pub-date">
|
||||||
|
<time datetime="2026-09-12T00:00:00-07:00">September 12, 2026</time>
|
||||||
|
</td>
|
||||||
|
<td class="views-field views-field-title">
|
||||||
|
Press Conference</td>
|
||||||
|
<td class="views-field"><a href="http://example.com/x">link</a></td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</body></html>
|
||||||
|
"""
|
||||||
|
|
||||||
|
# A recess: the table is present but carries no data rows (empty view-content).
|
||||||
|
RECESS_HTML = """
|
||||||
|
<html><body>
|
||||||
|
<div class="view-content">
|
||||||
|
<table class="cols-3">
|
||||||
|
<thead><tr>
|
||||||
|
<th class="views-field views-field-media-source-pub-date">Date</th>
|
||||||
|
<th class="views-field views-field-title">Title</th>
|
||||||
|
</tr></thead>
|
||||||
|
<tbody></tbody>
|
||||||
|
</table>
|
||||||
|
<p>There are currently no upcoming committee hearings scheduled.</p>
|
||||||
|
</body></html>
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Layout with no <table class="cols-3">: rows must be recovered by <time>.
|
||||||
|
NO_COLS3_HTML = """
|
||||||
|
<html><body>
|
||||||
|
<tr>
|
||||||
|
<td><time datetime="2026-09-13T00:00:00-07:00">September 13, 2026</time></td>
|
||||||
|
<td>Budget Committee</td>
|
||||||
|
</tr>
|
||||||
|
</body></html>
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
# ---- kind classification ---------------------------------------------------
|
||||||
|
|
||||||
|
def test_classify_floor():
|
||||||
|
assert classify_event_kind("Assembly Floor Session") == "FLOOR"
|
||||||
|
|
||||||
|
|
||||||
|
def test_classify_committee():
|
||||||
|
assert classify_event_kind("Judiciary Committee") == "COMMITTEE"
|
||||||
|
|
||||||
|
|
||||||
|
def test_classify_other():
|
||||||
|
assert classify_event_kind("Press Conference") == "OTHER"
|
||||||
|
|
||||||
|
|
||||||
|
def test_classify_none():
|
||||||
|
assert classify_event_kind(None) == "OTHER"
|
||||||
|
|
||||||
|
|
||||||
|
# ---- populated page --------------------------------------------------------
|
||||||
|
|
||||||
|
def test_populated_page_parses_three_events():
|
||||||
|
evs = parse_events_page(POPULATED_HTML, chamber="assembly",
|
||||||
|
source="assembly_events",
|
||||||
|
base_url="https://www.assembly.ca.gov/events/upcoming")
|
||||||
|
assert len(evs) == 3
|
||||||
|
by_date = {e["date"]: e for e in evs}
|
||||||
|
assert by_date["2026-09-10"]["kind"] == "FLOOR"
|
||||||
|
assert by_date["2026-09-11"]["kind"] == "COMMITTEE"
|
||||||
|
assert by_date["2026-09-12"]["kind"] == "OTHER"
|
||||||
|
|
||||||
|
|
||||||
|
def test_dates_come_from_time_attribute():
|
||||||
|
evs = parse_events_page(POPULATED_HTML)
|
||||||
|
dates = sorted(e["date"] for e in evs)
|
||||||
|
assert dates == ["2026-09-10", "2026-09-11", "2026-09-12"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_chamber_normalised_to_upper():
|
||||||
|
evs = parse_events_page(POPULATED_HTML, chamber=" Assembly ")
|
||||||
|
assert all(e["chamber"] == "ASSEMBLY" for e in evs)
|
||||||
|
|
||||||
|
|
||||||
|
def test_relative_link_resolves_against_host():
|
||||||
|
evs = parse_events_page(POPULATED_HTML,
|
||||||
|
base_url="https://www.assembly.ca.gov/events/upcoming")
|
||||||
|
floor = next(e for e in evs if e["kind"] == "FLOOR")
|
||||||
|
assert floor["source_url"] == \
|
||||||
|
"https://www.assembly.ca.gov/media/assembly-floor-session-20260910"
|
||||||
|
|
||||||
|
|
||||||
|
def test_row_without_link_falls_back_to_base_url():
|
||||||
|
evs = parse_events_page(POPULATED_HTML,
|
||||||
|
base_url="https://www.assembly.ca.gov/events/upcoming")
|
||||||
|
comm = next(e for e in evs if e["kind"] == "COMMITTEE")
|
||||||
|
assert comm["source_url"] == "https://www.assembly.ca.gov/events/upcoming"
|
||||||
|
|
||||||
|
|
||||||
|
def test_external_link_kept_verbatim():
|
||||||
|
evs = parse_events_page(POPULATED_HTML,
|
||||||
|
base_url="https://www.assembly.ca.gov/events/upcoming")
|
||||||
|
other = next(e for e in evs if e["kind"] == "OTHER")
|
||||||
|
assert other["source_url"] == "http://example.com/x"
|
||||||
|
|
||||||
|
|
||||||
|
def test_header_row_skipped():
|
||||||
|
evs = parse_events_page(POPULATED_HTML)
|
||||||
|
assert all(e["title"] not in ("Date", "Title") for e in evs)
|
||||||
|
|
||||||
|
|
||||||
|
# ---- empty / recess --------------------------------------------------------
|
||||||
|
|
||||||
|
def test_recess_yields_no_events():
|
||||||
|
assert parse_events_page(RECESS_HTML) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_empty_string_yields_no_events():
|
||||||
|
assert parse_events_page("") == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_html_with_no_time_cells_yields_no_events():
|
||||||
|
assert parse_events_page("<html><body><p>Nothing here.</p></body></html>") == []
|
||||||
|
|
||||||
|
|
||||||
|
# ---- layout fallback + bad rows -------------------------------------------
|
||||||
|
|
||||||
|
class _Broken:
|
||||||
|
"""A soup stand-in whose find() raises, exercised via parse error path."""
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_cols3_falls_back_to_time_rows():
|
||||||
|
evs = parse_events_page(NO_COLS3_HTML, chamber="assembly")
|
||||||
|
assert len(evs) == 1
|
||||||
|
assert evs[0]["date"] == "2026-09-13"
|
||||||
|
assert evs[0]["kind"] == "COMMITTEE"
|
||||||
|
|
||||||
|
|
||||||
|
def test_malformed_row_skipped_not_fatal():
|
||||||
|
# A row with a <time> but no title cell is skipped, the good row survives.
|
||||||
|
html = """
|
||||||
|
<html><body><table class="cols-3"><tbody>
|
||||||
|
<tr><td><time datetime="2026-09-14">Sep 14</time></td><td></td></tr>
|
||||||
|
<tr><td><time datetime="2026-09-15">Sep 15</time></td>
|
||||||
|
<td class="views-field views-field-title">Rules Committee</td></tr>
|
||||||
|
</tbody></table></body></html>
|
||||||
|
"""
|
||||||
|
evs = parse_events_page(html, chamber="assembly")
|
||||||
|
assert len(evs) == 1
|
||||||
|
assert evs[0]["title"] == "Rules Committee"
|
||||||
|
|
||||||
|
|
||||||
|
def test_broken_soup_returns_empty():
|
||||||
|
# A document that cannot be parsed at all yields [] (never raises).
|
||||||
|
result = parse_events_page(None, parser="lxml")
|
||||||
|
assert result == []
|
||||||
|
|
||||||
|
|
||||||
|
# ---- live sample (regression, network-independent) -------------------------
|
||||||
|
|
||||||
|
def test_sample_past_page_parses(tmp_path):
|
||||||
|
sample = os.path.join(os.path.dirname(os.path.dirname(__file__)),
|
||||||
|
"samples", "assembly_events_past.html")
|
||||||
|
if not os.path.exists(sample):
|
||||||
|
pytest.skip("sample not present")
|
||||||
|
import os as _os
|
||||||
|
html = open(sample, encoding="utf-8").read()
|
||||||
|
evs = parse_events_page(html, chamber="assembly", source="assembly_events")
|
||||||
|
assert len(evs) >= 2
|
||||||
|
assert all(e["chamber"] == "ASSEMBLY" for e in evs)
|
||||||
|
assert all(e["date"] and len(e["date"]) == 10 for e in evs)
|
||||||
Reference in new issue
Block a user