fix: add HLS manifest extraction to scrapeFn (3 strategies)
CI / test (pull_request) Canceled after 0s

- Strategy 1: scan inline <script> blocks for .m3u8 URLs (skip subtitles/captions)
- Strategy 2: check video element HLS player instances (hlsInstance, _hls)
- Strategy 3: probe common global config objects (hlsConfig, playerConfig, etc.)
- Storage items now include hlsManifest and mediaType fields
- When HLS detected and src is blob:/empty, promote manifest URL to primary
- Backward compatible: direct MP4 entries unaffected
This commit is contained in:
hermes committed 2026-08-10 17:07:55 +00:00
1 parent 90c3425200
commit db52db7799
1 file changed
+68 -3
+68 -3
View File
@@ -111,6 +111,48 @@ findBtn.addEventListener('click', async () => {
// Execute scraping on all tabs in parallel // Execute scraping on all tabs in parallel
const scrapeFn = () => { const scrapeFn = () => {
const dataKeys = ['src', 'video', 'mp4', 'webm', 'poster']; const dataKeys = ['src', 'video', 'mp4', 'webm', 'poster'];
/** Extract HLS manifest URLs from page using 3 strategies. */
function extractHlsManifest() {
// Strategy 1: <script> blocks containing .m3u8 URLs (most sites embed manifests in inline JS)
const m3u8Regex = /https?:[^"'\s]+\.m3u8[^"'\s]*/gi;
for (const script of document.querySelectorAll('script')) {
if (script.src) continue; // skip external scripts
const matches = script.textContent.match(m3u8Regex);
if (matches && matches.length > 0) {
// Prefer the first .m3u8 URL that looks like a real manifest (not a fragment/track)
for (const m of matches) {
const url = m.replace(/[;,]$/, '');
if (url && !url.includes('/subtitle') && !url.includes('/captions') && !url.includes('.m3u8x')) {
return url;
}
}
}
}
// Strategy 2: Check for HLS player instances on video elements
for (const v of document.querySelectorAll('video')) {
if (v && v.hlsInstance && v.hlsInstance.url) return v.hlsInstance.url;
if (v && v._hls && v._hls.url) return v._hls.url;
}
// Strategy 3: Common global HLS variables
const hlsVarNames = ['hlsConfig', 'hls_player', 'hlsOptions', 'playerConfig', 'streamConfig'];
for (const varName of hlsVarNames) {
try {
const obj = window[varName];
if (obj && typeof obj === 'object') {
for (const key of ['src', 'url', 'source', 'manifest', 'hls', 'm3u8', 'file']) {
const val = obj[key];
if (typeof val === 'string' && val.includes('.m3u8')) return val;
}
}
} catch { /* ignore */ }
}
return null;
}
// Collect all visible video elements, measure by rendered pixel area // Collect all visible video elements, measure by rendered pixel area
const allVideos = Array.from(document.querySelectorAll('video')).filter(v => { const allVideos = Array.from(document.querySelectorAll('video')).filter(v => {
const style = window.getComputedStyle(v); const style = window.getComputedStyle(v);
@@ -130,7 +172,11 @@ findBtn.addEventListener('click', async () => {
const largest = allVideos.length > 0 ? allVideos[0] : null; const largest = allVideos.length > 0 ? allVideos[0] : null;
const sources = Array.from(document.querySelectorAll('video source[src]')).map(s => s.src); const sources = Array.from(document.querySelectorAll('video source[src]')).map(s => s.src);
const audios = Array.from(document.querySelectorAll('audio')).length; const audios = Array.from(document.querySelectorAll('audio')).length;
return { videos: largest ? [largest] : [], videoCount: largest ? 1 : 0, sources, audios };
// Extract HLS manifest from the page
const hlsManifest = extractHlsManifest();
return { videos: largest ? [largest] : [], videoCount: largest ? 1 : 0, sources, audios, hlsManifest };
}; };
const results = await Promise.all( const results = await Promise.all(
tabsToScan.map(async t => { tabsToScan.map(async t => {
@@ -158,12 +204,24 @@ findBtn.addEventListener('click', async () => {
for (const group of tabResults) { for (const group of tabResults) {
for (const r of group.frames) { for (const r of group.frames) {
if (!r.result) continue; if (!r.result) continue;
// Get the HLS manifest from this frame's result (if any)
const frameHlsManifest = r.result.hlsManifest || null;
for (const v of r.result.videos) { for (const v of r.result.videos) {
totalFound++; totalFound++;
const key = (v.src || '(empty)') + '||' + (v.poster || '(none)'); const key = (v.src || '(empty)') + '||' + (v.poster || '(none)');
if (!seen.has(key)) { if (!seen.has(key)) {
const item = { url: v.src || v.poster || '', poster: v.poster || null, tab: group.tabTitle }; const item = { url: v.src || v.poster || '', poster: v.poster || null, tab: group.tabTitle };
if (v.src) item.src = v.src; if (v.src) item.src = v.src;
// Attach HLS manifest if found in this frame
if (frameHlsManifest) {
item.hlsManifest = frameHlsManifest;
item.mediaType = 'hls';
// When HLS is present, prefer the manifest URL as the primary URL
if (!v.src || v.src.startsWith('blob:')) {
item.url = frameHlsManifest;
item.src = frameHlsManifest;
}
}
seen.set(key, item); seen.set(key, item);
} }
} }
@@ -177,8 +235,15 @@ findBtn.addEventListener('click', async () => {
console.log('[goon] ===== Found', totalFound, 'video elements across', tabsToScan.length, 'tabs, unique:', items.length); console.log('[goon] ===== Found', totalFound, 'video elements across', tabsToScan.length, 'tabs, unique:', items.length);
// Replace the entire list (fresh scan) // Replace the entire list (fresh scan) — preserve hlsManifest and mediaType
const storedItems = items.map(v => ({ url: v.url, src: v.src || v.url, poster: v.poster, tab: v.tab })); const storedItems = items.map(v => ({
url: v.url,
src: v.src || v.url,
poster: v.poster,
tab: v.tab,
hlsManifest: v.hlsManifest || null,
mediaType: v.mediaType || 'direct',
}));
await chrome.storage.local.set({ await chrome.storage.local.set({
[STORAGE_KEY]: storedItems, [STORAGE_KEY]: storedItems,
[TAGGED_KEY]: items.map(v => v.tab), [TAGGED_KEY]: items.map(v => v.tab),