From 4357ce4094d7a079b4df29786fab5c1f71b7c1e8 Mon Sep 17 00:00:00 2001 From: kfj001 Date: Sun, 30 Aug 2026 04:48:23 +0000 Subject: [PATCH] Fixes issue with pornhub. Removes references. --- .claude/settings.local.json | 8 +- __tests__/data_flow.test.mjs | 31 ++++++ __tests__/referer_proxy.test.mjs | 139 +++++++++++++++++++++++++-- package-lock.json | 22 +---- package.json | 3 + src/background/background.js | 160 +++++++++++++++++++------------ src/viewer/index.js | 27 ++++++ src/viewer/schema.json | 5 + src/viewer/viewer.js | 42 ++++++-- 9 files changed, 337 insertions(+), 100 deletions(-) diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 50dfd2a..33213ad 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -11,7 +11,13 @@ "Bash(node -e \"console.log\\(\\(require\\('./package.json'\\).scripts||{}\\).build || '\\(no build script\\)'\\)\")", "Bash(git check-ignore *)", "Bash(node --check __tests__/simple_video.test.mjs)", - "Bash(node --check __tests__/run.mjs)" + "Bash(node --check __tests__/run.mjs)", + "WebSearch", + "Bash(node *)", + "Bash(python3 *)", + "Bash(git stash *)", + "Bash(npm -v)", + "Bash(npm install *)" ] } } diff --git a/__tests__/data_flow.test.mjs b/__tests__/data_flow.test.mjs index ef9e78a..b39b428 100644 --- a/__tests__/data_flow.test.mjs +++ b/__tests__/data_flow.test.mjs @@ -185,4 +185,35 @@ describe('Data flow: popup -> storage -> viewer', () => { throw new Error('Entry with invalid mediaType should be rejected'); } }); + + it('an HLS entry carrying a scan-time referer passes schema + round-trips', () => { + // The referer is bound to the entry at scan time so it survives the + // watch-later flow (the source tab may be long closed). Because the schema + // sets additionalProperties:false, a `referer` the store carries must be an + // explicit property or storage-layer validation would reject the entry. + const scrapedEntry = { + url: 'https://pageA.example.com/watch', + src: 'https://hv-h.phncdn.com/hls/live/master.m3u8', + hlsManifest: 'https://hv-h.phncdn.com/hls/live/master.m3u8', + mediaType: 'hls', + poster: 'https://pageA.example.com/thumb.jpg', + tab: 'Referer-protected stream', + referer: 'https://pageA.example.com/watch', + }; + + const valid = validate(scrapedEntry); + if (!valid) { + throw new Error(`Entry with referer fails schema: ${JSON.stringify(validate.errors)}`); + } + + // The referer must survive the storage round-trip intact so the viewer can + // send it to the proxy when the user opens a previously-scanned entry. + const roundTripped = JSON.parse(JSON.stringify(scrapedEntry)); + if (roundTripped.referer !== 'https://pageA.example.com/watch') { + throw new Error('referer lost during round-trip'); + } + if (!validate(roundTripped)) { + throw new Error('Round-tripped entry with referer fails schema'); + } + }); }); diff --git a/__tests__/referer_proxy.test.mjs b/__tests__/referer_proxy.test.mjs index eefb8f7..a15f04c 100644 --- a/__tests__/referer_proxy.test.mjs +++ b/__tests__/referer_proxy.test.mjs @@ -38,7 +38,7 @@ function makeChromeEnv() { let sessionRules = {}; const webRequest = { - onBeforeSendHeaders: { addListener: (cb) => { handlers.onBeforeSendHeaders = cb; } }, + onBeforeSendHeaders: { addListener: (cb, filter) => { handlers.onBeforeSendHeaders = cb; handlers.onBeforeSendHeadersFilter = filter; } }, onBeforeRequest: { addListener: (cb) => { handlers.onBeforeRequest = cb; } }, }; @@ -113,7 +113,7 @@ describe('PR#5: per-host referer capture + proxy race', () => { if (!cap) throw new Error('onBeforeSendHeaders listener was not registered'); await cap(videoDetails(11, 'https://cdnA.example.com/hls/live/seg-1.ts', 'https://pageA.example.com/watch')); - const rA = env.storage['goon_referers']?.['cdna.example.com']; + const rA = env.storage['goon_referers']?.['example.com']; if (!rA) throw new Error('referer was not captured for cdnA'); if (rA.referer !== 'https://pageA.example.com/watch') { throw new Error('captured referer wrong: ' + JSON.stringify(rA)); @@ -123,10 +123,10 @@ describe('PR#5: per-host referer capture + proxy race', () => { await env.handlers.onRemoved(22); const after = env.storage['goon_referers']; - if (!after || !after['cdna.example.com']) { + if (!after || !after['example.com']) { throw new Error('closing tab 22 wiped goon_referers — host A referer lost'); } - if (after['cdna.example.com'].referer !== 'https://pageA.example.com/watch') { + if (after['example.com'].referer !== 'https://pageA.example.com/watch') { throw new Error('closing tab 22 corrupted the host A referer: ' + JSON.stringify(after)); } }); @@ -143,7 +143,7 @@ describe('PR#5: per-host referer capture + proxy race', () => { // The very tab that captured the referer now closes. await env.handlers.onRemoved(11); - const after = env.storage['goon_referers']?.['cdnb.example.com']; + const after = env.storage['goon_referers']?.['example.com']; if (!after) throw new Error('goon_referers for the capturing host was wiped on its own tab close'); if (after.referer !== 'https://pageB.example.com/v') { throw new Error('capturing-tab close corrupted referer: ' + JSON.stringify(after)); @@ -153,7 +153,7 @@ describe('PR#5: per-host referer capture + proxy race', () => { // (b) Should-fix + coverage: two hosts proxied concurrently must each keep // their own referer. With a shared rule id + non-atomic install this regressed; // per-host rule ids + the per-host lock keep them distinct. - it('concurrent proxy fetches for two hosts keep distinct per-host referer rules', async () => { + it('concurrent proxy fetches for two distinct registrable domains keep distinct referer rules', async () => { const env = makeChromeEnv(); let fetchCalls = 0; global.chrome = env.chrome; @@ -173,7 +173,7 @@ describe('PR#5: per-host referer capture + proxy race', () => { const handler = env.handlers.onMessage[env.handlers.onMessage.length - 1]; let respA, respB; handler({ type: 'GOON_PROXY_FETCH', url: 'https://cdnA.example.com/hls/live/seg-1.ts', referer: 'https://pageA.example.com/watch' }, {}, (r) => { respA = r; }); - handler({ type: 'GOON_PROXY_FETCH', url: 'https://cdnB.example.com/hls/live/seg-1.ts', referer: 'https://pageB.example.com/watch' }, {}, (r) => { respB = r; }); + handler({ type: 'GOON_PROXY_FETCH', url: 'https://cdnB.other.com/hls/live/seg-1.ts', referer: 'https://pageB.other.com/watch' }, {}, (r) => { respB = r; }); await new Promise(r => setTimeout(r, 30)); // let async sends + installs flush @@ -184,8 +184,8 @@ describe('PR#5: per-host referer capture + proxy race', () => { // Each host must have its OWN installed DNR rule with its OWN referer — // host A must NOT be served host B's referer (the 412 regression). const rules = env.getSessionRules(); - const ruleA = Object.values(rules).find(r => r.condition?.requestDomains?.includes('cdna.example.com')); - const ruleB = Object.values(rules).find(r => r.condition?.requestDomains?.includes('cdnb.example.com')); + const ruleA = Object.values(rules).find(r => r.condition?.requestDomains?.includes('example.com')); + const ruleB = Object.values(rules).find(r => r.condition?.requestDomains?.includes('other.com')); if (!ruleA) throw new Error('no DNR rule installed for cdna; rules=' + JSON.stringify(rules)); if (!ruleB) throw new Error('no DNR rule installed for cdnb; rules=' + JSON.stringify(rules)); if (ruleA.id === ruleB.id) { @@ -196,7 +196,7 @@ describe('PR#5: per-host referer capture + proxy race', () => { if (refA !== 'https://pageA.example.com/watch') { throw new Error('cdnA served wrong referer: ' + refA); } - if (refB !== 'https://pageB.example.com/watch') { + if (refB !== 'https://pageB.other.com/watch') { throw new Error('cdnB served wrong referer: ' + refB); } if (refA === refB) throw new Error('both hosts got the same referer — cross-host bleed'); @@ -236,4 +236,123 @@ describe('PR#5: per-host referer capture + proxy race', () => { throw new Error(`expected idempotent install, but re-added rule ${installs - installsAfterFirst} extra time(s)`); } }); + + // (c) Domain generalization — the core 412 fix. A provider referer captured for + // ONE edge subdomain must cover a SIBLING subdomain: the DNR rule is scoped to + // the registrable domain and the store is domain-keyed, so a proxy for ev-h + // needs no per-subdomain capture. + it('a proxy for a sibling subdomain reuses the captured referer via the registrable domain', async () => { + const env = makeChromeEnv(); + global.chrome = env.chrome; + global.fetch = async () => ({ + ok: true, + status: 200, + headers: { get: () => 'application/vnd.apple.mpegurl' }, + arrayBuffer: async () => new Uint8Array([0]).buffer, + }); + await loadBackground(env); + + // Capture the provider referer for hv-h.phncdn.com (one edge subdomain). + await env.handlers.onBeforeSendHeaders( + videoDetails(11, 'https://hv-h.phncdn.com/hls/live/seg-1.ts', 'https://provider.example.com/watch')); + + // Proxy a SIBLING subdomain ev-h.phncdn.com with NO referer in the message: + // the SW must resolve it from the domain-keyed store. + const handler = env.handlers.onMessage[env.handlers.onMessage.length - 1]; + let resp; + handler({ type: 'GOON_PROXY_FETCH', url: 'https://ev-h.phncdn.com/hls/live/seg-1.ts' }, {}, (r) => { resp = r; }); + await new Promise(r => setTimeout(r, 20)); + + if (!resp || !resp.ok) throw new Error('sibling proxy failed: ' + JSON.stringify(resp)); + const rules = env.getSessionRules(); + const rule = Object.values(rules).find(r => r.condition?.requestDomains?.includes('phncdn.com')); + if (!rule) throw new Error('no domain-scoped DNR rule for phncdn.com; rules=' + JSON.stringify(rules)); + const ref = rule.action.requestHeaders?.find(h => h.header.toLowerCase() === 'referer')?.value; + if (ref !== 'https://provider.example.com/watch') { + throw new Error('sibling edge served wrong referer: ' + ref); + } + // The rule is scoped to the registrable domain, NOT the exact subdomain. + if (rule.condition.requestDomains?.includes('ev-h.phncdn.com')) { + throw new Error('rule scoped to exact subdomain instead of registrable domain'); + } + }); + + // (d) With no captured referer and none in the message, install NO rule: the + // self-origin fallback (guaranteed-wrong for hotlink protection) is dropped. + it('no captured referer and no message referer installs no DNR rule', async () => { + const env = makeChromeEnv(); + let fetchCalls = 0; + global.chrome = env.chrome; + global.fetch = async () => { + fetchCalls++; + return { + ok: true, + status: 200, + headers: { get: () => 'application/vnd.apple.mpegurl' }, + arrayBuffer: async () => new Uint8Array([0]).buffer, + }; + }; + await loadBackground(env); + const handler = env.handlers.onMessage[env.handlers.onMessage.length - 1]; + let resp; + handler({ type: 'GOON_PROXY_FETCH', url: 'https://cdnA.example.com/hls/live/seg-1.ts' }, {}, (r) => { resp = r; }); + await new Promise(r => setTimeout(r, 20)); + + if (!resp || !resp.ok) throw new Error('proxy fetch failed: ' + JSON.stringify(resp)); + if (fetchCalls < 1) throw new Error('proxy fetch did not run'); + const rules = env.getSessionRules(); + if (Object.keys(rules).length !== 0) { + throw new Error('installed a rule with no referer (self-origin fallback not dropped): ' + JSON.stringify(rules)); + } + }); + + // (e) GOON_GET_REFERERS (used by scan-time capture) resolves each manifest host + // via the domain-keyed store: a sibling subdomain gets the provider referer, + // and an uncaptured host is omitted. + it('GOON_GET_REFERERS resolves manifest hosts through the domain-keyed store', async () => { + const env = makeChromeEnv(); + global.chrome = env.chrome; + await loadBackground(env); + + // Capture for hv-h.phncdn.com only. + await env.handlers.onBeforeSendHeaders( + videoDetails(11, 'https://hv-h.phncdn.com/hls/live/seg-1.ts', 'https://provider.example.com/watch')); + + const handler = env.handlers.onMessage[env.handlers.onMessage.length - 1]; + let resp; + handler( + { type: 'GOON_GET_REFERERS', hosts: ['ev-h.phncdn.com', 'hv-h.phncdn.com', 'cdnA.example.com'] }, + {}, + (r) => { resp = r; }); + await new Promise(r => setTimeout(r, 15)); + + if (!resp || !resp.ok) throw new Error('GOON_GET_REFERERS failed: ' + JSON.stringify(resp)); + const referers = resp.referers || {}; + if (referers['ev-h.phncdn.com'] !== 'https://provider.example.com/watch') { + throw new Error('sibling subdomain did not resolve the captured referer: ' + JSON.stringify(referers)); + } + if (referers['hv-h.phncdn.com'] !== 'https://provider.example.com/watch') { + throw new Error('captured subdomain did not resolve its own referer: ' + JSON.stringify(referers)); + } + if ('cdnA.example.com' in referers) { + throw new Error('uncaptured host should be omitted from referers: ' + JSON.stringify(referers)); + } + }); + + // Regression for the live 412 bug: the capture listener MUST request + // requestHeaders via extraInfoSpec, or details.requestHeaders is undefined and + // readReferer() returns '' for every request — so no referer is ever stored, + // every entry stays referer-less, and the phncdn proxy 412s. The callback-level + // tests above pass even without this (they hand requestHeaders in directly), so + // this asserts the registration itself. + it('the onBeforeSendHeaders capture listener requests requestHeaders via extraInfoSpec', async () => { + const env = makeChromeEnv(); + global.chrome = env.chrome; + await loadBackground(env); + const filter = env.handlers.onBeforeSendHeadersFilter; + if (!filter) throw new Error('onBeforeSendHeaders listener registered no filter object'); + if (!Array.isArray(filter.extraInfoSpec) || !filter.extraInfoSpec.includes('requestHeaders')) { + throw new Error('capture listener did not request requestHeaders via extraInfoSpec: ' + JSON.stringify(filter)); + } + }); }); diff --git a/package-lock.json b/package-lock.json index 439604c..bbe7e55 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,7 +8,7 @@ "name": "vidplex", "version": "0.1.0", "dependencies": { - "@rollup/rollup-darwin-arm64": "^4.63.1" + "@rollup/rollup-darwin-arm64": "^4.60.4" }, "devDependencies": { "ajv": "^8.17.1", @@ -1338,9 +1338,9 @@ ] }, "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.63.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.63.1.tgz", - "integrity": "sha512-FQNqd1lRy/0QhDk3xeRIkSBiCpXCiDnZO3YLVdcDKN1UBiKToNftCzcXYNLshmPDUMlu2TdeS8tGcsU6f3YF1Q==", + "version": "4.60.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.60.4.tgz", + "integrity": "sha512-tua0TaJxMOB1R0V0RS1jFZ/RpURFDJIOR2A6jWwQeawuFyS4gBW+rntLRaQd0EQ4bd6Vp44Z2rXW+YYDBsj6IA==", "cpu": [ "arm64" ], @@ -3423,20 +3423,6 @@ "fsevents": "~2.3.2" } }, - "node_modules/rollup/node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.60.4", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.60.4.tgz", - "integrity": "sha512-tua0TaJxMOB1R0V0RS1jFZ/RpURFDJIOR2A6jWwQeawuFyS4gBW+rntLRaQd0EQ4bd6Vp44Z2rXW+YYDBsj6IA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ] - }, "node_modules/rollup/node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", diff --git a/package.json b/package.json index 54f46fb..10edb03 100644 --- a/package.json +++ b/package.json @@ -23,5 +23,8 @@ "typescript": "^5.7.0", "vite": "^6.0.0", "vitest": "^3.0.0" + }, + "dependencies": { + "@rollup/rollup-darwin-arm64": "^4.60.4" } } diff --git a/src/background/background.js b/src/background/background.js index 0052612..c5b3023 100644 --- a/src/background/background.js +++ b/src/background/background.js @@ -227,6 +227,11 @@ async function injectFreshToken(url) { // re-query storage for a freshly-rotated token. function invalidateTokenCache() { tokenCache.clear(); } +// CDN registrable domains whose referer we have already captured this SW +// lifetime, so we do not re-query the tab for every segment of a stream. +// Lost if the SW is killed; persistReferer's storage dedupe covers a fresh SW. +const refererCapturedDomains = new Set(); + chrome.webRequest.onBeforeRequest.addListener( async (details) => { if (details.tabId < 0) return; @@ -235,6 +240,28 @@ chrome.webRequest.onBeforeRequest.addListener( if (isVideoRequest(details.url)) { console.log('[goon-bg] webRequest: video request in tab', details.tabId, ':', details.url.substring(0, 180)); await persistVideoRequest(details.tabId, details.url); + // Capture the provider referer for this CDN host from the source tab's + // URL: the browser sends the referring page's URL as the Referer for a + // cross-origin video request, and that IS the tab's URL. extraInfoSpec + // (reading the actual request headers) is rejected by some Chromium + // builds and fails the whole service worker, so we derive the referer + // from the tab instead. The `tabs` + host_permissions already allow it. + const host = hostOf(details.url); + const domain = host ? registrableDomain(host) : ''; + if (domain && !refererCapturedDomains.has(domain)) { + try { + const tab = await chrome.tabs.get(details.tabId); + const ref = tab && tab.url; + if (ref) { + await persistReferer(host, ref); + refererCapturedDomains.add(domain); + console.log('[goon-bg] captured referer for', domain, '->', ref); + } + } catch { + // Tab may have closed mid-stream; a later request from a live tab + // re-captures. Best-effort. + } + } } // Also capture HLS manifest URLs (original logic) @@ -258,7 +285,7 @@ chrome.webRequest.onBeforeRequest.addListener( ); // Clean up stale PER-TAB entries when tabs are closed. NOTE: 'goon_referers' is -// a host-keyed GLOBAL store — it must outlive its capture tab so a proxy fetch +// a domain-keyed GLOBAL store — it must outlive its capture tab so a proxy fetch // for that host still finds its referer after the source tab is closed. It is // therefore NOT removed here (only the per-tab vr_requests_/wr_manifests_ keys // are). Stale referers are pruned opportunistically in persistReferer. @@ -308,87 +335,74 @@ function hostOf(url) { try { return new URL(url).host.toLowerCase(); } catch { return ''; } } -// Best-effort origin for a host when no referer was captured for it. Used only -// as a fallback; a captured referer always wins. -function originFromHost(host) { - return host ? 'https://' + host + '/' : ''; +// Registrable domain of a CDN host — the label a single provider's referer +// applies to. A referer-protecting CDN serves manifests and segments from many +// edge subdomains that share ONE +// provider referer, so we key the referer store, the DNR rule, and proxy-host +// matching on this domain rather than the exact subhost. +// +// Heuristic: the last two DNS labels. Correct for the overwhelmingly common +// single-label public suffixes (.com, .net, .org, .io, ...). Known caveat: +// multi-label public suffixes such as .co.uk / .com.au collapse to the public +// suffix itself (foo.co.uk -> co.uk); that is an exotic edge case and no worse +// than the exact-host behavior it replaces. A single-level host (no dot, or a +// bare TLD) is returned unchanged. +function registrableDomain(host) { + if (!host) return ''; + const parts = host.split('.'); + if (parts.length < 2) return host; + return parts.slice(-2).join('.'); } -// Read the `Referer` request header (the browser may title-case it; the - // (onBeforeSendHeaders cannot read request headers in MV3 without a blocking API, - // so we fall back to the page origin for hosts that send no referer). -function readReferer(headers) { - if (!Array.isArray(headers)) return ''; - for (const h of headers) { - if (h && h.name && h.name.toLowerCase() === 'referer') return h.value || ''; - } - return ''; -} // How long a captured referer is trusted before it is pruned on the next -// persist. Referers are short-lived session tokens; this keeps the per-host -// global store from accumulating stale entries without tying cleanup to tab -// close (a still-open source tab's referer must survive other tabs closing). -const REFERER_TTL_MS = 30 * 60 * 1000; +// persist. Kept long enough for the scan-now / watch-later flow (a user scans a +// page, then opens the viewer much later) without tying cleanup to tab close (a +// still-open source tab's referer must survive other tabs closing). +const REFERER_TTL_MS = 7 * 24 * 60 * 60 * 1000; -// Persist the captured referer per CDN host so the proxy can re-use it even -// after this capture tab closes. Keyed by host so a proxy for that host can -// look its referer up without knowing the source tab. Each entry is stamped -// with a capture time; entries older than the TTL are pruned opportunistically -// here (never on tab close). +// Persist the captured referer per CDN registrable domain so the proxy can +// re-use it even after the capture tab closes. Keyed by the host's registrable +// domain so a single provider referer covers every edge subdomain (a CDN serves +// manifests/segments from many subhosts that share one referer), and so a proxy +// for any sibling subhost can look it up without knowing the source tab. Each +// entry is stamped with a capture time; entries older than the TTL are pruned +// opportunistically here (never on tab close). async function persistReferer(host, referer) { - if (!host || !referer) return; + const domain = registrableDomain(host); + if (!domain || !referer) return; try { const stored = await chrome.storage.local.get('goon_referers'); const map = stored['goon_referers'] || {}; const now = Date.now(); - // Opportunistic per-host prune: drop entries older than the TTL. + // Opportunistic per-domain prune: drop entries older than the TTL. for (const [h, entry] of Object.entries(map)) { - if (h !== host && (!entry || !entry.at || now - entry.at > REFERER_TTL_MS)) { + if (h !== domain && (!entry || !entry.at || now - entry.at > REFERER_TTL_MS)) { delete map[h]; } } - const current = map[host]; + const current = map[domain]; if (current && current.referer === referer) return; - map[host] = { referer, at: now }; + map[domain] = { referer, at: now }; await chrome.storage.local.set({ goon_referers: map }); } catch (e) { console.warn('[goon-bg] persistReferer failed:', e && e.message ? e.message : e); } } -// Look up the referer captured for a host. '' when none has been captured -// (or the stored entry is malformed). +// Look up the referer captured for a host's registrable domain. '' when none +// has been captured (or the stored entry is malformed). async function getReferer(host) { if (!host) return ''; try { const stored = await chrome.storage.local.get('goon_referers'); - const entry = (stored['goon_referers'] || {})[host]; + const entry = (stored['goon_referers'] || {})[registrableDomain(host)]; return entry && entry.referer ? entry.referer : ''; } catch { return ''; } } -// Capture the referer the real page sends for its video requests. -chrome.webRequest.onBeforeSendHeaders.addListener( - async (details) => { - if (details.tabId < 0) return; - if (!isVideoRequest(details.url)) return; - const host = hostOf(details.url); - if (!host) return; - const referer = readReferer(details.requestHeaders); - if (referer) { - // The browser already built a referer from the source page — use it. - await persistReferer(host, referer); - } else { - // No referer was sent; assume the page's origin is the expected referer. - await persistReferer(host, originFromHost(host)); - } - }, -{ urls: [''] } -); - // Install (idempotently) a DNR rule that sets the captured referer for a given // CDN host. The value comes from capture (or a fallback origin), never a // hardcoded site. Scoped to the requested host so only proxy fetches for that @@ -404,7 +418,8 @@ const refererRuleInstalled = new Map(); const refererRuleLocks = new Map(); async function ensureRefererRule(host, referer) { - if (!host || !referer) return; + const domain = registrableDomain(host); + if (!domain || !referer) return; const api = chrome.declarativeNetRequest; if (!api || typeof api.updateSessionRules !== 'function') { console.warn('[goon-bg] declarativeNetRequest unavailable; skipping referer rule'); @@ -413,10 +428,10 @@ async function ensureRefererRule(host, referer) { // Serialize per host so the idempotency check + updateSessionRules is atomic. const run = (async () => { - if (refererRuleInstalled.has(host) && refererRuleInstalled.get(host) === referer) { + if (refererRuleInstalled.has(domain) && refererRuleInstalled.get(domain) === referer) { return; } - const ruleId = refererRuleIdFor(host); + const ruleId = refererRuleIdFor(domain); try { // Remove only THIS host's own rule id, then add it back scoped to the host. await api.updateSessionRules({ @@ -435,17 +450,17 @@ async function ensureRefererRule(host, referer) { condition: { // Host + subdomains ONLY, so only proxy fetches for this host get the // rewritten referer; the source page's own requests keep theirs. - requestDomains: [host], + requestDomains: [domain], }, }], }); - refererRuleInstalled.set(host, referer); - console.log('[goon-bg] DNR referer rule installed for', host, '(' + ruleId + ')'); + refererRuleInstalled.set(domain, referer); + console.log('[goon-bg] DNR referer rule installed for', domain, '(' + ruleId + ')'); } catch (e) { console.warn('[goon-bg] ensureRefererRule failed:', e && e.message ? e.message : e); } - })().finally(() => { refererRuleLocks.set(host, null); }); - refererRuleLocks.set(host, run); + })().finally(() => { refererRuleLocks.set(domain, null); }); + refererRuleLocks.set(domain, run); return run; } @@ -533,10 +548,13 @@ chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => { const proxyHost = hostOf(message.url); if (proxyHost) { // Prefer the provider's real referer captured from the live - // tab; fall back to a viewer-supplied or host-derived origin. + // tab (domain-aware), or the entry's scan-time referer the viewer sent. + // With neither, install no rule: a self-origin referer is + // guaranteed-wrong for hotlink protection, so we never fall + // back to one (it would just 412). const captured = await getReferer(proxyHost); - const referer = captured || message.referer || originFromHost(proxyHost); - await ensureRefererRule(proxyHost, referer); + const referer = captured || message.referer || ''; + if (referer) await ensureRefererRule(proxyHost, referer); } // hls.js resolves sub-playslists/segments relative to the master // and can DROP the token query, handing us a token-less @@ -571,6 +589,24 @@ chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => { return true; } + case 'GOON_GET_REFERERS': { + // Resolve the provider referer captured for each manifest host + // (domain-aware: one provider referer covers every edge subdomain). + // Used at scan time to bind the referer to a stored video entry so it + // survives the watch-later flow. + async function getReferers(message, sendResponse) { + const hosts = Array.isArray(message.hosts) ? message.hosts : []; + const referers = {}; + for (const host of hosts) { + const r = await getReferer(host); + if (r) referers[host] = r; + } + sendResponse({ ok: true, referers }); + } + getReferers(message, sendResponse); + return true; + } + default: console.warn('[goon-plugin] Unknown message type:', message.type); } diff --git a/src/viewer/index.js b/src/viewer/index.js index 40570fd..e97f261 100644 --- a/src/viewer/index.js +++ b/src/viewer/index.js @@ -1147,6 +1147,7 @@ async function scanVideos(tabsToScan) { } const items = [...seen.values()]; + await attachReferers(items); const skipped = scrapeResults.filter(r => r === null).length; log('scanVideos complete: scanned=', tabResults.length, 'skipped=', skipped, 'unique items=', items.length); @@ -1157,6 +1158,31 @@ async function scanVideos(tabsToScan) { return { items, scanned: tabResults.length, skipped }; } +/** + * Bind the provider referer the SW captured (domain-aware) to each item so it + * survives the watch-later flow and covers the sibling edge subdomains a segment + * may resolve to. Best-effort: playback still falls back to the SW's live store. + */ +async function attachReferers(items) { + const hostOf = (url) => { try { return new URL(url).host.toLowerCase(); } catch { return ''; } }; + const hosts = []; + for (const it of items) { + const host = hostOf(it.hlsManifest || it.src || ''); + if (host) hosts.push(host); + } + if (hosts.length === 0) return; + let referers = {}; + try { + const resp = await chrome.runtime.sendMessage({ type: 'GOON_GET_REFERERS', hosts }); + if (resp && resp.referers) referers = resp.referers; + } catch { /* referer is best-effort; the SW still resolves it at play time */ } + if (!Object.keys(referers).length) return; + for (const it of items) { + const host = hostOf(it.hlsManifest || it.src || ''); + if (host && referers[host]) it.referer = referers[host]; + } +} + /** Store video entries in chrome.storage.local. */ async function storeVideos(items) { const stored = items.map(v => ({ @@ -1165,6 +1191,7 @@ async function storeVideos(items) { tab: v.tab, mediaType: v.mediaType || 'video', hlsManifest: v.hlsManifest || undefined, + referer: v.referer || undefined, })); await chrome.storage.local.set({ [STORAGE_KEY]: stored, diff --git a/src/viewer/schema.json b/src/viewer/schema.json index 5fe41ec..d6f5339 100644 --- a/src/viewer/schema.json +++ b/src/viewer/schema.json @@ -33,6 +33,11 @@ "type": "string", "enum": ["direct", "hls"], "description": "Media delivery format. Defaults to 'direct' when omitted. Set to 'hls' for HTTP Live Streaming." + }, + "referer": { + "type": ["string", "null"], + "format": "uri", + "description": "Provider Referer captured at scan time for the manifest's CDN; the proxy sends it so a referer-protecting CDN accepts sibling edge subdomains. Optional." } }, "additionalProperties": false, diff --git a/src/viewer/viewer.js b/src/viewer/viewer.js index 739a557..9a851e4 100644 --- a/src/viewer/viewer.js +++ b/src/viewer/viewer.js @@ -178,7 +178,7 @@ const panelsEl = document.getElementById('panels'); // Route through the service worker to bypass the CDN's CORS/Referer // restrictions. The SW adds the required Referer and returns base64. console.log('[goon-viewer] Proxying via SW:', (url || '').substring(0, 120)); - chrome.runtime.sendMessage({ type: 'GOON_PROXY_FETCH', url, referer: originForUrl(url) }, (resp) => { + chrome.runtime.sendMessage({ type: 'GOON_PROXY_FETCH', url, referer: refererForUrl(url) }, (resp) => { if (self._aborted) return; const lastErr = chrome.runtime.lastError; if (lastErr || !resp || !resp.ok) { @@ -264,19 +264,34 @@ HlsLoader.canSupportLoader = function () { return true; }; // hardcode a particular CDN. const PROXY_HOSTS_KEY = 'goon_proxy_hosts'; const proxyHosts = new Set(); + // Provider referers bound to a stored entry at scan time, keyed by the entry's + // manifest registrable domain. A single provider referer covers every edge + // subdomain, so a segment on a sibling subhost resolves to the same referer. + const refererByDomain = new Map(); (async () => { try { const stored = await chrome.storage.local.get(PROXY_HOSTS_KEY); const hosts = stored[PROXY_HOSTS_KEY]; - if (Array.isArray(hosts)) for (const h of hosts) proxyHosts.add(String(h).toLowerCase()); + if (Array.isArray(hosts)) for (const h of hosts) proxyHosts.add(registrableDomainOf(String(h).toLowerCase())); } catch { /* ignore */ } })(); - // Origin for a URL's host — used as the Referer to request when a proxy fetch - // is made for that host (the SW also captures the live referer per host). - function originForUrl(url) { - if (!url) return ''; - try { return new URL(url).origin + '/'; } catch { return ''; } + // Registrable domain of a CDN host — the label a single provider referer + // applies to (a referer-protecting CDN serves from many edge subdomains that + // share one referer). Last-two-labels heuristic; mirrors the SW's copy. + function registrableDomainOf(host) { + if (!host) return ''; + const parts = host.split('.'); + if (parts.length < 2) return host; + return parts.slice(-2).join('.'); + } + + // The provider referer to send for a proxied request: the referer bound to the + // entry at scan time (domain-keyed), or '' when none was captured. No self-origin + // fallback — a self-origin referer is guaranteed-wrong for hotlink protection. + function refererForUrl(url) { + const ref = refererByDomain.get(registrableDomainOf(hostOfUrl(url))); + return ref || ''; } // Whether a given video URL should be proxied: its host is a known @@ -284,7 +299,7 @@ HlsLoader.canSupportLoader = function () { return true; }; function needsProxy(url) { if (!url || typeof url !== 'string') return false; const host = hostOfUrl(url); - return host ? proxyHosts.has(host) : false; + return host ? proxyHosts.has(registrableDomainOf(host)) : false; } // host of a URL (lowercased); '' when malformed. @@ -1184,11 +1199,20 @@ async function init() { // Probe CORS on each video URL. Hosts that block extension-origin // requests are learned as referer-protected and persisted so the viewer // routes them through the service-worker proxy. + // Bind each entry's scan-time referer to its manifest's registrable + // domain, so the proxy sends it for every edge subdomain the provider + // serves from (segments often resolve to a sibling subhost). + for (const entry of videoUrls) { + const ref = entry && entry.referer; + if (!ref) continue; + const dom = registrableDomainOf(hostOfUrl(entry.src || entry.hlsManifest || '')); + if (dom) refererByDomain.set(dom, ref); + } const urlsToTest = videoUrls.map(v => v.src || v.hlsManifest || 'none').filter(u => u !== 'none' && typeof u === 'string'); const blockedHosts = new Set(); for (const url of [...new Set(urlsToTest)]) { const corsOk = await corsCheck(url); - if (!corsOk) blockedHosts.add(hostOfUrl(url)); + if (!corsOk) blockedHosts.add(registrableDomainOf(hostOfUrl(url))); console.log('[goon-viewer] CORS', corsOk ? 'OK' : 'BLOCKED:', url.substring(0, 120)); } if (blockedHosts.size > 0) {