From 7139c5ea6f87ae7027d3aaede359906356fa1971 Mon Sep 17 00:00:00 2001
From: kfj001 <kfj001@gmail.com>
Date: Tue, 25 Aug 2026 20:27:14 -0700
Subject: [PATCH] fix(scan): run deep probe/scraper in MAIN world; reject
 route-template "manifests"
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The video-detection pipeline was finding nothing on pages that have a bare
<video> element. Two root causes:

1. world: 'MAIN' placement. The deep probe + scraper are injected with
   chrome.scripting.executeScript. Their code (and its own comments) requires
   the MAIN world to see page globals — window.Hls, per-<video> Hls instances,
   video._goon_hls_manifest, and window.__goon_plugin_data (written by the
   main-world interceptors). The calls were missing it, so every main-world
   strategy (A/A2/E/H) found 0. It was also briefly placed inside `target`,
   which chrome.scripting rejects ("Unexpected property: 'world'"); `world` is
   a sibling of `target` on ScriptInjection. Restore it as a sibling of target.

2. fake "manifest" classification. Strategy H was reporting 10 nav links /
   server-side route TEMPLATES as manifests — e.g. /playlists/,
   /playlists/liked/, /playlist/%1%/video/%2%/, /members/%ID%/playlists/ —
   because the old /playlist|/manifest substring check matches nav menus and
   leaves %1%/%ID%/%DIR% template placeholders intact. Those then became the
   stored "video" entries, drowning out the real element.

   isManifestUrl and isValidManifestUrl now reject:
     - dynamic route templates (%NAME% or {name} tokens), using a negative
       lookahead so genuine %-encoded tokens like %2B/%2F are preserved
       (the invariant the codebase explicitly protects),
     - plural collection paths (/playlists, /manifests),
     - trailing-slash directory listings.

   The 'World: ISOLATED' probe log was also misleading (typeof chrome is not a
   world discriminator — the probe provably runs in MAIN); replaced it.

Adds a regression test (hls_qa) reproducing thisvid's pageContext.langUrls
scenario: all 10 templates are rejected while a real .m3u8 carrying a %2B token
is still detected. 90/90 tests pass; 0 lint errors; build OK.
---
 __tests__/hls_qa.test.mjs | 132 ++++++++++++++++++++++++++++++++------
 src/viewer/index.js       |  84 +++++++++++++++---------
 2 files changed, 168 insertions(+), 48 deletions(-)

diff --git a/__tests__/hls_qa.test.mjs b/__tests__/hls_qa.test.mjs
index b6c1152..a293121 100644
--- a/__tests__/hls_qa.test.mjs
+++ b/__tests__/hls_qa.test.mjs
@@ -14,8 +14,8 @@ const viewerSource = fs.readFileSync(
 );
 const interceptorSource = fs.readFileSync(
   path.join(__dirname, '../src/inject/interceptors.js'),
-  'utf-8'
-);
+     'utf-8'
+    );
 
 /* ========================================================================== */
 /* Helpers                                                                    */
@@ -1027,21 +1027,48 @@ describe('HLS QA: iframe and multi-frame scenarios', () => {
     }
   });
 
-  it('should scrape video elements from all frames', () => {
-    const indexSource = fs.readFileSync(
-      path.join(__dirname, '../src/viewer/index.js'),
-      'utf-8'
-    );
+   it('should scrape video elements from all frames', () => {
+     const indexSource = fs.readFileSync(
+       path.join(__dirname, '../src/viewer/index.js'),
+       'utf-8'
+      );
 
-    // Verify scrapeTab runs in all frames
-    const scrapeMatch = indexSource.match(/scrapeTab[\s\S]*?executeScript[\s\S]*?\}/);
-    if (!scrapeMatch) {
-      throw new Error('scrapeTab function not found');
-    }
-    if (!scrapeMatch[0].includes('allFrames: true')) {
-      throw new Error('scrapeTab must use allFrames: true');
-    }
-  });
+      // Verify scrapeTab runs in all frames
+     const scrapeMatch = indexSource.match(/scrapeTab[\s\S]*?executeScript[\s\S]*?\}/);
+     if (!scrapeMatch) {
+       throw new Error('scrapeTab function not found');
+      }
+     if (!scrapeMatch[0].includes('allFrames: true')) {
+       throw new Error('scrapeTab must use allFrames: true');
+      }
+    });
+
+   it('deep probe + scrape must run in the MAIN world (see page globals)', () => {
+      // The deep probe and scraper read page globals (window.Hls, player
+      // instances, video._goon_hls_manifest) and window.__goon_plugin_data,
+      // which the main-world interceptors write. The isolated world cannot see
+      // these, so a missing `world: 'MAIN'` makes every main-world strategy find
+      // 0 manifests — the "detects nothing on pages with just <video> elements"
+      // regression. Guard the exact executeScript calls that run each function.
+     const indexSource = fs.readFileSync(
+       path.join(__dirname, '../src/viewer/index.js'),
+       'utf-8'
+       );
+
+      // Capture each call's target + func block (world: 'MAIN' lives on the
+      // target: line, which precedes func:).
+     const probeMatch = indexSource.match(/executeScript\(\{[\s\S]*?func: m3u8DeepProbeFn/);
+     if (!probeMatch) throw new Error('deep probe executeScript call not found');
+     if (!probeMatch[0].includes("world: 'MAIN'")) {
+       throw new Error('deep probe (m3u8DeepProbeFn) must use world: "MAIN" to see page globals');
+       }
+
+     const scrapeMatch = indexSource.match(/executeScript\(\{[\s\S]*?func: scrapeFn/);
+     if (!scrapeMatch) throw new Error('scrape executeScript call not found');
+     if (!scrapeMatch[0].includes("world: 'MAIN'")) {
+       throw new Error('scrape (scrapeFn) must use world: "MAIN" to see page globals');
+       }
+     });
 
   it('should inject interceptors into all frames (content_scripts.all_frames)', () => {
     const manifestSource = fs.readFileSync(
@@ -1211,6 +1238,73 @@ describe('HLS QA: Edge cases', () => {
 
     if (intercepted.length !== 1) {
       throw new Error(`Expected 1 intercept for /manifest URL, got ${intercepted.length}`);
-    }
-  });
-});
+       }
+     });
+    });
+
+/* ========================================================================== */
+/* Test: deep probe must not report route-template / nav "manifests"            */
+/* ========================================================================== */
+
+describe('HLS QA: route-template / nav URL rejection', () => {
+   const probeMatch = indexSource.match(/function m3u8DeepProbeFn\(\)[\s\S]*?^}/m);
+   if (!probeMatch) throw new Error('Could not find m3u8DeepProbeFn in index.js');
+   const probeFnSource = probeMatch[0];
+
+   function runProbeWin(url, globals) {
+     const dom = new JSDOM(
+      '<!DOCTYPE html><html><body><video></video></body></html>',
+        { url, runScripts: 'dangerously' });
+     const win = dom.window;
+     for (const [k, v] of Object.entries(globals)) win[k] = v;
+     const result = win.eval(`(${probeFnSource})();`);
+     if (!Array.isArray(result)) throw new Error('m3u8DeepProbeFn did not return an array');
+     return result;
+      }
+
+     // Reproduces thisvid.com: pageContext.langUrls holds server-side route
+     // TEMPLATES / nav links (e.g. /playlists/, /playlist/%1%/video/%2%/) that the
+     // old /playlist|/manifest substring check misclassified as "manifests".
+   const TEMPLATE_URLS = [
+      'https://thisvid.com/playlists/',
+      'https://thisvid.com/playlists/liked/',
+      'https://thisvid.com/playlists/popular/',
+      'https://thisvid.com/playlists/commented/',
+      'https://thisvid.com/playlists/updated/',
+      'https://thisvid.com/playlists/largest/',
+      'https://thisvid.com/playlist/%1%/video/%2%/',
+      'https://thisvid.com/playlists/?category=%DIR%',
+      'https://thisvid.com/members/%ID%/playlists/',
+      'https://thisvid.com/my/playlists/%ID%/',
+      ];
+
+   it('route templates / nav links are NOT reported as manifests', () => {
+     const result = runProbeWin('https://thisvid.com/videos/redneck/', {
+       pageContext: { langUrls: TEMPLATE_URLS },
+       kvsplayer: { config: { langUrls: TEMPLATE_URLS } },
+        __goon_plugin_data: { manifests: [], sources: [] },
+        });
+     const urls = result.map(m => m.url);
+     const leaked = TEMPLATE_URLS.filter(u => urls.includes(u));
+     if (leaked.length > 0) {
+       throw new Error(`template/nav URLs leaked as manifests: ${leaked.join(', ')}`);
+         }
+       });
+
+   it('a real .m3u8 with a %2B token is still found (no over-rejection)', () => {
+         // %2B is a valid percent-encoded token, not a placeholder — it must survive.
+     const real = 'https://cdn.example.com/live/master.m3u8?h=a%2Bb%2Fc&e=1234';
+     const result = runProbeWin('https://thisvid.com/videos/redneck/', {
+       hls: { url: real },
+       pageContext: { langUrls: TEMPLATE_URLS },
+        });
+     if (!result.some(m => m.url === real)) {
+       throw new Error(`real manifest with %2B token was dropped: ${real} — got: ${result.map(m => m.url).join(', ')}`);
+        }
+         // And the templates still must not leak in the same run.
+     const leaked = TEMPLATE_URLS.filter(u => result.some(m => m.url === u));
+     if (leaked.length > 0) {
+       throw new Error(`templates leaked alongside real manifest: ${leaked.join(', ')}`);
+        }
+       });
+     });
diff --git a/src/viewer/index.js b/src/viewer/index.js
index 32a83e8..04470ef 100644
--- a/src/viewer/index.js
+++ b/src/viewer/index.js
@@ -122,7 +122,11 @@ function m3u8DeepProbeFn() {
   function probeLog(...args) {
     try { console.log('[goon-probe]', ...args); } catch(e) {}
   }
-  probeLog('Deep probe started on:', window.location.href, 'World:', typeof chrome !== 'undefined' ? 'ISOLATED' : 'MAIN');
+  // NOTE: the probe is injected with chrome.scripting's `world: 'MAIN'`, so it
+  // runs in the page's main world and can see page globals. `typeof chrome` is
+  // NOT a reliable world discriminator (extensions expose `chrome` in the
+  // isolated world too), so don't infer it that way.
+  probeLog('Deep probe started on:', window.location.href, '(main world)');
 
   // Helpers available in the MAIN world context
   function resolveUrl(url) {
@@ -134,14 +138,23 @@ function m3u8DeepProbeFn() {
     } catch { return null; }
   }
 
-  function isManifestUrl(url) {
+   function isManifestUrl(url) {
     if (!url) return false;
     const u = url.toLowerCase();
-    return u.includes('.m3u8') || u.includes('.hlsvod') ||
-           (u.includes('/hls/') && u.includes('http')) ||
-           (u.includes('/playlist') && u.includes('http')) ||
-           (u.includes('/manifest') && u.includes('http'));
-  }
+       // Dynamic route TEMPLATES (e.g. /playlist/%1%/ or /videos/{id}/) are nav
+       // links, not fetchable manifests. A placeholder is a `%` that is NOT the
+       // start of a valid %-encoded hex pair (so real tokens like %2B, %2F are
+       // preserved) or a {name} brace token — so genuine %-encoded auth tokens are
+       // never mistaken for template placeholders.
+    if (/%(?!([0-9A-Fa-f]{2}))/i.test(u) || /\{[a-z0-9_]+\}/i.test(u)) return false;
+    // .m3u8 / .hlsvod / /hls/ are concrete HLS signals.
+    if (u.includes('.m3u8') || u.includes('.hlsvod') || u.includes('/hls/')) return true;
+      // Plural collection paths are nav menus, not manifests.
+    if (u.includes('/playlists') || u.includes('/manifests')) return false;
+      // A singular manifest-ish path segment with no concrete file may still be a
+      // real (rare, extensionless) endpoint, so allow it when it carries http.
+    return (u.includes('/manifest') || u.includes('/playlist')) && u.includes('http');
+     }
 
   function walk(obj, path, depth) {
     if (depth > 5) return;
@@ -666,8 +679,9 @@ async function probeHlsManifests(tabId) {
     log('probeHlsManifests: executing deep probe on tab', tabId);
     const raw = await chrome.scripting.executeScript({
       target: { tabId, allFrames: true },
+      world: 'MAIN',
       func: m3u8DeepProbeFn,
-    });
+     });
     // When allFrames is true, destroyed / sandboxed frames may return null entries.
     // Guard: ensure raw is an array, then filter to objects with a defined .result.
     const manifests = Array.isArray(raw)
@@ -778,13 +792,14 @@ async function queryVideoRequests(tabId) {
  * @param {Array} manifests - HLS manifests found by probing
  * @returns {Promise<Object|null>} scrape result or null on error
  */
-async function scrapeTab(tabId, manifests) {
-  try {
+ async function scrapeTab(tabId, manifests) {
+   try {
     const raw = await chrome.scripting.executeScript({
       target: { tabId, allFrames: true },
+      world: 'MAIN',
       func: scrapeFn,
-      args: [manifests],
-    });
+       args: [manifests],
+      });
     // When allFrames is true, destroyed / sandboxed frames may return null entries.
     // Guard: ensure raw is an array, then filter to objects with a defined .result.
     const frameResults = Array.isArray(raw)
@@ -871,23 +886,33 @@ function filterManifests(manifests) {
   * Rejects malformed paths; a manifest-like URL must live in a directory (have a
   * path separator), which filters out obviously-broken single-segment paths.
   */
- function isValidManifestUrl(url) {
-  if (!url || typeof url !== 'string') return false;
-  let urlObj;
-  try {
-    urlObj = new URL(url);
-   } catch {
-    return false;
-    }
-  const path = urlObj.pathname.toLowerCase();
-  const isManifestLike = path.includes('.m3u8') || path.includes('.hlsvod') ||
-         (path.includes('/playlist') && !path.endsWith('.ts')) ||
-         (path.includes('/master') && path.includes('.m3u'));
-  // Not a manifest path — allow it (it may be a direct media source, not HLS).
-  if (!isManifestLike) return true;
-  // A manifest path must sit in a directory, not be a bare top-level file.
-  return path.includes('/');
- }
+  function isValidManifestUrl(url) {
+   if (!url || typeof url !== 'string') return false;
+    // Dynamic route templates / nav links are not fetchable manifests. A
+    // placeholder is a `%` not followed by two hex digits (real %2B etc. are
+    // kept) or a {name} brace token.
+   if (/%(?!([0-9A-Fa-f]{2}))/i.test(url) || /\{[a-z0-9_]+\}/i.test(url)) return false;
+   let urlObj;
+   try {
+     urlObj = new URL(url);
+     } catch {
+     return false;
+      }
+   const path = urlObj.pathname.toLowerCase();
+   // Plural collection paths and trailing-slash directory listings are nav menus,
+   // not manifest files (e.g. /playlists/, /playlists/liked/, /playlist/%id%/).
+   if (path.includes('/playlists') || path.includes('/manifests') || path.endsWith('/')) {
+     return false;
+     }
+   const isManifestLike = path.includes('.m3u8') || path.includes('.hlsvod') ||
+           (path.includes('/master') && path.includes('.m3u')) ||
+           (path.includes('/playlist') && !path.endsWith('.ts')) ||
+           path.includes('/manifest');
+    // Not a manifest path — allow it (it may be a direct media source, not HLS).
+   if (!isManifestLike) return true;
+    // A manifest path must sit in a directory, not be a bare top-level file.
+   return path.includes('/');
+   }
 
 /**
  * Build a normalized video entry from a scraped video object.
@@ -1150,6 +1175,7 @@ findBtn.addEventListener('click', async () => {
         try {
           await chrome.scripting.executeScript({
             target: { tabId: activeTab.id },
+            world: 'MAIN',
             func: () => {
               const diag = [];
               // Check interceptor data