From 0030d031f14f6aa99f9f40a59f5645b0cb1554b2 Mon Sep 17 00:00:00 2001
From: kfj001 <kfj001@gmail.com>
Date: Fri, 28 Aug 2026 01:17:35 -0700
Subject: [PATCH] fix(scan): recover simple <video> detection site-agnostically
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The no-mention-of-pornhub branch stopped finding plain <video> elements on
pages like thisvid.com. Two site-agnostic causes, both fixed without any
site-specific references:

1. Predicate over-rejection: isValidManifestUrl (and the nested isManifestUrl
   in m3u8DeepProbeFn) ran the placeholder/`%`/trailing-slash rejections
   BEFORE the concrete media signals. thisvid serves its direct MP4 WITH a
   trailing slash (.../14644093.mp4/?rnd=...), so `path.endsWith('/')`
   rejected a real, fetchable source. Concrete media signals (.mp4/.webm/.ogv/
   .mpd/.m4s/.m3u8/.hlsvod//hls/) now win outright, so a coincidental '%' or
   trailing slash never sinks a real source — while genuine route templates
   (/playlists/, /playlist/%1%/, /videos/{id}/) are still rejected.

2. Runtime cause on ad-iframe-heavy pages: scrapeTab ran only a MAIN-world
   scrape. A MAIN-world + allFrames injection can reject on pages with
   sandboxed/opaque-origin ad iframes, returning 0 videos even when a <video>
   is visible. scrapeTab now falls back to an isolated-world scrape (no `world`
   key — always permitted) when the MAIN-world run finds nothing. The per-frame
   reduction was extracted into a shared reduceScrapeFrames helper.

Adds __tests__/simple_video.test.mjs: a thisvid isValidManifestUrl battery
(trailing-slash MP4 accepted, templates still rejected, %2B/%2F tokens
preserved) plus a structural guard that scrapeTab keeps both a MAIN-world
primary and an isolated-world fallback.

Co-Authored-By: Claude <noreply@anthropic.com>
---
 __tests__/run.mjs               |   1 +
 __tests__/simple_video.test.mjs | 131 +++++++++++++++++++++++++++
 src/viewer/index.js             | 156 +++++++++++++++++++-------------
 3 files changed, 225 insertions(+), 63 deletions(-)
 create mode 100644 __tests__/simple_video.test.mjs

diff --git a/__tests__/run.mjs b/__tests__/run.mjs
index 88bbdd1..5d1270b 100644
--- a/__tests__/run.mjs
+++ b/__tests__/run.mjs
@@ -7,6 +7,7 @@ import './data_flow.test.mjs';
 import './viewer_integration.test.mjs';
 import './config_validation.test.mjs';
 import './hls_qa.test.mjs';
+import './simple_video.test.mjs';
 import './referer_proxy.test.mjs';
 import { run } from './runner.mjs';
 
diff --git a/__tests__/simple_video.test.mjs b/__tests__/simple_video.test.mjs
new file mode 100644
index 0000000..268e199
--- /dev/null
+++ b/__tests__/simple_video.test.mjs
@@ -0,0 +1,131 @@
+// Regression tests: simple <video> / direct-media detection.
+//
+// The feature branch broke detection of pages that expose a plain <video>
+// (e.g. thisvid.com, which serves a direct MP4 with a TRAILING SLASH and a
+// ?rnd= query: .../14644093/14644093.mp4/?rnd=...). Two site-agnostic fixes
+// are guarded here:
+//   1) The URL predicate (isValidManifestUrl) must let concrete media signals
+//      (.mp4/.m3u8/.webm/... and /hls/) WIN over the placeholder/`%`/trailing-
+//      slash rejections, so a real direct-media URL is not mistaken for a
+//      route template.
+//   2) scrapeTab must fall back to an ISOLATED-world scrape when the MAIN-world
+//      scrape yields nothing — the runtime cause of "detects nothing" on thisvid
+//      is a MAIN-world + allFrames rejection on its ad-iframe-heavy page.
+import { describe, it } from './runner.mjs';
+import { JSDOM } from 'jsdom';
+import * as fs from 'fs';
+import * as path from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const indexSource = fs.readFileSync(
+  path.join(__dirname, '../src/viewer/index.js'),
+   'utf-8'
+);
+
+// Extract a `function <name>(...) {…}` by balanced-brace scan. Robust to the
+// file's quirky/mixed indentation (a column-0 `}` heuristic would mis-truncate
+// nested or oddly-indented bodies).
+function extractFn(src, name) {
+  const re = new RegExp('function ' + name + '\\s*\\(');
+  const m = re.exec(src);
+  if (!m) throw new Error('not found: ' + name);
+  const start = m.index;
+  let i = src.indexOf('{', start), depth = 0;
+  for (; i < src.length; i++) {
+    const c = src[i];
+    if (c === '{') depth++;
+    else if (c === '}') { depth--; if (depth === 0) return src.slice(start, i + 1); }
+   }
+  throw new Error('unbalanced: ' + name);
+}
+
+// The thisvid direct-media URL, exactly as served (note the trailing slash +
+// ?rnd= query). This is the concrete case the predicate used to reject.
+const THISVID_MP4 =
+  'https://thisvid.com/get_file/5/6cae22f8b3ba9d74f10cc2f8d060af738c8d30ddf5/14644000/14644093/14644093.mp4/?rnd=1787811205406';
+
+describe('HLS QA: simple <video> direct-media predicate (thisvid regression)', () => {
+  // Extract and evaluate the REAL isValidManifestUrl from index.js in a browser
+  // window so its `new URL(...)` / string behavior matches production.
+  const isValidSrc = extractFn(indexSource, 'isValidManifestUrl');
+
+  function assertValid(url, expectValid, label) {
+    const dom = new JSDOM(
+      '<!DOCTYPE html><html><body></body></html>',
+       { url: 'https://thisvid.com/videos/x/', runScripts: 'dangerously' }
+     );
+    const got = dom.window.eval('(' + isValidSrc + ')(' + JSON.stringify(url) + ')');
+    if (got !== expectValid) {
+      throw new Error(
+        `${label || 'isValidManifestUrl'}(${url}) = ${got}, expected ${expectValid}`
+       );
+    }
+  }
+
+  it('accepts thisvid\'s real direct-MP4 URL WITH a trailing slash + ?rnd=', () => {
+    // Before the fix, `path.endsWith('/')` rejected this real media source.
+    assertValid(THISVID_MP4, true, 'thisvid trailing-slash MP4');
+   });
+
+  it('accepts a direct .mp4 with no trailing slash', () => {
+    assertValid('https://cdn.example.com/vids/abc123.mp4', true, 'plain MP4');
+   });
+
+  it('accepts real .m3u8 / .webm / .mpd / .m4s / /hls/ media signals', () => {
+    assertValid('https://cdn.example.com/live/master.m3u8', true, 'master.m3u8');
+    assertValid('https://cdn.example.com/seg/webm/v.webm', true, 'webm');
+    assertValid('https://cdn.example.com/seg/dash/master.mpd', true, 'mpd');
+    assertValid('https://cdn.example.com/seg/1-0.m4s', true, 'm4s');
+    assertValid('https://cdn.example.com/hls/abc/playlist.m3u8', true, '/hls/');
+   });
+
+  it('does not over-reject a real .m3u8 carrying a valid %2B/%2F token', () => {
+     // %2B/%2F are valid percent-encoded hex pairs — a genuine auth token, not a
+    // route-template placeholder, so it must survive.
+    assertValid('https://cdn.example.com/live/master.m3u8?h=a%2Bb%2Fc&e=1234', true, '%2B token');
+   });
+
+  it('still rejects thisvid route-template / nav URLs (no over-acceptance)', () => {
+     // The fix must not trade one regression for the other: the route templates
+    // thisvid\'s pageContext.langUrls leaks must still be rejected.
+    assertValid('https://thisvid.com/playlists/', false, '/playlists/');
+    assertValid('https://thisvid.com/playlists/liked/', false, '/playlists/liked/');
+    assertValid('https://thisvid.com/playlist/%1%/video/%2%/', false, '/playlist/%1%/');
+    assertValid('https://thisvid.com/members/%ID%/playlists/', false, '/members/%ID%/playlists/');
+    assertValid('https://thisvid.com/videos/%DIR%/', false, '/videos/%DIR%/');
+   });
+});
+
+describe('HLS QA: scrapeTab isolated-world fallback (thisvid regression)', () => {
+  // Extract the whole scrapeTab function (balanced braces, indentation-agnostic).
+  const scrapeSrc = extractFn(indexSource, 'scrapeTab');
+
+  it('runs an isolated-world scrape attempt (no `world` key) as a fallback', () => {
+    const calls = scrapeSrc.match(/executeScript\(\s*\{[\s\S]*?\}\s*\)/g) || [];
+    if (calls.length === 0) {
+      throw new Error('scrapeTab has no executeScript call');
+     }
+     // At least one executeScript must omit `world` so it runs in the always-
+    // permitted isolated world (recovers simple videos a MAIN-world injection
+    // rejects on sandboxed/opaque-origin ad iframes).
+    const hasIsolated = calls.some((c) => !/\bworld\s*:/.test(c));
+    if (!hasIsolated) {
+      throw new Error(
+        'scrapeTab has no isolated-world (no `world:`) executeScript fallback — ' +
+         'a MAIN-world+allFrames rejection on an ad-iframe-heavy page leaves 0 videos'
+       );
+    }
+   });
+
+  it('primary scrape still uses world: \'MAIN\' (reads page globals)', () => {
+    const calls = scrapeSrc.match(/executeScript\(\s*\{[\s\S]*?\}\s*\)/g) || [];
+    const hasMain = calls.some((c) => /world\s*:\s*['"]MAIN['"]/.test(c));
+    if (!hasMain) {
+      throw new Error(
+        'scrapeTab primary attempt must use world: "MAIN" to read page globals ' +
+         '(interceptor data, Hls instances, custom <video> props)'
+       );
+    }
+   });
+});
diff --git a/src/viewer/index.js b/src/viewer/index.js
index 04470ef..3089ee4 100644
--- a/src/viewer/index.js
+++ b/src/viewer/index.js
@@ -139,21 +139,23 @@ function m3u8DeepProbeFn() {
   }
 
    function isManifestUrl(url) {
-    if (!url) return false;
-    const u = url.toLowerCase();
-       // Dynamic route TEMPLATES (e.g. /playlist/%1%/ or /videos/{id}/) are nav
-       // links, not fetchable manifests. A placeholder is a `%` that is NOT the
-       // start of a valid %-encoded hex pair (so real tokens like %2B, %2F are
-       // preserved) or a {name} brace token — so genuine %-encoded auth tokens are
-       // never mistaken for template placeholders.
-    if (/%(?!([0-9A-Fa-f]{2}))/i.test(u) || /\{[a-z0-9_]+\}/i.test(u)) return false;
-    // .m3u8 / .hlsvod / /hls/ are concrete HLS signals.
-    if (u.includes('.m3u8') || u.includes('.hlsvod') || u.includes('/hls/')) return true;
+  if (!url) return false;
+  const u = url.toLowerCase();
+      // Concrete HLS/manifest signals win over the template heuristics below: a
+      // real .m3u8/.hlsvod//hls/ path is fetchable even if it happens to carry a
+      // trailing '%' query token or a trailing slash — so check these first.
+  if (u.includes('.m3u8') || u.includes('.hlsvod') || u.includes('/hls/')) return true;
+      // Dynamic route TEMPLATES (e.g. /playlist/%1%/ or /videos/{id}/) are nav
+      // links, not fetchable manifests. A placeholder is a `%` that is NOT the
+      // start of a valid %-encoded hex pair (so real tokens like %2B, %2F are
+      // preserved) or a {name} brace token — so genuine %-encoded auth tokens are
+      // never mistaken for template placeholders.
+  if (/%(?!([0-9A-Fa-f]{2}))/i.test(u) || /\{[a-z0-9_]+\}/i.test(u)) return false;
       // Plural collection paths are nav menus, not manifests.
-    if (u.includes('/playlists') || u.includes('/manifests')) return false;
+  if (u.includes('/playlists') || u.includes('/manifests')) return false;
       // A singular manifest-ish path segment with no concrete file may still be a
       // real (rare, extensionless) endpoint, so allow it when it carries http.
-    return (u.includes('/manifest') || u.includes('/playlist')) && u.includes('http');
+  return (u.includes('/manifest') || u.includes('/playlist')) && u.includes('http');
      }
 
   function walk(obj, path, depth) {
@@ -792,37 +794,64 @@ async function queryVideoRequests(tabId) {
  * @param {Array} manifests - HLS manifests found by probing
  * @returns {Promise<Object|null>} scrape result or null on error
  */
- async function scrapeTab(tabId, manifests) {
-   try {
-    const raw = await chrome.scripting.executeScript({
-      target: { tabId, allFrames: true },
-      world: 'MAIN',
-      func: scrapeFn,
-       args: [manifests],
-      });
+/**
+ * Reduce a per-frame executeScript result array to the single largest-video
+ * result (or a "no videos" result / null). Shared by the MAIN- and
+ * isolated-world scrape attempts in scrapeTab.
+ */
+function reduceScrapeFrames(raw) {
     // When allFrames is true, destroyed / sandboxed frames may return null entries.
     // Guard: ensure raw is an array, then filter to objects with a defined .result.
-    const frameResults = Array.isArray(raw)
-      ? raw.filter(r => r != null && typeof r === 'object' && 'result' in r)
-           .map(r => r.result)
-      : [];
-    const valid = frameResults.filter(r => r && r.videos && r.videos.length > 0);
-    if (valid.length === 0) {
-      const firstWithResult = frameResults.find(r => r != null);
-      return firstWithResult || null;
-    }
-    valid.sort((a, b) => {
-      const aArea = a.videos[0]?.area || 0;
-      const bArea = b.videos[0]?.area || 0;
-      return bArea - aArea;
-    });
-    return valid[0];
-  } catch(e) {
-    warn('scrapeTab: tab', tabId, 'error:', e && e.message ? e.message : e);
-    return null;
-  }
+  const frameResults = Array.isArray(raw)
+       ? raw.filter(r => r != null && typeof r === 'object' && 'result' in r)
+            .map(r => r.result)
+       : [];
+  const valid = frameResults.filter(r => r && r.videos && r.videos.length > 0);
+  if (valid.length === 0) {
+    const firstWithResult = frameResults.find(r => r != null);
+    return firstWithResult || null;
+     }
+  valid.sort((a, b) => {
+    const aArea = a.videos[0]?.area || 0;
+    const bArea = b.videos[0]?.area || 0;
+    return bArea - aArea;
+     });
+  return valid[0];
 }
 
+ async function scrapeTab(tabId, manifests) {
+       // Primary: MAIN world — scrapeFn can read page globals the main frame
+       // exposes (interceptor annotations, Hls instances, custom <video> props).
+    try {
+      const mainRaw = await chrome.scripting.executeScript({
+        target: { tabId, allFrames: true },
+        world: 'MAIN',
+        func: scrapeFn,
+         args: [manifests],
+          });
+      let result = reduceScrapeFrames(mainRaw);
+          // Isolated-world fallback: some pages embed sandboxed / opaque-origin ad
+          // iframes that make the MAIN-world injection reject, so the primary run can
+          // return no videos even when a <video> is visible. scrapeFn only needs the
+          // shared DOM plus the passed-in manifest list, so an isolated-world run (no
+          // `world` key — always permitted) still sees the same <video> elements.
+      if (!result || !(result.videos && result.videos.length > 0)) {
+        const isoRaw = await chrome.scripting.executeScript({
+          target: { tabId, allFrames: true },
+          func: scrapeFn,
+           args: [manifests],
+            });
+        const iso = reduceScrapeFrames(isoRaw);
+        if (iso && iso.videos && iso.videos.length > 0) result = iso;
+        else if (!result) result = iso;
+          }
+       return result;
+        } catch(e) {
+      warn('scrapeTab: tab', tabId, 'error:', e && e.message ? e.message : e);
+      return null;
+        }
+     }
+
 /**
  * Filter intercepted URLs to prefer actual HLS manifests over segment files.
  * Interceptors capture both .m3u8 manifests and .ts segment URLs (any URL
@@ -887,31 +916,32 @@ function filterManifests(manifests) {
   * path separator), which filters out obviously-broken single-segment paths.
   */
   function isValidManifestUrl(url) {
-   if (!url || typeof url !== 'string') return false;
-    // Dynamic route templates / nav links are not fetchable manifests. A
-    // placeholder is a `%` not followed by two hex digits (real %2B etc. are
-    // kept) or a {name} brace token.
-   if (/%(?!([0-9A-Fa-f]{2}))/i.test(url) || /\{[a-z0-9_]+\}/i.test(url)) return false;
-   let urlObj;
-   try {
-     urlObj = new URL(url);
-     } catch {
-     return false;
-      }
-   const path = urlObj.pathname.toLowerCase();
-   // Plural collection paths and trailing-slash directory listings are nav menus,
-   // not manifest files (e.g. /playlists/, /playlists/liked/, /playlist/%id%/).
-   if (path.includes('/playlists') || path.includes('/manifests') || path.endsWith('/')) {
-     return false;
+  if (!url || typeof url !== 'string') return false;
+  let urlObj;
+  try {
+    urlObj = new URL(url);
+   } catch {
+    return false;
      }
-   const isManifestLike = path.includes('.m3u8') || path.includes('.hlsvod') ||
-           (path.includes('/master') && path.includes('.m3u')) ||
-           (path.includes('/playlist') && !path.endsWith('.ts')) ||
-           path.includes('/manifest');
-    // Not a manifest path — allow it (it may be a direct media source, not HLS).
-   if (!isManifestLike) return true;
-    // A manifest path must sit in a directory, not be a bare top-level file.
-   return path.includes('/');
+  const path = urlObj.pathname.toLowerCase();
+       // 1) Concrete media/manifest signals win outright: a real file extension or
+       // /hls/ path is fetchable, even with a trailing slash (thisvid serves
+       // .../14644093.mp4/) or a token-bearing query — so check these before the
+       // template heuristics and never let a coincidental '%'/'/' sink a source.
+  if (path.includes('.m3u8') || path.includes('.hlsvod') || path.includes('/hls/') ||
+        path.includes('/manifest') || path.includes('.mp4') || path.includes('.webm') ||
+        path.includes('.ogv') || path.includes('.mpd') || path.includes('.m4s')) return true;
+       // 2) Dynamic route templates / nav links: a `%` that isn't a valid %-encoded
+       // hex pair (real %2B etc. are kept) or a {name} brace token.
+  if (/%(?!([0-9A-Fa-f]{2}))/i.test(url) || /\{[a-z0-9_]+\}/i.test(url)) return false;
+       // 3) Plural collection paths and trailing-slash directory listings are nav
+       // menus, not manifest files (e.g. /playlists/, /playlists/liked/).
+  if (path.includes('/playlists') || path.includes('/manifests') || path.endsWith('/')) return false;
+       // 4) A singular /playlist endpoint may be a real (extensionless) manifest;
+       // allow it when it sits in a directory.
+  if (path.includes('/playlist')) return path.includes('/');
+       // 5) Otherwise treat as a direct media source — allow it.
+  return true;
    }
 
 /**