fix(scan): recover simple <video> detection site-agnostically
The no-mention-of-pornhub branch stopped finding plain <video> elements on
pages like thisvid.com. Two site-agnostic causes, both fixed without any
site-specific references:
1. Predicate over-rejection: isValidManifestUrl (and the nested isManifestUrl
in m3u8DeepProbeFn) ran the placeholder/`%`/trailing-slash rejections
BEFORE the concrete media signals. thisvid serves its direct MP4 WITH a
trailing slash (.../14644093.mp4/?rnd=...), so `path.endsWith('/')`
rejected a real, fetchable source. Concrete media signals (.mp4/.webm/.ogv/
.mpd/.m4s/.m3u8/.hlsvod//hls/) now win outright, so a coincidental '%' or
trailing slash never sinks a real source — while genuine route templates
(/playlists/, /playlist/%1%/, /videos/{id}/) are still rejected.
2. Runtime cause on ad-iframe-heavy pages: scrapeTab ran only a MAIN-world
scrape. A MAIN-world + allFrames injection can reject on pages with
sandboxed/opaque-origin ad iframes, returning 0 videos even when a <video>
is visible. scrapeTab now falls back to an isolated-world scrape (no `world`
key — always permitted) when the MAIN-world run finds nothing. The per-frame
reduction was extracted into a shared reduceScrapeFrames helper.
Adds __tests__/simple_video.test.mjs: a thisvid isValidManifestUrl battery
(trailing-slash MP4 accepted, templates still rejected, %2B/%2F tokens
preserved) plus a structural guard that scrapeTab keeps both a MAIN-world
primary and an isolated-world fallback.
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
1 parent
7139c5ea6f
commit
0030d031f1
3 files changed
+225
-63
No files matched your search
@@ -7,6 +7,7 @@ import './data_flow.test.mjs';
|
||||
import './viewer_integration.test.mjs';
|
||||
import './config_validation.test.mjs';
|
||||
import './hls_qa.test.mjs';
|
||||
import './simple_video.test.mjs';
|
||||
import './referer_proxy.test.mjs';
|
||||
import { run } from './runner.mjs';
|
||||
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
// Regression tests: simple <video> / direct-media detection.
|
||||
//
|
||||
// The feature branch broke detection of pages that expose a plain <video>
|
||||
// (e.g. thisvid.com, which serves a direct MP4 with a TRAILING SLASH and a
|
||||
// ?rnd= query: .../14644093/14644093.mp4/?rnd=...). Two site-agnostic fixes
|
||||
// are guarded here:
|
||||
// 1) The URL predicate (isValidManifestUrl) must let concrete media signals
|
||||
// (.mp4/.m3u8/.webm/... and /hls/) WIN over the placeholder/`%`/trailing-
|
||||
// slash rejections, so a real direct-media URL is not mistaken for a
|
||||
// route template.
|
||||
// 2) scrapeTab must fall back to an ISOLATED-world scrape when the MAIN-world
|
||||
// scrape yields nothing — the runtime cause of "detects nothing" on thisvid
|
||||
// is a MAIN-world + allFrames rejection on its ad-iframe-heavy page.
|
||||
import { describe, it } from './runner.mjs';
|
||||
import { JSDOM } from 'jsdom';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const indexSource = fs.readFileSync(
|
||||
path.join(__dirname, '../src/viewer/index.js'),
|
||||
'utf-8'
|
||||
);
|
||||
|
||||
// Extract a `function <name>(...) {…}` by balanced-brace scan. Robust to the
|
||||
// file's quirky/mixed indentation (a column-0 `}` heuristic would mis-truncate
|
||||
// nested or oddly-indented bodies).
|
||||
function extractFn(src, name) {
|
||||
const re = new RegExp('function ' + name + '\\s*\\(');
|
||||
const m = re.exec(src);
|
||||
if (!m) throw new Error('not found: ' + name);
|
||||
const start = m.index;
|
||||
let i = src.indexOf('{', start), depth = 0;
|
||||
for (; i < src.length; i++) {
|
||||
const c = src[i];
|
||||
if (c === '{') depth++;
|
||||
else if (c === '}') { depth--; if (depth === 0) return src.slice(start, i + 1); }
|
||||
}
|
||||
throw new Error('unbalanced: ' + name);
|
||||
}
|
||||
|
||||
// The thisvid direct-media URL, exactly as served (note the trailing slash +
|
||||
// ?rnd= query). This is the concrete case the predicate used to reject.
|
||||
const THISVID_MP4 =
|
||||
'https://thisvid.com/get_file/5/6cae22f8b3ba9d74f10cc2f8d060af738c8d30ddf5/14644000/14644093/14644093.mp4/?rnd=1787811205406';
|
||||
|
||||
describe('HLS QA: simple <video> direct-media predicate (thisvid regression)', () => {
|
||||
// Extract and evaluate the REAL isValidManifestUrl from index.js in a browser
|
||||
// window so its `new URL(...)` / string behavior matches production.
|
||||
const isValidSrc = extractFn(indexSource, 'isValidManifestUrl');
|
||||
|
||||
function assertValid(url, expectValid, label) {
|
||||
const dom = new JSDOM(
|
||||
'<!DOCTYPE html><html><body></body></html>',
|
||||
{ url: 'https://thisvid.com/videos/x/', runScripts: 'dangerously' }
|
||||
);
|
||||
const got = dom.window.eval('(' + isValidSrc + ')(' + JSON.stringify(url) + ')');
|
||||
if (got !== expectValid) {
|
||||
throw new Error(
|
||||
`${label || 'isValidManifestUrl'}(${url}) = ${got}, expected ${expectValid}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
it('accepts thisvid\'s real direct-MP4 URL WITH a trailing slash + ?rnd=', () => {
|
||||
// Before the fix, `path.endsWith('/')` rejected this real media source.
|
||||
assertValid(THISVID_MP4, true, 'thisvid trailing-slash MP4');
|
||||
});
|
||||
|
||||
it('accepts a direct .mp4 with no trailing slash', () => {
|
||||
assertValid('https://cdn.example.com/vids/abc123.mp4', true, 'plain MP4');
|
||||
});
|
||||
|
||||
it('accepts real .m3u8 / .webm / .mpd / .m4s / /hls/ media signals', () => {
|
||||
assertValid('https://cdn.example.com/live/master.m3u8', true, 'master.m3u8');
|
||||
assertValid('https://cdn.example.com/seg/webm/v.webm', true, 'webm');
|
||||
assertValid('https://cdn.example.com/seg/dash/master.mpd', true, 'mpd');
|
||||
assertValid('https://cdn.example.com/seg/1-0.m4s', true, 'm4s');
|
||||
assertValid('https://cdn.example.com/hls/abc/playlist.m3u8', true, '/hls/');
|
||||
});
|
||||
|
||||
it('does not over-reject a real .m3u8 carrying a valid %2B/%2F token', () => {
|
||||
// %2B/%2F are valid percent-encoded hex pairs — a genuine auth token, not a
|
||||
// route-template placeholder, so it must survive.
|
||||
assertValid('https://cdn.example.com/live/master.m3u8?h=a%2Bb%2Fc&e=1234', true, '%2B token');
|
||||
});
|
||||
|
||||
it('still rejects thisvid route-template / nav URLs (no over-acceptance)', () => {
|
||||
// The fix must not trade one regression for the other: the route templates
|
||||
// thisvid\'s pageContext.langUrls leaks must still be rejected.
|
||||
assertValid('https://thisvid.com/playlists/', false, '/playlists/');
|
||||
assertValid('https://thisvid.com/playlists/liked/', false, '/playlists/liked/');
|
||||
assertValid('https://thisvid.com/playlist/%1%/video/%2%/', false, '/playlist/%1%/');
|
||||
assertValid('https://thisvid.com/members/%ID%/playlists/', false, '/members/%ID%/playlists/');
|
||||
assertValid('https://thisvid.com/videos/%DIR%/', false, '/videos/%DIR%/');
|
||||
});
|
||||
});
|
||||
|
||||
describe('HLS QA: scrapeTab isolated-world fallback (thisvid regression)', () => {
|
||||
// Extract the whole scrapeTab function (balanced braces, indentation-agnostic).
|
||||
const scrapeSrc = extractFn(indexSource, 'scrapeTab');
|
||||
|
||||
it('runs an isolated-world scrape attempt (no `world` key) as a fallback', () => {
|
||||
const calls = scrapeSrc.match(/executeScript\(\s*\{[\s\S]*?\}\s*\)/g) || [];
|
||||
if (calls.length === 0) {
|
||||
throw new Error('scrapeTab has no executeScript call');
|
||||
}
|
||||
// At least one executeScript must omit `world` so it runs in the always-
|
||||
// permitted isolated world (recovers simple videos a MAIN-world injection
|
||||
// rejects on sandboxed/opaque-origin ad iframes).
|
||||
const hasIsolated = calls.some((c) => !/\bworld\s*:/.test(c));
|
||||
if (!hasIsolated) {
|
||||
throw new Error(
|
||||
'scrapeTab has no isolated-world (no `world:`) executeScript fallback — ' +
|
||||
'a MAIN-world+allFrames rejection on an ad-iframe-heavy page leaves 0 videos'
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it('primary scrape still uses world: \'MAIN\' (reads page globals)', () => {
|
||||
const calls = scrapeSrc.match(/executeScript\(\s*\{[\s\S]*?\}\s*\)/g) || [];
|
||||
const hasMain = calls.some((c) => /world\s*:\s*['"]MAIN['"]/.test(c));
|
||||
if (!hasMain) {
|
||||
throw new Error(
|
||||
'scrapeTab primary attempt must use world: "MAIN" to read page globals ' +
|
||||
'(interceptor data, Hls instances, custom <video> props)'
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user