/** * Search engine spider / bot detection based on User-Agent headers. * * TypeScript mirror of backend/app/spider_detection.py. * Used by the SSR server to decide whether to render full HTML (for spiders) * or serve the static CSR app shell (for regular users). * * Usage: * import { isSpider, getSpiderInfo } from './spider_detection'; * * if (isSpider(userAgent)) { * // Full SSR rendering with fresh API data * } else { * // Serve static CSR app shell * } */ export class SpiderEngine { public static readonly GOOGLE = 'Google'; public static readonly BING = 'Bing'; public static readonly YAHOO = 'Yahoo'; public static readonly DUCKDUCKGO = 'DuckDuckGo'; public static readonly YANDEX = 'Yandex'; public static readonly BAIDU = 'Baidu'; public static readonly SOGOU = 'Sogou'; public static readonly APPLE = 'Apple'; public static readonly PETALBOT = 'PetalBot'; public static readonly IA_ARCHIVER = 'IA Archiver'; public static readonly FACEBOOK = 'Facebook'; public static readonly TWITTER = 'Twitter'; public static readonly LINKEDIN = 'LinkedIn'; public static readonly WHATSAPP = 'WhatsApp'; public static readonly OTHER = 'Other'; } export interface SpiderInfo { engine: string; name: string; category: 'search' | 'social' | 'other'; } // ── Spider patterns ────────────────────────────────────────── // Each tuple: [compiled-regex, engine-label, category] // Order matters: more specific patterns first to avoid false positives. type SpiderPattern = [RegExp, string, 'search' | 'social' | 'other']; const SPIDER_PATTERNS: SpiderPattern[] = [ // ── Google family ───────────────────────────────────── [/(?:Googlebot\/[\d.]+)/i, 'Google', 'search'], [/(?:Googlebot-Mobile)/i, 'Google', 'search'], [/(?:Googlebot-Image)/i, 'Google', 'search'], [/(?:Googlebot-News)/i, 'Google', 'search'], [/(?:Mediapartners-Google)/i, 'Google', 'search'], [/(?:Storebot-Google)/i, 'Google', 'search'], [/(?:Google-AdSense)/i, 'Google', 'search'], // ── Bing / Microsoft ────────────────────────────────── [/(?:bingbot\/[\d.]+)/i, 'Bing', 'search'], [/(?:msnbot\/[\d.]+)/i, 'Bing', 'search'], [/(?:msnbot-media\/[\d.]+)/i, 'Bing', 'search'], [/(?:bingadsbot\/[\d.]+)/i, 'Bing', 'search'], [/(?:webpreview)/i, 'Bing', 'search'], // ── Yahoo / Slurp ───────────────────────────────────── [/(?:Yahoo!\s*Slurp)/i, 'Yahoo', 'search'], [/(?:Yahoo!\s*Deer\s*Spider)/i, 'Yahoo', 'search'], [/(?:YandexBot)/i, 'Yandex', 'search'], // ── DuckDuckGo ──────────────────────────────────────── [/(?:DuckDuckBot\/[\d.]+)/i, 'DuckDuckGo', 'search'], // ── Yandex (RU) ─────────────────────────────────────── [/(?:YandexBot\/[\d.]+)/i, 'Yandex', 'search'], [/(?:YandexImages\/[\d.]+)/i, 'Yandex', 'search'], [/(?:YandexAccessibilityBot)/i, 'Yandex', 'search'], [/(?:YandexMobileBot)/i, 'Yandex', 'search'], [/(?:YandexMetrika)/i, 'Yandex', 'search'], [/(?:YandexDirect)/i, 'Yandex', 'search'], [/(?:YandexMobileBot)/i, 'Yandex', 'search'], // ── Baidu (CN) ──────────────────────────────────────── [/(?:Baiduspider\/[\d.]+)/i, 'Baidu', 'search'], [/(?:Baiduboxapp)/i, 'Baidu', 'search'], // ── Sogou (CN) ──────────────────────────────────────── [/(?:Sogou\s*Spider)/i, 'Sogou', 'search'], [/(?:Sogou\s*web\s*spider)/i, 'Sogou', 'search'], // ── Apple ───────────────────────────────────────────── [/(?:Applebot\/[\d.]+)/i, 'Apple', 'search'], [/(?:Applebot-Extended)/i, 'Apple', 'search'], // ── PetalBot (China) ────────────────────────────────── [/(?:petalbot)/i, 'PetalBot', 'search'], // ── Internet Archive ────────────────────────────────── [/(?:ia_archiver)/i, 'IA Archiver', 'other'], [/(?:archive\.org_bot)/i, 'IA Archiver', 'other'], // ── Social media bots ───────────────────────────────── [/(?:facebookexternalhit\/[\d.]+)/i, 'Facebook', 'social'], [/(?:facebookplatform)/i, 'Facebook', 'social'], [/(?:Twitterbot\/[\d.]+)/i, 'Twitter', 'social'], [/(?:LinkedInBot\/[\d.]+)/i, 'LinkedIn', 'social'], [/(?:WhatsApp\/[\d.]+)/i, 'WhatsApp', 'social'], [/(?:W3C_Validator)/i, 'W3C', 'other'], // ── Generic / catch-all bot signatures ──────────────── [/(?:Bot\/[\d.])/i, 'Other', 'other'], [/(?:;\s*bot\s*;?)/i, 'Other', 'other'], [/(?:spider)/i, 'Other', 'other'], [/(?:crawler)/i, 'Other', 'other'], ]; // ── Public API ─────────────────────────────────────────────── export function isSpider(userAgent: string): boolean { if (!userAgent) return false; for (const [pattern] of SPIDER_PATTERNS) { if (pattern.test(userAgent)) return true; } return false; } export function getSpiderInfo(userAgent: string): SpiderInfo | null { if (!userAgent) return null; for (const [pattern, engine, category] of SPIDER_PATTERNS) { const match = userAgent.match(pattern); if (match) { return { engine, name: match[0], category }; } } return null; } export function isSearchSpider(userAgent: string): boolean { const info = getSpiderInfo(userAgent); return info?.category === 'search'; } export function getUserAgent(headers: Record): string { const ua = headers['user-agent'] || headers['User-Agent']; if (typeof ua === 'string') return ua; if (Array.isArray(ua) && ua.length > 0) return ua[0]; return ''; }