kmtnflower/spider_detection.ts

/**
 * Search engine spider / bot detection based on User-Agent headers.
 *
 * TypeScript mirror of backend/app/spider_detection.py.
 * Used by the SSR server to decide whether to render full HTML (for spiders)
 * or serve the static CSR app shell (for regular users).
 *
 * Usage:
 *   import { isSpider, getSpiderInfo } from './spider_detection';
 *
 *   if (isSpider(userAgent)) {
 *     // Full SSR rendering with fresh API data
 *   } else {
 *     // Serve static CSR app shell
 *   }
 */

export class SpiderEngine {
  public static readonly GOOGLE = 'Google';
  public static readonly BING = 'Bing';
  public static readonly YAHOO = 'Yahoo';
  public static readonly DUCKDUCKGO = 'DuckDuckGo';
  public static readonly YANDEX = 'Yandex';
  public static readonly BAIDU = 'Baidu';
  public static readonly SOGOU = 'Sogou';
  public static readonly APPLE = 'Apple';
  public static readonly PETALBOT = 'PetalBot';
  public static readonly IA_ARCHIVER = 'IA Archiver';
  public static readonly FACEBOOK = 'Facebook';
  public static readonly TWITTER = 'Twitter';
  public static readonly LINKEDIN = 'LinkedIn';
  public static readonly WHATSAPP = 'WhatsApp';
  public static readonly OTHER = 'Other';
}

export interface SpiderInfo {
  engine: string;
  name: string;
  category: 'search' | 'social' | 'other';
}

// ── Spider patterns ──────────────────────────────────────────
// Each tuple: [compiled-regex, engine-label, category]
// Order matters: more specific patterns first to avoid false positives.

type SpiderPattern = [RegExp, string, 'search' | 'social' | 'other'];

const SPIDER_PATTERNS: SpiderPattern[] = [
  // ── Google family ─────────────────────────────────────
  [/(?:Googlebot\/[\d.]+)/i, 'Google', 'search'],
  [/(?:Googlebot-Mobile)/i, 'Google', 'search'],
  [/(?:Googlebot-Image)/i, 'Google', 'search'],
  [/(?:Googlebot-News)/i, 'Google', 'search'],
  [/(?:Mediapartners-Google)/i, 'Google', 'search'],
  [/(?:Storebot-Google)/i, 'Google', 'search'],
  [/(?:Google-AdSense)/i, 'Google', 'search'],

  // ── Bing / Microsoft ──────────────────────────────────
  [/(?:bingbot\/[\d.]+)/i, 'Bing', 'search'],
  [/(?:msnbot\/[\d.]+)/i, 'Bing', 'search'],
  [/(?:msnbot-media\/[\d.]+)/i, 'Bing', 'search'],
  [/(?:bingadsbot\/[\d.]+)/i, 'Bing', 'search'],
  [/(?:webpreview)/i, 'Bing', 'search'],

  // ── Yahoo / Slurp ─────────────────────────────────────
  [/(?:Yahoo!\s*Slurp)/i, 'Yahoo', 'search'],
  [/(?:Yahoo!\s*Deer\s*Spider)/i, 'Yahoo', 'search'],
  [/(?:YandexBot)/i, 'Yandex', 'search'],

  // ── DuckDuckGo ────────────────────────────────────────
  [/(?:DuckDuckBot\/[\d.]+)/i, 'DuckDuckGo', 'search'],

  // ── Yandex (RU) ───────────────────────────────────────
  [/(?:YandexBot\/[\d.]+)/i, 'Yandex', 'search'],
  [/(?:YandexImages\/[\d.]+)/i, 'Yandex', 'search'],
  [/(?:YandexAccessibilityBot)/i, 'Yandex', 'search'],
  [/(?:YandexMobileBot)/i, 'Yandex', 'search'],
  [/(?:YandexMetrika)/i, 'Yandex', 'search'],
  [/(?:YandexDirect)/i, 'Yandex', 'search'],
  [/(?:YandexMobileBot)/i, 'Yandex', 'search'],

  // ── Baidu (CN) ────────────────────────────────────────
  [/(?:Baiduspider\/[\d.]+)/i, 'Baidu', 'search'],
  [/(?:Baiduboxapp)/i, 'Baidu', 'search'],

  // ── Sogou (CN) ────────────────────────────────────────
  [/(?:Sogou\s*Spider)/i, 'Sogou', 'search'],
  [/(?:Sogou\s*web\s*spider)/i, 'Sogou', 'search'],

  // ── Apple ─────────────────────────────────────────────
  [/(?:Applebot\/[\d.]+)/i, 'Apple', 'search'],
  [/(?:Applebot-Extended)/i, 'Apple', 'search'],

  // ── PetalBot (China) ──────────────────────────────────
  [/(?:petalbot)/i, 'PetalBot', 'search'],

  // ── Internet Archive ──────────────────────────────────
  [/(?:ia_archiver)/i, 'IA Archiver', 'other'],
  [/(?:archive\.org_bot)/i, 'IA Archiver', 'other'],

  // ── Social media bots ─────────────────────────────────
  [/(?:facebookexternalhit\/[\d.]+)/i, 'Facebook', 'social'],
  [/(?:facebookplatform)/i, 'Facebook', 'social'],
  [/(?:Twitterbot\/[\d.]+)/i, 'Twitter', 'social'],
  [/(?:LinkedInBot\/[\d.]+)/i, 'LinkedIn', 'social'],
  [/(?:WhatsApp\/[\d.]+)/i, 'WhatsApp', 'social'],
  [/(?:W3C_Validator)/i, 'W3C', 'other'],

  // ── Generic / catch-all bot signatures ────────────────
  [/(?:Bot\/[\d.])/i, 'Other', 'other'],
  [/(?:;\s*bot\s*;?)/i, 'Other', 'other'],
  [/(?:spider)/i, 'Other', 'other'],
  [/(?:crawler)/i, 'Other', 'other'],
];

// ── Public API ───────────────────────────────────────────────

export function isSpider(userAgent: string): boolean {
  if (!userAgent) return false;
  for (const [pattern] of SPIDER_PATTERNS) {
    if (pattern.test(userAgent)) return true;
  }
  return false;
}

export function getSpiderInfo(userAgent: string): SpiderInfo | null {
  if (!userAgent) return null;
  for (const [pattern, engine, category] of SPIDER_PATTERNS) {
    const match = userAgent.match(pattern);
    if (match) {
      return { engine, name: match[0], category };
    }
  }
  return null;
}

export function isSearchSpider(userAgent: string): boolean {
  const info = getSpiderInfo(userAgent);
  return info?.category === 'search';
}

export function getUserAgent(headers: Record<string, string | string[] | undefined>): string {
  const ua = headers['user-agent'] || headers['User-Agent'];
  if (typeof ua === 'string') return ua;
  if (Array.isArray(ua) && ua.length > 0) return ua[0];
  return '';
}