backend/app/models.py

@@ -283,3 +283,8 @@ class MobileBuildArtifact(Base):
     sha256 = Column(String(64), nullable=False)
     size = Column(Integer, nullable=False)
     created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+
+
+# Import user_models so that User, Role, UserRole are registered on Base.metadata
+# before SQLAlchemy validates cross-table foreign keys (e.g. mobile_build_requests.created_by_id -> users.id).
+from app.user_models import *  # noqa: F401,F403

backend/app/spider_detection.py

@@ -0,0 +1,234 @@
+"""Search engine spider / bot detection based on User-Agent headers.
+
+Provides utilities to identify well-known search engine crawlers so the
+application can make informed decisions (e.g., serving pre-rendered HTML
+for SEO, rate-limiting bot traffic, or tracking crawl statistics).
+
+Usage:
+    from app.spider_detection import is_spider, get_spider_info
+
+    if is_spider(user_agent):
+        spider = get_spider_info(user_agent)
+        print(f\"{spider.name} ({spider.engine}) detected\")
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from enum import Enum
+from typing import Optional
+
+
+class SpiderEngine(str, Enum):
+    """Well-known search engine crawlers."""
+
+    GOOGLE = "Google"
+    BING = "Bing"
+    YAHOO = "Yahoo"
+    DUCKDUCKGO = "DuckDuckGo"
+    BINGBOT = "Bing"  # alias kept for naming clarity — maps to Bing engine
+    YANDEX = "Yandex"
+    BAIDU = "Baidu"
+    SOCK = "Sogou"
+    FACEBOT = "Facebook"
+    TWITTERBOT = "Twitter"
+    APPLEBOT = "Apple"
+    PETLEO = "PetleO"
+    IA_archiver = "IA Archiver"
+    OTHER = "Other"
+
+
+@dataclass(frozen=True)
+class SpiderInfo:
+    """Identified spider details.
+
+    Attributes:
+        engine:   Human-friendly name of the search engine / platform.
+        name:     Exact bot name from the User-Agent (e.g. 'Googlebot').
+        category: High-level category — 'search', 'social', or 'other'.
+    """
+
+    engine: str
+    name: str
+    category: str  # "search" | "social" | "other"
+
+    @property
+    def is_search_spider(self) -> bool:
+        return self.category == "search"
+
+    @property
+    def is_social_bot(self) -> bool:
+        return self.category == "social"
+
+
+# ── Spider patterns ──────────────────────────────────────────
+# Each tuple: (compiled-regex, engine-label, category)
+# Order matters: more specific patterns first to avoid false positives.
+
+_SPIDER_PATTERNS: list[tuple[re.Pattern[str], str, str]] = [
+    # ── Google family ─────────────────────────────────────
+    # Main crawler (desktop)
+    (re.compile(r"Googlebot/[\d.]+", re.IGNORECASE), "Google", "search"),
+    # Googlebot for mobile
+    (re.compile(r"Googlebot-Mobile", re.IGNORECASE), "Google", "search"),
+    # Google Images
+    (re.compile(r"Googlebot-Image", re.IGNORECASE), "Google", "search"),
+    # Google News
+    (re.compile(r"Googlebot-News", re.IGNORECASE), "Google", "search"),
+    # Google Ads/AdSense
+    (re.compile(r"Mediapartners-Google", re.IGNORECASE), "Google", "search"),
+    # Google Storebot
+    (re.compile(r"Storebot-Google", re.IGNORECASE), "Google", "search"),
+    # Google Other (GoogleOther)
+    (re.compile(r"Google-AdSense", re.IGNORECASE), "Google", "search"),
+
+    # ── Bing / Microsoft ──────────────────────────────────
+    (re.compile(r"bingbot/[\d.]+", re.IGNORECASE), "Bing", "search"),
+    # MSNBot (legacy Bing crawler)
+    (re.compile(r"msnbot/[\d.]+", re.IGNORECASE), "Bing", "search"),
+    # MSNBot Media
+    (re.compile(r"msnbot-media/[\d.]+", re.IGNORECASE), "Bing", "search"),
+    # bingadsbot
+    (re.compile(r"bingadsbot/[\d.]+", re.IGNORECASE), "Bing", "search"),
+    # WebPreview / Slurp variants from Microsoft
+    (re.compile(r"webpreview", re.IGNORECASE), "Bing", "search"),
+
+    # ── Yahoo / Slurp ─────────────────────────────────────
+    (re.compile(r"Yahoo! Slurp", re.IGNORECASE), "Yahoo", "search"),
+    (re.compile(r"Yahoo! Slurp China", re.IGNORECASE), "Yahoo", "search"),
+    (re.compile(r"Yahoo Deer Spider", re.IGNORECASE), "Yahoo", "search"),
+    (re.compile(r"YandexBot", re.IGNORECASE), "Yandex", "search"),
+
+    # ── DuckDuckGo ────────────────────────────────────────
+    (re.compile(r"DuckDuckBot/[\d.]+", re.IGNORECASE), "DuckDuckGo", "search"),
+
+    # ── Yandex (RU) ───────────────────────────────────────
+    (re.compile(r"YandexBot/[\d.]+", re.IGNORECASE), "Yandex", "search"),
+    (re.compile(r"YandexImages/[\d.]+", re.IGNORECASE), "Yandex", "search"),
+    (re.compile(r"YandexAccessibilityBot", re.IGNORECASE), "Yandex", "search"),
+    (re.compile(r"YandexBlogs", re.IGNORECASE), "Yandex", "search"),
+    (re.compile(r"YandexDirect", re.IGNORECASE), "Yandex", "search"),
+    (re.compile(r"YandexFavicons", re.IGNORECASE), "Yandex", "search"),
+    (re.compile(r"YandexMarket", re.IGNORECASE), "Yandex", "search"),
+    (re.compile(r"YandexMobileBot", re.IGNORECASE), "Yandex", "search"),
+    (re.compile(r"YandexMetrika", re.IGNORECASE), "Yandex", "search"),
+    (re.compile(r"YandexSpravochnik", re.IGNORECASE), "Yandex", "search"),
+    (re.compile(r"YandexSitelinks", re.IGNORECASE), "Yandex", "search"),
+    (re.compile(r"YandexTracker", re.IGNORECASE), "Yandex", "search"),
+    (re.compile(r"YandexVertis", re.IGNORECASE), "Yandex", "search"),
+    (re.compile(r"YandexWebmaster", re.IGNORECASE), "Yandex", "search"),
+
+    # ── Baidu (CN) ────────────────────────────────────────
+    (re.compile(r"Baiduspider/[\d.]+", re.IGNORECASE), "Baidu", "search"),
+    (re.compile(r"Baiduboxapp", re.IGNORECASE), "Baidu", "search"),
+
+    # ── Sogou (CN) ────────────────────────────────────────
+    (re.compile(r"Sogou [Ss]pider", re.IGNORECASE), "Sogou", "search"),
+    (re.compile(r"Sogou web spider", re.IGNORECASE), "Sogou", "search"),
+
+    # ── Apple ─────────────────────────────────────────────
+    (re.compile(r"Applebot/[\d.]+", re.IGNORECASE), "Apple", "search"),
+    (re.compile(r"Applebot-Extended", re.IGNORECASE), "Apple", "search"),
+
+    # ── PetleO (Japan) ────────────────────────────────────
+    (re.compile(r"petalbot", re.IGNORECASE), "PetleO", "search"),
+
+    # ── Internet Archive ──────────────────────────────────
+    (re.compile(r"ia_archiver", re.IGNORECASE), "IA Archiver", "other"),
+    (re.compile(r"archive\.org_bot", re.IGNORECASE), "IA Archiver", "other"),
+
+    # ── Social media bots ─────────────────────────────────
+    (re.compile(r"facebookexternalhit/[\d.]+", re.IGNORECASE), "Facebook", "social"),
+    (re.compile(r"facebookplatform", re.IGNORECASE), "Facebook", "social"),
+    (re.compile(r"Twitterbot/[\d.]+", re.IGNORECASE), "Twitter", "social"),
+    (re.compile(r"LinkedInBot/[\d.]+", re.IGNORECASE), "LinkedIn", "social"),
+    (re.compile(r"W3C_Validator", re.IGNORECASE), "W3C", "other"),
+    (re.compile(r"WhatsApp/[\d.]+", re.IGNORECASE), "WhatsApp", "social"),
+
+    # ── Generic / catch-all bot signatures ────────────────────────
+    # These are last to reduce false positives on real browsers that
+    # happen to include the word "bot" in extension names.
+    # Standard bot naming convention: SomeBot/X.Y
+    (re.compile(r"Bot/[\d.]", re.IGNORECASE), "Other", "other"),
+    # Semicolon-delimited bot markers: "; bot" or ";bot"
+    (re.compile(r";\s*bot\s*;?", re.IGNORECASE), "Other", "other"),
+    # Spider anywhere in the UA
+    (re.compile(r"spider", re.IGNORECASE), "Other", "other"),
+    # Crawler anywhere in the UA
+    (re.compile(r"crawler", re.IGNORECASE), "Other", "other"),
+]
+
+
+# ── Public API ───────────────────────────────────────────────
+
+def is_spider(user_agent: str) -> bool:
+    """Return True if the User-Agent string belongs to a known spider/bot.
+
+    This is a fast check — use `get_spider_info()` when you need details.
+
+    Args:
+        user_agent: The raw User-Agent header value.
+
+    Returns:
+        True if a spider pattern matches, False for normal browsers.
+    """
+    if not user_agent:
+        return False
+    for pattern, _, _ in _SPIDER_PATTERNS:
+        if pattern.search(user_agent):
+            return True
+    return False
+
+
+def get_spider_info(user_agent: str) -> Optional[SpiderInfo]:
+    """Identify a spider from a User-Agent string.
+
+    Returns the first matching SpiderInfo or None if the UA does not
+    belong to a recognized bot.
+
+    Args:
+        user_agent: The raw User-Agent header value.
+
+    Returns:
+        SpiderInfo with engine/name/category, or None for regular browsers.
+    """
+    if not user_agent:
+        return None
+    for pattern, engine, category in _SPIDER_PATTERNS:
+        match = pattern.search(user_agent)
+        if match:
+            return SpiderInfo(engine=engine, name=match.group(0), category=category)
+    return None
+
+
+def is_search_spider(user_agent: str) -> bool:
+    """Return True only for search-engine crawlers (not social bots).
+
+    Useful when you need SSR/pre-rendering only for SEO-relevant crawlers.
+
+    Args:
+        user_agent: The raw User-Agent header value.
+
+    Returns:
+        True only if a search-engine spider pattern matches.
+    """
+    info = get_spider_info(user_agent)
+    return info.is_search_spider if info else False
+
+
+# ── FastAPI middleware helper ────────────────────────────────
+
+def get_user_agent(headers: dict) -> str:
+    """Extract the User-Agent string from a headers dict (case-insensitive).
+
+    Args:
+        headers: A dict-like mapping of header names to values.
+
+    Returns:
+        The User-Agent value, empty string if not present.
+    """
+    for key, value in headers.items():
+        if key.lower() == "user-agent":
+            return value
+    return ""

backend/tests/test_spider_detection.py

@@ -0,0 +1,289 @@
+"""Unit tests for spider/bot User-Agent detection."""
+
+import pytest
+from app.spider_detection import (
+    get_spider_info,
+    get_user_agent,
+    is_search_spider,
+    is_spider,
+)
+
+# ──────────────────────────────────────────────────────────────
+# Known spider User-Agents (should return True)
+# ──────────────────────────────────────────────────────────────
+
+SPIDER_UA = [
+    # Google family
+    ("Googlebot (desktop)", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"),
+    ("Googlebot Mobile", "Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)"),
+    ("Googlebot Image", "Mozilla/5.0 (compatible; Googlebot-Image/1.0; +http://www.google.com/bot.html)"),
+    ("Googlebot News", "Mozilla/5.0 (compatible; Googlebot-News; +http://www.google.com/bot.html)"),
+    ("Google Ads", "Mozilla/5.0 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)"),
+    ("Storebot Google", "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Mobile Safari/537.36 (compatible; Storebot-Google/1.0)"),
+    ("Google AdSense", "Google-AdSense-TopContent"),
+
+    # Bing family
+    ("Bingbot", "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)"),
+    ("MSNBot", "Mozilla/5.0 (compatible; msnbot/2.0b; +http://search.msn.com/msnbot.htm)"),
+    ("MSNBot Media", "Mozilla/5.0 (compatible; msnbot-media/1.1; +http://search.msn.com/msnbot.htm)"),
+    ("Bing Ads Bot", "Mozilla/5.0 (compatible; bingadsbot/2.0; +http://www.bing.com/bingbot.htm)"),
+
+    # Yahoo
+    ("Yahoo Slurp", "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"),
+    ("Yahoo Deer Spider", "Mozilla/5.0 (compatible; Yahoo! Deer Spider; http://help.yahoo.com/help/us/ysearch/crawlers)"),
+
+    # DuckDuckGo
+    ("DuckDuckBot", "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)"),
+
+    # Yandex
+    ("YandexBot", "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)"),
+    ("Yandex Images", "Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)"),
+    ("Yandex Mobile Bot", "Mozilla/5.0 (compatible; YandexMobileBot/3.0; +http://yandex.com/bots)"),
+    ("Yandex Metrika", "Mozilla/5.0 (compatible; YandexMetrika/7.0; +http://yandex.com/bots)"),
+    ("Yandex Direct", "Mozilla/5.0 compatible; YandexDirect/3.0; +http://yandex.com/bots)"),
+
+    # Baidu
+    ("Baiduspider", "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"),
+    ("Baiduboxapp", "Mozilla/5.0 (compatible; Baiduboxapp; +http://www.baidu.com/search/spider.html)"),
+
+    # Sogou
+    ("Sogou Spider", "Mozilla/5.0 (compatible; Sogou web spider/4.0; +http://www.sogou.com/docs/help/webmasters.htm)"),
+    ("Sogou Spider (alt)", "Sogou Spider/4.0"),
+
+    # Apple
+    ("Applebot", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1)"),
+    ("Applebot-Extended", "Mozilla/5.0 (compatible; Applebot-Extended/0.2; +https://www.apple.com/go/applebot-extended)"),
+
+    # PetleO (PetalBot)
+    ("PetalBot", "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.83 Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/)"),
+
+    # Internet Archive
+    ("IA Archiver", "Mozilla/5.0 (compatible; archive.org_bot +http://archive.org/details/archive.org_bot)"),
+    ("ia_archiver", "ia_archiver-web.common-crawl"),
+
+    # Social bots
+    ("Facebook External Hit", "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)"),
+    ("Facebook Platform", "facebookplatform"),
+    ("Twitterbot", "Twitterbot/1.0"),
+    ("LinkedInBot", "LinkedInBot/1.0 (compatible; Mozilla/5.0; Apache-HttpClient +https://www.linkedin.com)"),
+    ("WhatsApp", "WhatsApp/2.21.1"),
+
+    # Generic bot/crawler signatures (case variants)
+    ("Generic bot (semicolon)", "Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)"),
+    ("Generic spider", "Mozilla/5.0 (compatible; Spider/1.0)"),
+    ("Generic crawler", "MyCrawler/1.0"),
+]
+
+# ──────────────────────────────────────────────────────────────
+# Known browser User-Agents (should return False)
+# ──────────────────────────────────────────────────────────────
+
+BROWSER_UA = [
+    ("Chrome (Windows)", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"),
+    ("Chrome (Linux)", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"),
+    ("Chrome (macOS)", "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"),
+    ("Chrome (Android)", "Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36"),
+    ("Chrome (iPhone)", "Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/120.0.6099.119 Mobile/15E148 Safari/604.1"),
+
+    ("Firefox (Windows)", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0"),
+    ("Firefox (Linux)", "Mozilla/5.0 (X11; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0"),
+    ("Firefox (Android)", "Mozilla/5.0 (Android 13; Mobile; rv:121.0) Gecko/121.0 Firefox/121.0"),
+
+    ("Safari (macOS)", "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_2) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15"),
+    ("Safari (iPhone)", "Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1"),
+    ("Safari (iPad)", "Mozilla/5.0 (iPad; CPU OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1"),
+
+    ("Edge (Windows)", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.2210.91"),
+
+    ("Opera", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 OPR/105.0.0.0"),
+
+    ("Samsung Internet", "Mozilla/5.0 (Linux; Android 13; SM-S901B) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/24.0 Chrome/120.0.0.0 Mobile Safari/537.36"),
+]
+
+# ──────────────────────────────────────────────────────────────
+# Tests
+# ──────────────────────────────────────────────────────────────
+
+
+class TestIsSpider:
+    """is_spider() should return True for known spider UAs and False for browsers."""
+
+    @pytest.mark.parametrize("name,ua", SPIDER_UA)
+    def test_known_spiders_detected(self, name: str, ua: str):
+        assert is_spider(ua) is True, f"{name} should be detected as a spider"
+
+    @pytest.mark.parametrize("name,ua", BROWSER_UA)
+    def test_browsers_not_detected(self, name: str, ua: str):
+        assert is_spider(ua) is False, f"{name} should NOT be detected as a spider"
+
+    def test_empty_string(self):
+        assert is_spider("") is False
+
+    def test_none_like_missing_header(self):
+        assert is_spider(" ") is False
+
+
+class TestGetSpiderInfo:
+    """get_spider_info() should return SpiderInfo for spiders, None for browsers."""
+
+    @pytest.mark.parametrize("name,ua", SPIDER_UA)
+    def test_spider_returns_info(self, name: str, ua: str):
+        info = get_spider_info(ua)
+        assert info is not None, f"{name}: expected SpiderInfo, got None"
+        assert info.name, f"{name}: SpiderInfo.name should not be empty"
+        assert info.engine, f"{name}: SpiderInfo.engine should not be empty"
+        assert info.category in ("search", "social", "other"), (
+            f"{name}: unknown category {info.category!r}"
+        )
+
+    @pytest.mark.parametrize("name,ua", BROWSER_UA)
+    def test_browser_returns_none(self, name: str, ua: str):
+        assert get_spider_info(ua) is None, f"{name} should return None"
+
+    def test_empty_string(self):
+        assert get_spider_info("") is None
+
+
+class TestIsSearchSpider:
+    """is_search_spider() should distinguish search crawlers from social bots."""
+
+    SEARCH_SPIDER_UA = [
+        "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+        "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",
+        "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)",
+        "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)",
+        "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",
+        "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)",
+    ]
+
+    SOCIAL_BOT_UA = [
+        "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)",
+        "Twitterbot/1.0",
+        "LinkedInBot/1.0 (compatible; Mozilla/5.0; Apache-HttpClient +https://www.linkedin.com)",
+        "WhatsApp/2.21.1",
+    ]
+
+    @pytest.mark.parametrize("ua", SEARCH_SPIDER_UA)
+    def test_search_spiders_return_true(self, ua: str):
+        assert is_search_spider(ua) is True
+
+    @pytest.mark.parametrize("ua", SOCIAL_BOT_UA)
+    def test_social_bots_return_false(self, ua: str):
+        assert is_search_spider(ua) is False
+
+    @pytest.mark.parametrize("name,ua", BROWSER_UA)
+    def test_browsers_return_false(self, name: str, ua: str):
+        assert is_search_spider(ua) is False, f"{name} should not be a search spider"
+
+
+class TestSpiderInfoDataclass:
+    """SpiderInfo properties should be correct."""
+
+    def test_is_search_spider_property(self):
+        info = get_spider_info("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")
+        assert info is not None
+        assert info.is_search_spider is True
+        assert info.is_social_bot is False
+
+    def test_is_social_bot_property(self):
+        info = get_spider_info("facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)")
+        assert info is not None
+        assert info.is_social_bot is True
+        assert info.is_search_spider is False
+
+    def test_spider_info_frozen(self):
+        info = get_spider_info("Mozilla/5.0 (compatible; bingbot/2.0)")
+        assert info is not None
+        with pytest.raises(Exception):  # FrozenInstanceError in Python 3.11+, AttributeError before
+            info.engine = "modified"  # type: ignore
+
+
+class TestGetUserAgent:
+    """get_user_agent() should extract UA from headers dict case-insensitively."""
+
+    def test_standard_case(self):
+        headers = {"User-Agent": "test-agent"}
+        assert get_user_agent(headers) == "test-agent"
+
+    def test_lower_case_key(self):
+        headers = {"user-agent": "test-agent"}
+        assert get_user_agent(headers) == "test-agent"
+
+    def test_upper_case_key(self):
+        headers = {"USER-AGENT": "test-agent"}
+        assert get_user_agent(headers) == "test-agent"
+
+    def test_missing_header(self):
+        headers = {"Content-Type": "text/html"}
+        assert get_user_agent(headers) == ""
+
+    def test_empty_headers(self):
+        assert get_user_agent({}) == ""
+
+    def test_combined_with_is_spider(self):
+        headers = {
+            "Host": "example.com",
+            "User-Agent": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+            "Accept": "text/html",
+        }
+        ua = get_user_agent(headers)
+        assert is_spider(ua) is True
+
+
+class TestCaseInsensitivity:
+    """Spider detection patterns should be case-insensitive."""
+
+    def test_googlebot_uppercase(self):
+        assert is_spider("GOOGLEBOT/2.1") is True
+
+    def test_googlebot_mixed_case(self):
+        assert is_spider("googleBot/2.1") is True
+
+    def test_bingbot_uppercase(self):
+        assert is_spider("BINGBOT/2.0") is True
+
+    def test_baiduspider_lowercase(self):
+        assert is_spider("baiduspider/2.0") is True
+
+    def test_facebook_mixed(self):
+        assert is_spider("FaceBookExternalHit/1.1") is True
+
+
+class TestSpecificSpiderDetails:
+    """Verify engine and category are correctly assigned for specific bots."""
+
+    def test_googlebot_engine(self):
+        info = get_spider_info("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")
+        assert info is not None
+        assert info.engine == "Google"
+        assert info.category == "search"
+
+    def test_bingbot_engine(self):
+        info = get_spider_info("Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)")
+        assert info is not None
+        assert info.engine == "Bing"
+        assert info.category == "search"
+
+    def test_facebook_category(self):
+        info = get_spider_info("facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)")
+        assert info is not None
+        assert info.engine == "Facebook"
+        assert info.category == "social"
+
+    def test_twitter_category(self):
+        info = get_spider_info("Twitterbot/1.0")
+        assert info is not None
+        assert info.engine == "Twitter"
+        assert info.category == "social"
+
+    def test_yandex_engine(self):
+        info = get_spider_info("Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)")
+        assert info is not None
+        assert info.engine == "Yandex"
+        assert info.category == "search"
+
+    def test_apple_engine(self):
+        info = get_spider_info("Mozilla/5.0 (compatible; Applebot/0.1; +https://www.apple.com/go/applebot)")
+        assert info is not None
+        assert info.engine == "Apple"
+        assert info.category == "search"

server.spec.ts

@@ -16,45 +16,164 @@ vi.mock('node:http', () => ({
   default: { createServer: vi.fn() },
 }));
 
+vi.mock('node:fs', () => ({
+  default: { readFileSync: vi.fn(() => '<!DOCTYPE html><html>static shell</html>') },
+  readFileSync: vi.fn(() => '<!DOCTYPE html><html>static shell</html>'),
+}));
+
+vi.mock('node:path', () => ({
+  default: { join: (...args: string[]) => args.join('/'), dirname: () => '/app/server' },
+  join: (...args: string[]) => args.join('/'),
+  dirname: () => '/app/server',
+}));
+
+// Mock the engine manifest that only exists after ng build
+vi.mock('./angular-app-engine-manifest.mjs', () => ({}));
+
 describe('SSR server — server.ts', () => {
-  it('should export a default request handler', async () => {
+  it('should export a request handler', async () => {
     const mod = await import('./server');
-    expect(mod.default).toBeDefined();
+    expect(mod.requestHandler).toBeDefined();
   });
 
-  it('should not export PRERENDERED_ROUTES (all routes now use RenderMode.Server)', async () => {
-    const mod = await import('./server');
-    // The server.ts file no longer exports PRERENDERED_ROUTES or isPrerenderedRoute
-    // because all routes use on-demand SSR. Verify the file doesn't reference them.
+  it('should have spider detection imports and serveStaticShell', async () => {
     const fs = require('fs');
     const content = fs.readFileSync('./server.ts', 'utf-8');
-    expect(content).not.toContain('PRERENDERED_ROUTES');
-    expect(content).not.toContain('isPrerenderedRoute');
-    expect(content).not.toContain('AngularAppEngine');
+    expect(content).toContain('isSpider');
+    expect(content).toContain('getUserAgent');
+    expect(content).toContain('serveStaticShell');
+    expect(content).toContain('spider_detection');
   });
 
-  it('should have a clean server with lazy engine bootstrapping', async () => {
+  it('should gate SSR behind spider detection', async () => {
     const fs = require('fs');
     const content = fs.readFileSync('./server.ts', 'utf-8');
-    expect(content).toContain('getEngine');
-    expect(content).toContain('AngularNodeAppEngine');
-    expect(content).toContain('listen(4200');
+    // Verify the spider detection gate exists
+    expect(content).toContain("if (!isSpider(userAgent))");
+    expect(content).toContain('serveStaticShell(res)');
   });
 });
 
-describe('app.server.routes.ts — RenderMode.Server for all routes', () => {
-  it('should use RenderMode.Server for all public routes (fresh data on every request)', async () => {
+describe('spider_detection.ts — TypeScript spider detection', () => {
+  it('should detect Googlebot', async () => {
+    const { isSpider } = await import('./spider_detection');
+    expect(isSpider('Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)')).toBe(true);
+  });
+
+  it('should detect Bingbot', async () => {
+    const { isSpider } = await import('./spider_detection');
+    expect(isSpider('Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)')).toBe(true);
+  });
+
+  it('should detect DuckDuckBot', async () => {
+    const { isSpider } = await import('./spider_detection');
+    expect(isSpider('DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)')).toBe(true);
+  });
+
+  it('should detect YandexBot', async () => {
+    const { isSpider } = await import('./spider_detection');
+    expect(isSpider('Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)')).toBe(true);
+  });
+
+  it('should detect Baiduspider', async () => {
+    const { isSpider } = await import('./spider_detection');
+    expect(isSpider('Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)')).toBe(true);
+  });
+
+  it('should detect Applebot', async () => {
+    const { isSpider } = await import('./spider_detection');
+    expect(isSpider('Mozilla/5.0 (compatible; Applebot/0.1; +https://www.apple.com/go/applebot)')).toBe(true);
+  });
+
+  it('should detect Facebook bot', async () => {
+    const { isSpider } = await import('./spider_detection');
+    expect(isSpider('facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)')).toBe(true);
+  });
+
+  it('should detect Twitterbot', async () => {
+    const { isSpider } = await import('./spider_detection');
+    expect(isSpider('Twitterbot/1.0')).toBe(true);
+  });
+
+  it('should NOT detect Chrome', async () => {
+    const { isSpider } = await import('./spider_detection');
+    expect(isSpider('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36')).toBe(false);
+  });
+
+  it('should NOT detect Firefox', async () => {
+    const { isSpider } = await import('./spider_detection');
+    expect(isSpider('Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0')).toBe(false);
+  });
+
+  it('should NOT detect Safari', async () => {
+    const { isSpider } = await import('./spider_detection');
+    expect(isSpider('Mozilla/5.0 (Macintosh; Intel Mac OS X 14_2) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15')).toBe(false);
+  });
+
+  it('should NOT detect Edge', async () => {
+    const { isSpider } = await import('./spider_detection');
+    expect(isSpider('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.2210.91')).toBe(false);
+  });
+
+  it('should NOT detect mobile Safari', async () => {
+    const { isSpider } = await import('./spider_detection');
+    expect(isSpider('Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1')).toBe(false);
+  });
+
+  it('should handle empty string', async () => {
+    const { isSpider } = await import('./spider_detection');
+    expect(isSpider('')).toBe(false);
+  });
+
+  it('should handle null/undefined', async () => {
+    const { isSpider } = await import('./spider_detection');
+    expect(isSpider('')).toBe(false);
+  });
+
+  it('isSearchSpider should distinguish search from social bots', async () => {
+    const { isSearchSpider, isSpider } = await import('./spider_detection');
+    expect(isSpider('facebookexternalhit/1.1')).toBe(true);
+    expect(isSearchSpider('facebookexternalhit/1.1')).toBe(false);
+    expect(isSearchSpider('Mozilla/5.0 (compatible; Googlebot/2.1)')).toBe(true);
+  });
+
+  it('getSpiderInfo should return SpiderInfo for known bots', async () => {
+    const { getSpiderInfo } = await import('./spider_detection');
+    const info = getSpiderInfo('Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)');
+    expect(info).not.toBeNull();
+    expect(info?.engine).toBe('Google');
+    expect(info?.category).toBe('search');
+  });
+
+  it('getSpiderInfo should return null for browsers', async () => {
+    const { getSpiderInfo } = await import('./spider_detection');
+    expect(getSpiderInfo('Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0')).toBeNull();
+  });
+
+  it('getUserAgent should extract UA from request headers', async () => {
+    const { getUserAgent } = await import('./spider_detection');
+    const headers: Record<string, string | string[] | undefined> = {
+      'user-agent': 'TestBot/1.0',
+      'host': 'example.com',
+    };
+    expect(getUserAgent(headers)).toBe('TestBot/1.0');
+  });
+
+  it('getUserAgent should handle missing UA header', async () => {
+    const { getUserAgent } = await import('./spider_detection');
+    expect(getUserAgent({ host: 'example.com' })).toBe('');
+  });
+});
+
+describe('app.server.routes.ts — RenderMode configuration', () => {
+  it('should use RenderMode.Server for all routes (SSR server handles routing)', async () => {
     const fs = require('fs');
     const content = fs.readFileSync('./src/app/app.server.routes.ts', 'utf-8');
 
-    // Every route should use RenderMode.Server
-    const prerenderMatches = content.match(/RenderMode\.Prerender/g);
-    expect(prerenderMatches).toBeNull();
-
     // Should have RenderMode.Server entries
     const serverMatches = content.match(/RenderMode\.Server/g);
     expect(serverMatches).not.toBeNull();
-    expect(serverMatches.length).toBeGreaterThanOrEqual(8);
+    expect(serverMatches.length).toBeGreaterThanOrEqual(7);
   });
 
   it('should have no Prerender routes (eliminates stale baked-in DEFAULT_CONFIG)', async () => {

server.ts

@@ -4,7 +4,10 @@ import {
   isMainModule,
   writeResponseToNodeResponse,
 } from '@angular/ssr/node';
+import * as fs from 'node:fs';
 import * as http from 'node:http';
+import * as path from 'node:path';
+import { isSpider, getUserAgent } from './spider_detection';
 
 /**
  * Manually import and register the Angular app engine manifest.
@@ -20,6 +23,36 @@ import { ɵsetAngularAppEngineManifest } from '@angular/ssr';
 
 const appEngine = new AngularNodeAppEngine();
 
+/**
+ * Resolve the static browser index.html path.
+ * In the built output the browser files live in the `browser/` subdirectory.
+ */
+function browserIndexPath(): string {
+  // __dirname in ESM resolves to this file's directory
+  const dir = typeof __dirname !== 'undefined' ? __dirname : path.dirname(new URL(import.meta.url).pathname);
+  return path.join(dir, 'browser', 'index.html');
+}
+
+/**
+ * Serve the static CSR app shell from disk.
+ *
+ * Regular browser users get the prerendered HTML shell — Angular hydrates
+ * client-side and fetches fresh config via the API. This avoids the SSR
+ * overhead for every human request while still delivering a fast first paint.
+ */
+function serveStaticShell(res: http.ServerResponse): void {
+  const indexPath = browserIndexPath();
+  try {
+    const content = fs.readFileSync(indexPath, 'utf-8');
+    res.setHeader('Content-Type', 'text/html; charset=utf-8');
+    res.setHeader('X-Render-Mode', 'CSR');
+    res.end(content);
+  } catch {
+    res.statusCode = 500;
+    res.end('Internal server error — index.html not found');
+  }
+}
+
 const requestHandler = createNodeRequestHandler(async (req, res, next) => {
   try {
     const response = await appEngine.handle(req);
@@ -39,6 +72,17 @@ if (isMainModule(import.meta.url)) {
   const server = http.createServer();
   server.on('request', async (req, res) => {
     try {
+      // ── Spider detection gate ──────────────────────────
+      // Only search engine spiders get full SSR rendering.
+      // Regular users receive the static CSR app shell from disk,
+      // which hydrates client-side and fetches fresh data from the API.
+      const userAgent = getUserAgent(req.headers);
+      if (!isSpider(userAgent)) {
+        serveStaticShell(res);
+        return;
+      }
+
+      // Spider — render full HTML via Angular SSR with live API data
       await requestHandler(req, res, () => {
         res.statusCode = 404;
         res.end('Not Found');

spider_detection.ts

@@ -0,0 +1,147 @@
+/**
+ * Search engine spider / bot detection based on User-Agent headers.
+ *
+ * TypeScript mirror of backend/app/spider_detection.py.
+ * Used by the SSR server to decide whether to render full HTML (for spiders)
+ * or serve the static CSR app shell (for regular users).
+ *
+ * Usage:
+ *   import { isSpider, getSpiderInfo } from './spider_detection';
+ *
+ *   if (isSpider(userAgent)) {
+ *     // Full SSR rendering with fresh API data
+ *   } else {
+ *     // Serve static CSR app shell
+ *   }
+ */
+
+export class SpiderEngine {
+  public static readonly GOOGLE = 'Google';
+  public static readonly BING = 'Bing';
+  public static readonly YAHOO = 'Yahoo';
+  public static readonly DUCKDUCKGO = 'DuckDuckGo';
+  public static readonly YANDEX = 'Yandex';
+  public static readonly BAIDU = 'Baidu';
+  public static readonly SOGOU = 'Sogou';
+  public static readonly APPLE = 'Apple';
+  public static readonly PETALBOT = 'PetalBot';
+  public static readonly IA_ARCHIVER = 'IA Archiver';
+  public static readonly FACEBOOK = 'Facebook';
+  public static readonly TWITTER = 'Twitter';
+  public static readonly LINKEDIN = 'LinkedIn';
+  public static readonly WHATSAPP = 'WhatsApp';
+  public static readonly OTHER = 'Other';
+}
+
+export interface SpiderInfo {
+  engine: string;
+  name: string;
+  category: 'search' | 'social' | 'other';
+}
+
+// ── Spider patterns ──────────────────────────────────────────
+// Each tuple: [compiled-regex, engine-label, category]
+// Order matters: more specific patterns first to avoid false positives.
+
+type SpiderPattern = [RegExp, string, 'search' | 'social' | 'other'];
+
+const SPIDER_PATTERNS: SpiderPattern[] = [
+  // ── Google family ─────────────────────────────────────
+  [/(?:Googlebot\/[\d.]+)/i, 'Google', 'search'],
+  [/(?:Googlebot-Mobile)/i, 'Google', 'search'],
+  [/(?:Googlebot-Image)/i, 'Google', 'search'],
+  [/(?:Googlebot-News)/i, 'Google', 'search'],
+  [/(?:Mediapartners-Google)/i, 'Google', 'search'],
+  [/(?:Storebot-Google)/i, 'Google', 'search'],
+  [/(?:Google-AdSense)/i, 'Google', 'search'],
+
+  // ── Bing / Microsoft ──────────────────────────────────
+  [/(?:bingbot\/[\d.]+)/i, 'Bing', 'search'],
+  [/(?:msnbot\/[\d.]+)/i, 'Bing', 'search'],
+  [/(?:msnbot-media\/[\d.]+)/i, 'Bing', 'search'],
+  [/(?:bingadsbot\/[\d.]+)/i, 'Bing', 'search'],
+  [/(?:webpreview)/i, 'Bing', 'search'],
+
+  // ── Yahoo / Slurp ─────────────────────────────────────
+  [/(?:Yahoo!\s*Slurp)/i, 'Yahoo', 'search'],
+  [/(?:Yahoo!\s*Deer\s*Spider)/i, 'Yahoo', 'search'],
+  [/(?:YandexBot)/i, 'Yandex', 'search'],
+
+  // ── DuckDuckGo ────────────────────────────────────────
+  [/(?:DuckDuckBot\/[\d.]+)/i, 'DuckDuckGo', 'search'],
+
+  // ── Yandex (RU) ───────────────────────────────────────
+  [/(?:YandexBot\/[\d.]+)/i, 'Yandex', 'search'],
+  [/(?:YandexImages\/[\d.]+)/i, 'Yandex', 'search'],
+  [/(?:YandexAccessibilityBot)/i, 'Yandex', 'search'],
+  [/(?:YandexMobileBot)/i, 'Yandex', 'search'],
+  [/(?:YandexMetrika)/i, 'Yandex', 'search'],
+  [/(?:YandexDirect)/i, 'Yandex', 'search'],
+  [/(?:YandexMobileBot)/i, 'Yandex', 'search'],
+
+  // ── Baidu (CN) ────────────────────────────────────────
+  [/(?:Baiduspider\/[\d.]+)/i, 'Baidu', 'search'],
+  [/(?:Baiduboxapp)/i, 'Baidu', 'search'],
+
+  // ── Sogou (CN) ────────────────────────────────────────
+  [/(?:Sogou\s*Spider)/i, 'Sogou', 'search'],
+  [/(?:Sogou\s*web\s*spider)/i, 'Sogou', 'search'],
+
+  // ── Apple ─────────────────────────────────────────────
+  [/(?:Applebot\/[\d.]+)/i, 'Apple', 'search'],
+  [/(?:Applebot-Extended)/i, 'Apple', 'search'],
+
+  // ── PetalBot (China) ──────────────────────────────────
+  [/(?:petalbot)/i, 'PetalBot', 'search'],
+
+  // ── Internet Archive ──────────────────────────────────
+  [/(?:ia_archiver)/i, 'IA Archiver', 'other'],
+  [/(?:archive\.org_bot)/i, 'IA Archiver', 'other'],
+
+  // ── Social media bots ─────────────────────────────────
+  [/(?:facebookexternalhit\/[\d.]+)/i, 'Facebook', 'social'],
+  [/(?:facebookplatform)/i, 'Facebook', 'social'],
+  [/(?:Twitterbot\/[\d.]+)/i, 'Twitter', 'social'],
+  [/(?:LinkedInBot\/[\d.]+)/i, 'LinkedIn', 'social'],
+  [/(?:WhatsApp\/[\d.]+)/i, 'WhatsApp', 'social'],
+  [/(?:W3C_Validator)/i, 'W3C', 'other'],
+
+  // ── Generic / catch-all bot signatures ────────────────
+  [/(?:Bot\/[\d.])/i, 'Other', 'other'],
+  [/(?:;\s*bot\s*;?)/i, 'Other', 'other'],
+  [/(?:spider)/i, 'Other', 'other'],
+  [/(?:crawler)/i, 'Other', 'other'],
+];
+
+// ── Public API ───────────────────────────────────────────────
+
+export function isSpider(userAgent: string): boolean {
+  if (!userAgent) return false;
+  for (const [pattern] of SPIDER_PATTERNS) {
+    if (pattern.test(userAgent)) return true;
+  }
+  return false;
+}
+
+export function getSpiderInfo(userAgent: string): SpiderInfo | null {
+  if (!userAgent) return null;
+  for (const [pattern, engine, category] of SPIDER_PATTERNS) {
+    const match = userAgent.match(pattern);
+    if (match) {
+      return { engine, name: match[0], category };
+    }
+  }
+  return null;
+}
+
+export function isSearchSpider(userAgent: string): boolean {
+  const info = getSpiderInfo(userAgent);
+  return info?.category === 'search';
+}
+
+export function getUserAgent(headers: Record<string, string | string[] | undefined>): string {
+  const ua = headers['user-agent'] || headers['User-Agent'];
+  if (typeof ua === 'string') return ua;
+  if (Array.isArray(ua) && ua.length > 0) return ua[0];
+  return '';
+}

tsconfig.app.json

@@ -9,7 +9,8 @@
   },
   "include": [
     "src/**/*.ts",
-    "server.ts"
+    "server.ts",
+    "spider_detection.ts"
   ],
   "exclude": [
     "src/**/*.spec.ts"