feat(ssr): spider-only SSR — serve CSR shell for regular users

Add spider detection gate to SSR server so that search engine
crawlers get full server-side rendering with fresh API data,
while regular users receive the static CSR app shell from disk.

- backend/app/spider_detection.py: Python spider detection module
- spider_detection.ts: TypeScript mirror of spider detection patterns
- server.ts: SSR gate — isSpider() check before dispatching to
  Angular SSR engine; non-spiders get static index.html
- server.spec.ts: Added spider detection unit tests
- tsconfig.app.json: Include spider_detection.ts in compilation
- backend/app/models.py: Fix SQLAlchemy FK resolution (user_models import)

backend/app/spider_detection.py

@@ -0,0 +1,234 @@
+"""Search engine spider / bot detection based on User-Agent headers.
+
+Provides utilities to identify well-known search engine crawlers so the
+application can make informed decisions (e.g., serving pre-rendered HTML
+for SEO, rate-limiting bot traffic, or tracking crawl statistics).
+
+Usage:
+    from app.spider_detection import is_spider, get_spider_info
+
+    if is_spider(user_agent):
+        spider = get_spider_info(user_agent)
+        print(f\"{spider.name} ({spider.engine}) detected\")
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from enum import Enum
+from typing import Optional
+
+
+class SpiderEngine(str, Enum):
+    """Well-known search engine crawlers."""
+
+    GOOGLE = "Google"
+    BING = "Bing"
+    YAHOO = "Yahoo"
+    DUCKDUCKGO = "DuckDuckGo"
+    BINGBOT = "Bing"  # alias kept for naming clarity — maps to Bing engine
+    YANDEX = "Yandex"
+    BAIDU = "Baidu"
+    SOCK = "Sogou"
+    FACEBOT = "Facebook"
+    TWITTERBOT = "Twitter"
+    APPLEBOT = "Apple"
+    PETLEO = "PetleO"
+    IA_archiver = "IA Archiver"
+    OTHER = "Other"
+
+
+@dataclass(frozen=True)
+class SpiderInfo:
+    """Identified spider details.
+
+    Attributes:
+        engine:   Human-friendly name of the search engine / platform.
+        name:     Exact bot name from the User-Agent (e.g. 'Googlebot').
+        category: High-level category — 'search', 'social', or 'other'.
+    """
+
+    engine: str
+    name: str
+    category: str  # "search" | "social" | "other"
+
+    @property
+    def is_search_spider(self) -> bool:
+        return self.category == "search"
+
+    @property
+    def is_social_bot(self) -> bool:
+        return self.category == "social"
+
+
+# ── Spider patterns ──────────────────────────────────────────
+# Each tuple: (compiled-regex, engine-label, category)
+# Order matters: more specific patterns first to avoid false positives.
+
+_SPIDER_PATTERNS: list[tuple[re.Pattern[str], str, str]] = [
+    # ── Google family ─────────────────────────────────────
+    # Main crawler (desktop)
+    (re.compile(r"Googlebot/[\d.]+", re.IGNORECASE), "Google", "search"),
+    # Googlebot for mobile
+    (re.compile(r"Googlebot-Mobile", re.IGNORECASE), "Google", "search"),
+    # Google Images
+    (re.compile(r"Googlebot-Image", re.IGNORECASE), "Google", "search"),
+    # Google News
+    (re.compile(r"Googlebot-News", re.IGNORECASE), "Google", "search"),
+    # Google Ads/AdSense
+    (re.compile(r"Mediapartners-Google", re.IGNORECASE), "Google", "search"),
+    # Google Storebot
+    (re.compile(r"Storebot-Google", re.IGNORECASE), "Google", "search"),
+    # Google Other (GoogleOther)
+    (re.compile(r"Google-AdSense", re.IGNORECASE), "Google", "search"),
+
+    # ── Bing / Microsoft ──────────────────────────────────
+    (re.compile(r"bingbot/[\d.]+", re.IGNORECASE), "Bing", "search"),
+    # MSNBot (legacy Bing crawler)
+    (re.compile(r"msnbot/[\d.]+", re.IGNORECASE), "Bing", "search"),
+    # MSNBot Media
+    (re.compile(r"msnbot-media/[\d.]+", re.IGNORECASE), "Bing", "search"),
+    # bingadsbot
+    (re.compile(r"bingadsbot/[\d.]+", re.IGNORECASE), "Bing", "search"),
+    # WebPreview / Slurp variants from Microsoft
+    (re.compile(r"webpreview", re.IGNORECASE), "Bing", "search"),
+
+    # ── Yahoo / Slurp ─────────────────────────────────────
+    (re.compile(r"Yahoo! Slurp", re.IGNORECASE), "Yahoo", "search"),
+    (re.compile(r"Yahoo! Slurp China", re.IGNORECASE), "Yahoo", "search"),
+    (re.compile(r"Yahoo Deer Spider", re.IGNORECASE), "Yahoo", "search"),
+    (re.compile(r"YandexBot", re.IGNORECASE), "Yandex", "search"),
+
+    # ── DuckDuckGo ────────────────────────────────────────
+    (re.compile(r"DuckDuckBot/[\d.]+", re.IGNORECASE), "DuckDuckGo", "search"),
+
+    # ── Yandex (RU) ───────────────────────────────────────
+    (re.compile(r"YandexBot/[\d.]+", re.IGNORECASE), "Yandex", "search"),
+    (re.compile(r"YandexImages/[\d.]+", re.IGNORECASE), "Yandex", "search"),
+    (re.compile(r"YandexAccessibilityBot", re.IGNORECASE), "Yandex", "search"),
+    (re.compile(r"YandexBlogs", re.IGNORECASE), "Yandex", "search"),
+    (re.compile(r"YandexDirect", re.IGNORECASE), "Yandex", "search"),
+    (re.compile(r"YandexFavicons", re.IGNORECASE), "Yandex", "search"),
+    (re.compile(r"YandexMarket", re.IGNORECASE), "Yandex", "search"),
+    (re.compile(r"YandexMobileBot", re.IGNORECASE), "Yandex", "search"),
+    (re.compile(r"YandexMetrika", re.IGNORECASE), "Yandex", "search"),
+    (re.compile(r"YandexSpravochnik", re.IGNORECASE), "Yandex", "search"),
+    (re.compile(r"YandexSitelinks", re.IGNORECASE), "Yandex", "search"),
+    (re.compile(r"YandexTracker", re.IGNORECASE), "Yandex", "search"),
+    (re.compile(r"YandexVertis", re.IGNORECASE), "Yandex", "search"),
+    (re.compile(r"YandexWebmaster", re.IGNORECASE), "Yandex", "search"),
+
+    # ── Baidu (CN) ────────────────────────────────────────
+    (re.compile(r"Baiduspider/[\d.]+", re.IGNORECASE), "Baidu", "search"),
+    (re.compile(r"Baiduboxapp", re.IGNORECASE), "Baidu", "search"),
+
+    # ── Sogou (CN) ────────────────────────────────────────
+    (re.compile(r"Sogou [Ss]pider", re.IGNORECASE), "Sogou", "search"),
+    (re.compile(r"Sogou web spider", re.IGNORECASE), "Sogou", "search"),
+
+    # ── Apple ─────────────────────────────────────────────
+    (re.compile(r"Applebot/[\d.]+", re.IGNORECASE), "Apple", "search"),
+    (re.compile(r"Applebot-Extended", re.IGNORECASE), "Apple", "search"),
+
+    # ── PetleO (Japan) ────────────────────────────────────
+    (re.compile(r"petalbot", re.IGNORECASE), "PetleO", "search"),
+
+    # ── Internet Archive ──────────────────────────────────
+    (re.compile(r"ia_archiver", re.IGNORECASE), "IA Archiver", "other"),
+    (re.compile(r"archive\.org_bot", re.IGNORECASE), "IA Archiver", "other"),
+
+    # ── Social media bots ─────────────────────────────────
+    (re.compile(r"facebookexternalhit/[\d.]+", re.IGNORECASE), "Facebook", "social"),
+    (re.compile(r"facebookplatform", re.IGNORECASE), "Facebook", "social"),
+    (re.compile(r"Twitterbot/[\d.]+", re.IGNORECASE), "Twitter", "social"),
+    (re.compile(r"LinkedInBot/[\d.]+", re.IGNORECASE), "LinkedIn", "social"),
+    (re.compile(r"W3C_Validator", re.IGNORECASE), "W3C", "other"),
+    (re.compile(r"WhatsApp/[\d.]+", re.IGNORECASE), "WhatsApp", "social"),
+
+    # ── Generic / catch-all bot signatures ────────────────────────
+    # These are last to reduce false positives on real browsers that
+    # happen to include the word "bot" in extension names.
+    # Standard bot naming convention: SomeBot/X.Y
+    (re.compile(r"Bot/[\d.]", re.IGNORECASE), "Other", "other"),
+    # Semicolon-delimited bot markers: "; bot" or ";bot"
+    (re.compile(r";\s*bot\s*;?", re.IGNORECASE), "Other", "other"),
+    # Spider anywhere in the UA
+    (re.compile(r"spider", re.IGNORECASE), "Other", "other"),
+    # Crawler anywhere in the UA
+    (re.compile(r"crawler", re.IGNORECASE), "Other", "other"),
+]
+
+
+# ── Public API ───────────────────────────────────────────────
+
+def is_spider(user_agent: str) -> bool:
+    """Return True if the User-Agent string belongs to a known spider/bot.
+
+    This is a fast check — use `get_spider_info()` when you need details.
+
+    Args:
+        user_agent: The raw User-Agent header value.
+
+    Returns:
+        True if a spider pattern matches, False for normal browsers.
+    """
+    if not user_agent:
+        return False
+    for pattern, _, _ in _SPIDER_PATTERNS:
+        if pattern.search(user_agent):
+            return True
+    return False
+
+
+def get_spider_info(user_agent: str) -> Optional[SpiderInfo]:
+    """Identify a spider from a User-Agent string.
+
+    Returns the first matching SpiderInfo or None if the UA does not
+    belong to a recognized bot.
+
+    Args:
+        user_agent: The raw User-Agent header value.
+
+    Returns:
+        SpiderInfo with engine/name/category, or None for regular browsers.
+    """
+    if not user_agent:
+        return None
+    for pattern, engine, category in _SPIDER_PATTERNS:
+        match = pattern.search(user_agent)
+        if match:
+            return SpiderInfo(engine=engine, name=match.group(0), category=category)
+    return None
+
+
+def is_search_spider(user_agent: str) -> bool:
+    """Return True only for search-engine crawlers (not social bots).
+
+    Useful when you need SSR/pre-rendering only for SEO-relevant crawlers.
+
+    Args:
+        user_agent: The raw User-Agent header value.
+
+    Returns:
+        True only if a search-engine spider pattern matches.
+    """
+    info = get_spider_info(user_agent)
+    return info.is_search_spider if info else False
+
+
+# ── FastAPI middleware helper ────────────────────────────────
+
+def get_user_agent(headers: dict) -> str:
+    """Extract the User-Agent string from a headers dict (case-insensitive).
+
+    Args:
+        headers: A dict-like mapping of header names to values.
+
+    Returns:
+        The User-Agent value, empty string if not present.
+    """
+    for key, value in headers.items():
+        if key.lower() == "user-agent":
+            return value
+    return ""