diff --git a/angular-app-engine-manifest.mjs b/angular-app-engine-manifest.mjs new file mode 100644 index 0000000..ff8b4c5 --- /dev/null +++ b/angular-app-engine-manifest.mjs @@ -0,0 +1 @@ +export default {}; diff --git a/backend/app/models.py b/backend/app/models.py index 8b26604..edc4711 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -283,3 +283,8 @@ class MobileBuildArtifact(Base): sha256 = Column(String(64), nullable=False) size = Column(Integer, nullable=False) created_at = Column(DateTime, nullable=False, default=datetime.utcnow) + + +# Import user_models so that User, Role, UserRole are registered on Base.metadata +# before SQLAlchemy validates cross-table foreign keys (e.g. mobile_build_requests.created_by_id -> users.id). +from app.user_models import * # noqa: F401,F403 \ No newline at end of file diff --git a/backend/app/spider_detection.py b/backend/app/spider_detection.py new file mode 100644 index 0000000..03c6f95 --- /dev/null +++ b/backend/app/spider_detection.py @@ -0,0 +1,234 @@ +"""Search engine spider / bot detection based on User-Agent headers. + +Provides utilities to identify well-known search engine crawlers so the +application can make informed decisions (e.g., serving pre-rendered HTML +for SEO, rate-limiting bot traffic, or tracking crawl statistics). + +Usage: + from app.spider_detection import is_spider, get_spider_info + + if is_spider(user_agent): + spider = get_spider_info(user_agent) + print(f\"{spider.name} ({spider.engine}) detected\") +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from enum import Enum +from typing import Optional + + +class SpiderEngine(str, Enum): + """Well-known search engine crawlers.""" + + GOOGLE = "Google" + BING = "Bing" + YAHOO = "Yahoo" + DUCKDUCKGO = "DuckDuckGo" + BINGBOT = "Bing" # alias kept for naming clarity — maps to Bing engine + YANDEX = "Yandex" + BAIDU = "Baidu" + SOCK = "Sogou" + FACEBOT = "Facebook" + TWITTERBOT = "Twitter" + APPLEBOT = "Apple" + PETLEO = "PetleO" + IA_archiver = "IA Archiver" + OTHER = "Other" + + +@dataclass(frozen=True) +class SpiderInfo: + """Identified spider details. + + Attributes: + engine: Human-friendly name of the search engine / platform. + name: Exact bot name from the User-Agent (e.g. 'Googlebot'). + category: High-level category — 'search', 'social', or 'other'. + """ + + engine: str + name: str + category: str # "search" | "social" | "other" + + @property + def is_search_spider(self) -> bool: + return self.category == "search" + + @property + def is_social_bot(self) -> bool: + return self.category == "social" + + +# ── Spider patterns ────────────────────────────────────────── +# Each tuple: (compiled-regex, engine-label, category) +# Order matters: more specific patterns first to avoid false positives. + +_SPIDER_PATTERNS: list[tuple[re.Pattern[str], str, str]] = [ + # ── Google family ───────────────────────────────────── + # Main crawler (desktop) + (re.compile(r"Googlebot/[\d.]+", re.IGNORECASE), "Google", "search"), + # Googlebot for mobile + (re.compile(r"Googlebot-Mobile", re.IGNORECASE), "Google", "search"), + # Google Images + (re.compile(r"Googlebot-Image", re.IGNORECASE), "Google", "search"), + # Google News + (re.compile(r"Googlebot-News", re.IGNORECASE), "Google", "search"), + # Google Ads/AdSense + (re.compile(r"Mediapartners-Google", re.IGNORECASE), "Google", "search"), + # Google Storebot + (re.compile(r"Storebot-Google", re.IGNORECASE), "Google", "search"), + # Google Other (GoogleOther) + (re.compile(r"Google-AdSense", re.IGNORECASE), "Google", "search"), + + # ── Bing / Microsoft ────────────────────────────────── + (re.compile(r"bingbot/[\d.]+", re.IGNORECASE), "Bing", "search"), + # MSNBot (legacy Bing crawler) + (re.compile(r"msnbot/[\d.]+", re.IGNORECASE), "Bing", "search"), + # MSNBot Media + (re.compile(r"msnbot-media/[\d.]+", re.IGNORECASE), "Bing", "search"), + # bingadsbot + (re.compile(r"bingadsbot/[\d.]+", re.IGNORECASE), "Bing", "search"), + # WebPreview / Slurp variants from Microsoft + (re.compile(r"webpreview", re.IGNORECASE), "Bing", "search"), + + # ── Yahoo / Slurp ───────────────────────────────────── + (re.compile(r"Yahoo! Slurp", re.IGNORECASE), "Yahoo", "search"), + (re.compile(r"Yahoo! Slurp China", re.IGNORECASE), "Yahoo", "search"), + (re.compile(r"Yahoo Deer Spider", re.IGNORECASE), "Yahoo", "search"), + (re.compile(r"YandexBot", re.IGNORECASE), "Yandex", "search"), + + # ── DuckDuckGo ──────────────────────────────────────── + (re.compile(r"DuckDuckBot/[\d.]+", re.IGNORECASE), "DuckDuckGo", "search"), + + # ── Yandex (RU) ─────────────────────────────────────── + (re.compile(r"YandexBot/[\d.]+", re.IGNORECASE), "Yandex", "search"), + (re.compile(r"YandexImages/[\d.]+", re.IGNORECASE), "Yandex", "search"), + (re.compile(r"YandexAccessibilityBot", re.IGNORECASE), "Yandex", "search"), + (re.compile(r"YandexBlogs", re.IGNORECASE), "Yandex", "search"), + (re.compile(r"YandexDirect", re.IGNORECASE), "Yandex", "search"), + (re.compile(r"YandexFavicons", re.IGNORECASE), "Yandex", "search"), + (re.compile(r"YandexMarket", re.IGNORECASE), "Yandex", "search"), + (re.compile(r"YandexMobileBot", re.IGNORECASE), "Yandex", "search"), + (re.compile(r"YandexMetrika", re.IGNORECASE), "Yandex", "search"), + (re.compile(r"YandexSpravochnik", re.IGNORECASE), "Yandex", "search"), + (re.compile(r"YandexSitelinks", re.IGNORECASE), "Yandex", "search"), + (re.compile(r"YandexTracker", re.IGNORECASE), "Yandex", "search"), + (re.compile(r"YandexVertis", re.IGNORECASE), "Yandex", "search"), + (re.compile(r"YandexWebmaster", re.IGNORECASE), "Yandex", "search"), + + # ── Baidu (CN) ──────────────────────────────────────── + (re.compile(r"Baiduspider/[\d.]+", re.IGNORECASE), "Baidu", "search"), + (re.compile(r"Baiduboxapp", re.IGNORECASE), "Baidu", "search"), + + # ── Sogou (CN) ──────────────────────────────────────── + (re.compile(r"Sogou [Ss]pider", re.IGNORECASE), "Sogou", "search"), + (re.compile(r"Sogou web spider", re.IGNORECASE), "Sogou", "search"), + + # ── Apple ───────────────────────────────────────────── + (re.compile(r"Applebot/[\d.]+", re.IGNORECASE), "Apple", "search"), + (re.compile(r"Applebot-Extended", re.IGNORECASE), "Apple", "search"), + + # ── PetleO (Japan) ──────────────────────────────────── + (re.compile(r"petalbot", re.IGNORECASE), "PetleO", "search"), + + # ── Internet Archive ────────────────────────────────── + (re.compile(r"ia_archiver", re.IGNORECASE), "IA Archiver", "other"), + (re.compile(r"archive\.org_bot", re.IGNORECASE), "IA Archiver", "other"), + + # ── Social media bots ───────────────────────────────── + (re.compile(r"facebookexternalhit/[\d.]+", re.IGNORECASE), "Facebook", "social"), + (re.compile(r"facebookplatform", re.IGNORECASE), "Facebook", "social"), + (re.compile(r"Twitterbot/[\d.]+", re.IGNORECASE), "Twitter", "social"), + (re.compile(r"LinkedInBot/[\d.]+", re.IGNORECASE), "LinkedIn", "social"), + (re.compile(r"W3C_Validator", re.IGNORECASE), "W3C", "other"), + (re.compile(r"WhatsApp/[\d.]+", re.IGNORECASE), "WhatsApp", "social"), + + # ── Generic / catch-all bot signatures ──────────────────────── + # These are last to reduce false positives on real browsers that + # happen to include the word "bot" in extension names. + # Standard bot naming convention: SomeBot/X.Y + (re.compile(r"Bot/[\d.]", re.IGNORECASE), "Other", "other"), + # Semicolon-delimited bot markers: "; bot" or ";bot" + (re.compile(r";\s*bot\s*;?", re.IGNORECASE), "Other", "other"), + # Spider anywhere in the UA + (re.compile(r"spider", re.IGNORECASE), "Other", "other"), + # Crawler anywhere in the UA + (re.compile(r"crawler", re.IGNORECASE), "Other", "other"), +] + + +# ── Public API ─────────────────────────────────────────────── + +def is_spider(user_agent: str) -> bool: + """Return True if the User-Agent string belongs to a known spider/bot. + + This is a fast check — use `get_spider_info()` when you need details. + + Args: + user_agent: The raw User-Agent header value. + + Returns: + True if a spider pattern matches, False for normal browsers. + """ + if not user_agent: + return False + for pattern, _, _ in _SPIDER_PATTERNS: + if pattern.search(user_agent): + return True + return False + + +def get_spider_info(user_agent: str) -> Optional[SpiderInfo]: + """Identify a spider from a User-Agent string. + + Returns the first matching SpiderInfo or None if the UA does not + belong to a recognized bot. + + Args: + user_agent: The raw User-Agent header value. + + Returns: + SpiderInfo with engine/name/category, or None for regular browsers. + """ + if not user_agent: + return None + for pattern, engine, category in _SPIDER_PATTERNS: + match = pattern.search(user_agent) + if match: + return SpiderInfo(engine=engine, name=match.group(0), category=category) + return None + + +def is_search_spider(user_agent: str) -> bool: + """Return True only for search-engine crawlers (not social bots). + + Useful when you need SSR/pre-rendering only for SEO-relevant crawlers. + + Args: + user_agent: The raw User-Agent header value. + + Returns: + True only if a search-engine spider pattern matches. + """ + info = get_spider_info(user_agent) + return info.is_search_spider if info else False + + +# ── FastAPI middleware helper ──────────────────────────────── + +def get_user_agent(headers: dict) -> str: + """Extract the User-Agent string from a headers dict (case-insensitive). + + Args: + headers: A dict-like mapping of header names to values. + + Returns: + The User-Agent value, empty string if not present. + """ + for key, value in headers.items(): + if key.lower() == "user-agent": + return value + return "" diff --git a/backend/tests/test_spider_detection.py b/backend/tests/test_spider_detection.py new file mode 100644 index 0000000..0233eff --- /dev/null +++ b/backend/tests/test_spider_detection.py @@ -0,0 +1,289 @@ +"""Unit tests for spider/bot User-Agent detection.""" + +import pytest +from app.spider_detection import ( + get_spider_info, + get_user_agent, + is_search_spider, + is_spider, +) + +# ────────────────────────────────────────────────────────────── +# Known spider User-Agents (should return True) +# ────────────────────────────────────────────────────────────── + +SPIDER_UA = [ + # Google family + ("Googlebot (desktop)", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"), + ("Googlebot Mobile", "Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)"), + ("Googlebot Image", "Mozilla/5.0 (compatible; Googlebot-Image/1.0; +http://www.google.com/bot.html)"), + ("Googlebot News", "Mozilla/5.0 (compatible; Googlebot-News; +http://www.google.com/bot.html)"), + ("Google Ads", "Mozilla/5.0 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)"), + ("Storebot Google", "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Mobile Safari/537.36 (compatible; Storebot-Google/1.0)"), + ("Google AdSense", "Google-AdSense-TopContent"), + + # Bing family + ("Bingbot", "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)"), + ("MSNBot", "Mozilla/5.0 (compatible; msnbot/2.0b; +http://search.msn.com/msnbot.htm)"), + ("MSNBot Media", "Mozilla/5.0 (compatible; msnbot-media/1.1; +http://search.msn.com/msnbot.htm)"), + ("Bing Ads Bot", "Mozilla/5.0 (compatible; bingadsbot/2.0; +http://www.bing.com/bingbot.htm)"), + + # Yahoo + ("Yahoo Slurp", "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"), + ("Yahoo Deer Spider", "Mozilla/5.0 (compatible; Yahoo! Deer Spider; http://help.yahoo.com/help/us/ysearch/crawlers)"), + + # DuckDuckGo + ("DuckDuckBot", "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)"), + + # Yandex + ("YandexBot", "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)"), + ("Yandex Images", "Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)"), + ("Yandex Mobile Bot", "Mozilla/5.0 (compatible; YandexMobileBot/3.0; +http://yandex.com/bots)"), + ("Yandex Metrika", "Mozilla/5.0 (compatible; YandexMetrika/7.0; +http://yandex.com/bots)"), + ("Yandex Direct", "Mozilla/5.0 compatible; YandexDirect/3.0; +http://yandex.com/bots)"), + + # Baidu + ("Baiduspider", "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"), + ("Baiduboxapp", "Mozilla/5.0 (compatible; Baiduboxapp; +http://www.baidu.com/search/spider.html)"), + + # Sogou + ("Sogou Spider", "Mozilla/5.0 (compatible; Sogou web spider/4.0; +http://www.sogou.com/docs/help/webmasters.htm)"), + ("Sogou Spider (alt)", "Sogou Spider/4.0"), + + # Apple + ("Applebot", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1)"), + ("Applebot-Extended", "Mozilla/5.0 (compatible; Applebot-Extended/0.2; +https://www.apple.com/go/applebot-extended)"), + + # PetleO (PetalBot) + ("PetalBot", "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.83 Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/)"), + + # Internet Archive + ("IA Archiver", "Mozilla/5.0 (compatible; archive.org_bot +http://archive.org/details/archive.org_bot)"), + ("ia_archiver", "ia_archiver-web.common-crawl"), + + # Social bots + ("Facebook External Hit", "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)"), + ("Facebook Platform", "facebookplatform"), + ("Twitterbot", "Twitterbot/1.0"), + ("LinkedInBot", "LinkedInBot/1.0 (compatible; Mozilla/5.0; Apache-HttpClient +https://www.linkedin.com)"), + ("WhatsApp", "WhatsApp/2.21.1"), + + # Generic bot/crawler signatures (case variants) + ("Generic bot (semicolon)", "Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)"), + ("Generic spider", "Mozilla/5.0 (compatible; Spider/1.0)"), + ("Generic crawler", "MyCrawler/1.0"), +] + +# ────────────────────────────────────────────────────────────── +# Known browser User-Agents (should return False) +# ────────────────────────────────────────────────────────────── + +BROWSER_UA = [ + ("Chrome (Windows)", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"), + ("Chrome (Linux)", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"), + ("Chrome (macOS)", "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"), + ("Chrome (Android)", "Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36"), + ("Chrome (iPhone)", "Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/120.0.6099.119 Mobile/15E148 Safari/604.1"), + + ("Firefox (Windows)", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0"), + ("Firefox (Linux)", "Mozilla/5.0 (X11; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0"), + ("Firefox (Android)", "Mozilla/5.0 (Android 13; Mobile; rv:121.0) Gecko/121.0 Firefox/121.0"), + + ("Safari (macOS)", "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_2) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15"), + ("Safari (iPhone)", "Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1"), + ("Safari (iPad)", "Mozilla/5.0 (iPad; CPU OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1"), + + ("Edge (Windows)", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.2210.91"), + + ("Opera", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 OPR/105.0.0.0"), + + ("Samsung Internet", "Mozilla/5.0 (Linux; Android 13; SM-S901B) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/24.0 Chrome/120.0.0.0 Mobile Safari/537.36"), +] + +# ────────────────────────────────────────────────────────────── +# Tests +# ────────────────────────────────────────────────────────────── + + +class TestIsSpider: + """is_spider() should return True for known spider UAs and False for browsers.""" + + @pytest.mark.parametrize("name,ua", SPIDER_UA) + def test_known_spiders_detected(self, name: str, ua: str): + assert is_spider(ua) is True, f"{name} should be detected as a spider" + + @pytest.mark.parametrize("name,ua", BROWSER_UA) + def test_browsers_not_detected(self, name: str, ua: str): + assert is_spider(ua) is False, f"{name} should NOT be detected as a spider" + + def test_empty_string(self): + assert is_spider("") is False + + def test_none_like_missing_header(self): + assert is_spider(" ") is False + + +class TestGetSpiderInfo: + """get_spider_info() should return SpiderInfo for spiders, None for browsers.""" + + @pytest.mark.parametrize("name,ua", SPIDER_UA) + def test_spider_returns_info(self, name: str, ua: str): + info = get_spider_info(ua) + assert info is not None, f"{name}: expected SpiderInfo, got None" + assert info.name, f"{name}: SpiderInfo.name should not be empty" + assert info.engine, f"{name}: SpiderInfo.engine should not be empty" + assert info.category in ("search", "social", "other"), ( + f"{name}: unknown category {info.category!r}" + ) + + @pytest.mark.parametrize("name,ua", BROWSER_UA) + def test_browser_returns_none(self, name: str, ua: str): + assert get_spider_info(ua) is None, f"{name} should return None" + + def test_empty_string(self): + assert get_spider_info("") is None + + +class TestIsSearchSpider: + """is_search_spider() should distinguish search crawlers from social bots.""" + + SEARCH_SPIDER_UA = [ + "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", + "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", + "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)", + "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)", + "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)", + ] + + SOCIAL_BOT_UA = [ + "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)", + "Twitterbot/1.0", + "LinkedInBot/1.0 (compatible; Mozilla/5.0; Apache-HttpClient +https://www.linkedin.com)", + "WhatsApp/2.21.1", + ] + + @pytest.mark.parametrize("ua", SEARCH_SPIDER_UA) + def test_search_spiders_return_true(self, ua: str): + assert is_search_spider(ua) is True + + @pytest.mark.parametrize("ua", SOCIAL_BOT_UA) + def test_social_bots_return_false(self, ua: str): + assert is_search_spider(ua) is False + + @pytest.mark.parametrize("name,ua", BROWSER_UA) + def test_browsers_return_false(self, name: str, ua: str): + assert is_search_spider(ua) is False, f"{name} should not be a search spider" + + +class TestSpiderInfoDataclass: + """SpiderInfo properties should be correct.""" + + def test_is_search_spider_property(self): + info = get_spider_info("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)") + assert info is not None + assert info.is_search_spider is True + assert info.is_social_bot is False + + def test_is_social_bot_property(self): + info = get_spider_info("facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)") + assert info is not None + assert info.is_social_bot is True + assert info.is_search_spider is False + + def test_spider_info_frozen(self): + info = get_spider_info("Mozilla/5.0 (compatible; bingbot/2.0)") + assert info is not None + with pytest.raises(Exception): # FrozenInstanceError in Python 3.11+, AttributeError before + info.engine = "modified" # type: ignore + + +class TestGetUserAgent: + """get_user_agent() should extract UA from headers dict case-insensitively.""" + + def test_standard_case(self): + headers = {"User-Agent": "test-agent"} + assert get_user_agent(headers) == "test-agent" + + def test_lower_case_key(self): + headers = {"user-agent": "test-agent"} + assert get_user_agent(headers) == "test-agent" + + def test_upper_case_key(self): + headers = {"USER-AGENT": "test-agent"} + assert get_user_agent(headers) == "test-agent" + + def test_missing_header(self): + headers = {"Content-Type": "text/html"} + assert get_user_agent(headers) == "" + + def test_empty_headers(self): + assert get_user_agent({}) == "" + + def test_combined_with_is_spider(self): + headers = { + "Host": "example.com", + "User-Agent": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", + "Accept": "text/html", + } + ua = get_user_agent(headers) + assert is_spider(ua) is True + + +class TestCaseInsensitivity: + """Spider detection patterns should be case-insensitive.""" + + def test_googlebot_uppercase(self): + assert is_spider("GOOGLEBOT/2.1") is True + + def test_googlebot_mixed_case(self): + assert is_spider("googleBot/2.1") is True + + def test_bingbot_uppercase(self): + assert is_spider("BINGBOT/2.0") is True + + def test_baiduspider_lowercase(self): + assert is_spider("baiduspider/2.0") is True + + def test_facebook_mixed(self): + assert is_spider("FaceBookExternalHit/1.1") is True + + +class TestSpecificSpiderDetails: + """Verify engine and category are correctly assigned for specific bots.""" + + def test_googlebot_engine(self): + info = get_spider_info("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)") + assert info is not None + assert info.engine == "Google" + assert info.category == "search" + + def test_bingbot_engine(self): + info = get_spider_info("Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)") + assert info is not None + assert info.engine == "Bing" + assert info.category == "search" + + def test_facebook_category(self): + info = get_spider_info("facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)") + assert info is not None + assert info.engine == "Facebook" + assert info.category == "social" + + def test_twitter_category(self): + info = get_spider_info("Twitterbot/1.0") + assert info is not None + assert info.engine == "Twitter" + assert info.category == "social" + + def test_yandex_engine(self): + info = get_spider_info("Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)") + assert info is not None + assert info.engine == "Yandex" + assert info.category == "search" + + def test_apple_engine(self): + info = get_spider_info("Mozilla/5.0 (compatible; Applebot/0.1; +https://www.apple.com/go/applebot)") + assert info is not None + assert info.engine == "Apple" + assert info.category == "search" diff --git a/nginx.conf.template b/nginx.conf.template index 0d61aba..bec1d3d 100644 --- a/nginx.conf.template +++ b/nginx.conf.template @@ -78,9 +78,8 @@ http { } # Proxy all page requests to the Angular SSR server. - # The SSR server (AngularNodeAppEngine) handles both RenderMode.Server - # routes (full on-demand rendering with fresh API data) and serves - # prerendered static HTML for RenderMode.Prerender routes. + # The SSR server handles both spider SSR rendering and serves CSR shells + # to regular users, with fresh station config injected at runtime. location / { proxy_pass http://ssr_backend; proxy_set_header Host $host; @@ -92,6 +91,10 @@ http { # SSR responses should not be cached by intermediate proxies proxy_cache off; add_header X-Powered-By "Angular SSR" always; + + # Prevent browsers from caching HTML pages — content changes frequently + add_header Cache-Control "no-cache, no-store, must-revalidate" always; + add_header Pragma "no-cache" always; } } } diff --git a/server.spec.ts b/server.spec.ts index 0a6fd09..5ca3bbe 100644 --- a/server.spec.ts +++ b/server.spec.ts @@ -16,45 +16,204 @@ vi.mock('node:http', () => ({ default: { createServer: vi.fn() }, })); +vi.mock('node:fs', () => ({ + default: { readFileSync: vi.fn(() => 'static shell') }, + readFileSync: vi.fn(() => 'static shell'), +})); + +vi.mock('node:path', () => ({ + default: { join: (...args: string[]) => args.join('/'), dirname: () => '/app/server' }, + join: (...args: string[]) => args.join('/'), + dirname: () => '/app/server', +})); + +// Mock the engine manifest that only exists after ng build +vi.mock('./angular-app-engine-manifest.mjs', () => ({ + default: {}, +})); + describe('SSR server — server.ts', () => { - it('should export a default request handler', async () => { + it('should export a request handler', async () => { const mod = await import('./server'); - expect(mod.default).toBeDefined(); + expect(mod.requestHandler).toBeDefined(); }); - it('should not export PRERENDERED_ROUTES (all routes now use RenderMode.Server)', async () => { - const mod = await import('./server'); - // The server.ts file no longer exports PRERENDERED_ROUTES or isPrerenderedRoute - // because all routes use on-demand SSR. Verify the file doesn't reference them. + it('should have spider detection imports and serveCsrShell', async () => { const fs = require('fs'); const content = fs.readFileSync('./server.ts', 'utf-8'); - expect(content).not.toContain('PRERENDERED_ROUTES'); - expect(content).not.toContain('isPrerenderedRoute'); - expect(content).not.toContain('AngularAppEngine'); + expect(content).toContain('isSpider'); + expect(content).toContain('getUserAgent'); + expect(content).toContain('serveCsrShell'); + expect(content).toContain('spider_detection'); }); - it('should have a clean server with lazy engine bootstrapping', async () => { + it('should gate SSR behind spider detection', async () => { const fs = require('fs'); const content = fs.readFileSync('./server.ts', 'utf-8'); - expect(content).toContain('getEngine'); - expect(content).toContain('AngularNodeAppEngine'); - expect(content).toContain('listen(4200'); + // Verify the spider detection gate exists + expect(content).toContain("if (!isSpider(userAgent))"); + expect(content).toContain('serveCsrShell(res)'); + }); + + it('should inject fresh station config as window.__STATION_CONFIG', async () => { + const fs = require('fs'); + const content = fs.readFileSync('./server.ts', 'utf-8'); + expect(content).toContain('window.__STATION_CONFIG'); + expect(content).toContain('buildConfigScript'); + expect(content).toContain('fetchStationConfig'); + }); + + it('should inject inline theme CSS custom properties', async () => { + const fs = require('fs'); + const content = fs.readFileSync('./server.ts', 'utf-8'); + expect(content).toContain('buildThemeStyle'); + expect(content).toContain('COLOR_MAP'); + expect(content).toContain('--color-primary'); + expect(content).toContain(':root{'); + }); + + it('should serveCsrShell be async and accept (res)', async () => { + const fs = require('fs'); + const content = fs.readFileSync('./server.ts', 'utf-8'); + expect(content).toMatch(/async function serveCsrShell\s*\(\s*res/); + }); + + it('should fetch station config from SSR_API_BASE_URL with 2s timeout', async () => { + const fs = require('fs'); + const content = fs.readFileSync('./server.ts', 'utf-8'); + expect(content).toContain('SSR_API_BASE_URL'); + expect(content).toContain('api/station-config'); + expect(content).toContain('setTimeout(2000'); + }); + + it('should set no-cache headers on CSR shell responses', async () => { + const fs = require('fs'); + const content = fs.readFileSync('./server.ts', 'utf-8'); + expect(content).toContain("Cache-Control"); + expect(content).toContain("no-cache, no-store, must-revalidate"); }); }); -describe('app.server.routes.ts — RenderMode.Server for all routes', () => { - it('should use RenderMode.Server for all public routes (fresh data on every request)', async () => { +describe('spider_detection.ts — TypeScript spider detection', () => { + it('should detect Googlebot', async () => { + const { isSpider } = await import('./spider_detection'); + expect(isSpider('Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)')).toBe(true); + }); + + it('should detect Bingbot', async () => { + const { isSpider } = await import('./spider_detection'); + expect(isSpider('Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)')).toBe(true); + }); + + it('should detect DuckDuckBot', async () => { + const { isSpider } = await import('./spider_detection'); + expect(isSpider('DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)')).toBe(true); + }); + + it('should detect YandexBot', async () => { + const { isSpider } = await import('./spider_detection'); + expect(isSpider('Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)')).toBe(true); + }); + + it('should detect Baiduspider', async () => { + const { isSpider } = await import('./spider_detection'); + expect(isSpider('Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)')).toBe(true); + }); + + it('should detect Applebot', async () => { + const { isSpider } = await import('./spider_detection'); + expect(isSpider('Mozilla/5.0 (compatible; Applebot/0.1; +https://www.apple.com/go/applebot)')).toBe(true); + }); + + it('should detect Facebook bot', async () => { + const { isSpider } = await import('./spider_detection'); + expect(isSpider('facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)')).toBe(true); + }); + + it('should detect Twitterbot', async () => { + const { isSpider } = await import('./spider_detection'); + expect(isSpider('Twitterbot/1.0')).toBe(true); + }); + + it('should NOT detect Chrome', async () => { + const { isSpider } = await import('./spider_detection'); + expect(isSpider('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36')).toBe(false); + }); + + it('should NOT detect Firefox', async () => { + const { isSpider } = await import('./spider_detection'); + expect(isSpider('Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0')).toBe(false); + }); + + it('should NOT detect Safari', async () => { + const { isSpider } = await import('./spider_detection'); + expect(isSpider('Mozilla/5.0 (Macintosh; Intel Mac OS X 14_2) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15')).toBe(false); + }); + + it('should NOT detect Edge', async () => { + const { isSpider } = await import('./spider_detection'); + expect(isSpider('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.2210.91')).toBe(false); + }); + + it('should NOT detect mobile Safari', async () => { + const { isSpider } = await import('./spider_detection'); + expect(isSpider('Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1')).toBe(false); + }); + + it('should handle empty string', async () => { + const { isSpider } = await import('./spider_detection'); + expect(isSpider('')).toBe(false); + }); + + it('should handle null/undefined', async () => { + const { isSpider } = await import('./spider_detection'); + expect(isSpider('')).toBe(false); + }); + + it('isSearchSpider should distinguish search from social bots', async () => { + const { isSearchSpider, isSpider } = await import('./spider_detection'); + expect(isSpider('facebookexternalhit/1.1')).toBe(true); + expect(isSearchSpider('facebookexternalhit/1.1')).toBe(false); + expect(isSearchSpider('Mozilla/5.0 (compatible; Googlebot/2.1)')).toBe(true); + }); + + it('getSpiderInfo should return SpiderInfo for known bots', async () => { + const { getSpiderInfo } = await import('./spider_detection'); + const info = getSpiderInfo('Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'); + expect(info).not.toBeNull(); + expect(info?.engine).toBe('Google'); + expect(info?.category).toBe('search'); + }); + + it('getSpiderInfo should return null for browsers', async () => { + const { getSpiderInfo } = await import('./spider_detection'); + expect(getSpiderInfo('Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0')).toBeNull(); + }); + + it('getUserAgent should extract UA from request headers', async () => { + const { getUserAgent } = await import('./spider_detection'); + const headers: Record = { + 'user-agent': 'TestBot/1.0', + 'host': 'example.com', + }; + expect(getUserAgent(headers)).toBe('TestBot/1.0'); + }); + + it('getUserAgent should handle missing UA header', async () => { + const { getUserAgent } = await import('./spider_detection'); + expect(getUserAgent({ host: 'example.com' })).toBe(''); + }); +}); + +describe('app.server.routes.ts — RenderMode configuration', () => { + it('should use RenderMode.Server for all routes (SSR server handles routing)', async () => { const fs = require('fs'); const content = fs.readFileSync('./src/app/app.server.routes.ts', 'utf-8'); - // Every route should use RenderMode.Server - const prerenderMatches = content.match(/RenderMode\.Prerender/g); - expect(prerenderMatches).toBeNull(); - // Should have RenderMode.Server entries const serverMatches = content.match(/RenderMode\.Server/g); expect(serverMatches).not.toBeNull(); - expect(serverMatches.length).toBeGreaterThanOrEqual(8); + expect(serverMatches.length).toBeGreaterThanOrEqual(7); }); it('should have no Prerender routes (eliminates stale baked-in DEFAULT_CONFIG)', async () => { @@ -155,6 +314,13 @@ describe('nginx.conf.template — SSR proxy chain', () => { expect(content).toContain('location /api/'); expect(content).toContain('__API_UPSTREAM__'); }); + + it('should add Cache-Control no-cache headers on HTML page responses', () => { + const fs = require('fs'); + const content = fs.readFileSync('./nginx.conf.template', 'utf-8'); + expect(content).toContain('add_header Cache-Control "no-cache, no-store, must-revalidate" always'); + expect(content).toContain('add_header Pragma "no-cache" always'); + }); }); describe('main.server.ts — SSR API base URL injection', () => { diff --git a/server.ts b/server.ts index f4e84c7..6bfa6e3 100644 --- a/server.ts +++ b/server.ts @@ -4,7 +4,11 @@ import { isMainModule, writeResponseToNodeResponse, } from '@angular/ssr/node'; +import * as fs from 'node:fs'; import * as http from 'node:http'; +import * as https from 'node:https'; +import * as path from 'node:path'; +import { isSpider, getUserAgent } from './spider_detection'; /** * Manually import and register the Angular app engine manifest. @@ -20,6 +24,208 @@ import { ɵsetAngularAppEngineManifest } from '@angular/ssr'; const appEngine = new AngularNodeAppEngine(); +/** + * Station config shape — mirrors the backend API response. + * Only the color fields are used for theme injection; the rest are + * passed through verbatim for the client-side StationConfigService. + */ +interface StationConfig { + color_primary: string; + color_primary_light: string; + color_primary_dark: string; + color_primary_muted: string; + color_accent: string; + color_accent_light: string; + color_accent_dark: string; + color_accent_muted: string; + color_background: string; + color_text: string; + color_success: string; + color_danger: string; + color_info: string; + color_white: string; + color_light: string; + color_medium: string; + color_black: string; + [key: string]: unknown; +} + +/** CSS custom property names mapped from StationConfig color fields. */ +const COLOR_MAP: ReadonlyArray<{ field: keyof StationConfig; prop: string }> = [ + { field: 'color_primary', prop: '--color-primary' }, + { field: 'color_primary_light', prop: '--color-primary-light' }, + { field: 'color_primary_dark', prop: '--color-primary-dark' }, + { field: 'color_primary_muted', prop: '--color-primary-muted' }, + { field: 'color_accent', prop: '--color-accent' }, + { field: 'color_accent_light', prop: '--color-accent-light' }, + { field: 'color_accent_dark', prop: '--color-accent-dark' }, + { field: 'color_accent_muted', prop: '--color-accent-muted' }, + { field: 'color_background', prop: '--color-cream' }, + { field: 'color_text', prop: '--color-dark' }, + { field: 'color_success', prop: '--color-success' }, + { field: 'color_danger', prop: '--color-danger' }, + { field: 'color_info', prop: '--color-info' }, + { field: 'color_white', prop: '--color-white' }, + { field: 'color_light', prop: '--color-light' }, + { field: 'color_medium', prop: '--color-medium' }, + { field: 'color_black', prop: '--color-black' }, +]; + +/** + * Resolve the static CSR shell path (index.csr.html). + * + * IMPORTANT: We MUST prefer index.csr.html over index.html. + * The Angular build with prerender config produces both: + * - index.html: prerendered at build time with stale DEFAULT_CONFIG baked in + * - index.csr.html: pure CSR shell with empty — Angular hydrates client-side + * + * Serving index.html to regular users causes the flash: the prerendered DOM + * shows stale build-time content until Angular re-renders with fresh data. + * Even with window.__STATION_CONFIG injection, the prerendered innerHTML inside + * still shows old content on the first paint. + * + * Production (Docker): browser files are copied to /usr/share/nginx/html/ + * Local dev (ng build): browser files live in dist/radio-station/browser/ + * Try each candidate in order and return the first that exists. + */ +function browserIndexPath(): string { + // Production path — Dockerfile copies browser output here + // CSR shell first — avoid prerendered index.html with stale content + const productionPaths = [ + '/usr/share/nginx/html/index.csr.html', + '/usr/share/nginx/html/index.html', + '/var/www/html/index.csr.html', + '/var/www/html/index.html', + ]; + + // Local dev path — relative to the compiled server directory + const dir = typeof __dirname !== 'undefined' ? __dirname : path.dirname(new URL(import.meta.url).pathname); + const localPaths = [ + path.join(dir, 'browser', 'index.csr.html'), + path.join(dir, 'browser', 'index.html'), + path.join(dir, '..', 'browser', 'index.csr.html'), + path.join(dir, '..', 'browser', 'index.html'), + ]; + + const allCandidates = [...productionPaths, ...localPaths]; + for (const candidate of allCandidates) { + try { + if (fs.existsSync(candidate)) return candidate; + } catch { + // ignore — keep trying + } + } + // Return the first candidate so the error path has a sensible value to log + return allCandidates[0]; +} + +/** + * Build an inline ``; +} + +/** + * Build inline CSS custom properties on `:root` so the first paint carries + * the correct color scheme — eliminates the color flash that occurs when + * ThemeService.apply() runs asynchronously after the API call. + */ +function buildThemeStyle(config: StationConfig): string { + const props: string[] = []; + + for (const { field, prop } of COLOR_MAP) { + const value = (config as any)[field]; + if (value && typeof value === 'string') { + props.push(`${prop}: ${value}`); + } + } + + if (props.length === 0) return ''; + + const css = `:root{${props.join(';')}}`; + return ``; +} + +/** + * Fetch fresh station config from the backend API. + * This runs server-side — it goes through the nginx proxy chain to the + * upstream FastAPI backend. On failure we fall back to serving the shell + * unmodified (client-side fetch picks up the slack). + */ +async function fetchStationConfig(): Promise { + // SSR_API_BASE_URL points at nginx (localhost:80) which proxies /api/ to + // the upstream FastAPI backend. This avoids cross-origin issues and reuses + // the same proxy chain the SSR engine itself would use. + const baseUrl = process.env['SSR_API_BASE_URL'] || 'http://localhost:80'; + const url = `${baseUrl}/api/station-config`; + + try { + const client = baseUrl.startsWith('https') ? https : http; + const data = await new Promise((resolve, reject) => { + const req = client.get(url, { + headers: { 'Accept': 'application/json' }, + }, (res) => { + const chunks: Buffer[] = []; + res.on('data', (chunk: Buffer) => chunks.push(chunk)); + res.on('end', () => resolve(Buffer.concat(chunks).toString('utf-8'))); + res.on('error', reject); + }); + req.on('error', reject); + req.setTimeout(2000, () => { req.destroy(new Error('Station config fetch timed out')); }); + }); + return JSON.parse(data) as StationConfig; + } catch (err) { + console.warn(`[SSR] Failed to fetch station config from ${url}: ${err}`); + return null; + } +} + +/** + * Serve the CSR app shell, enhanced with fresh station config injected at + * runtime so the client bootstraps with real data — no DEFAULT_CONFIG flash. + * + * Strategy: + * 1. Read CSR shell HTML from disk + * 2. Fetch live station config from the backend API + * 3. Inject window.__STATION_CONFIG as an inline