Compare commits
11
Commits
c0b538a7cc
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
db402f8192 | ||
|
|
bb45e8d6ae | ||
|
|
e78a725839 | ||
|
|
849856aecb | ||
|
|
1b387700d3 | ||
|
|
b20fefc432 | ||
|
|
e44f27d1cd | ||
|
|
9561339cf5 | ||
|
|
c5d80f4caa | ||
|
|
96011ef200 | ||
|
|
54bd869042 |
No files matched your search
@@ -0,0 +1 @@
|
|||||||
|
export default {};
|
||||||
@@ -283,3 +283,8 @@ class MobileBuildArtifact(Base):
|
|||||||
sha256 = Column(String(64), nullable=False)
|
sha256 = Column(String(64), nullable=False)
|
||||||
size = Column(Integer, nullable=False)
|
size = Column(Integer, nullable=False)
|
||||||
created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
|
created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
|
||||||
|
|
||||||
|
|
||||||
|
# Import user_models so that User, Role, UserRole are registered on Base.metadata
|
||||||
|
# before SQLAlchemy validates cross-table foreign keys (e.g. mobile_build_requests.created_by_id -> users.id).
|
||||||
|
from app.user_models import * # noqa: F401,F403
|
||||||
@@ -0,0 +1,234 @@
|
|||||||
|
"""Search engine spider / bot detection based on User-Agent headers.
|
||||||
|
|
||||||
|
Provides utilities to identify well-known search engine crawlers so the
|
||||||
|
application can make informed decisions (e.g., serving pre-rendered HTML
|
||||||
|
for SEO, rate-limiting bot traffic, or tracking crawl statistics).
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
from app.spider_detection import is_spider, get_spider_info
|
||||||
|
|
||||||
|
if is_spider(user_agent):
|
||||||
|
spider = get_spider_info(user_agent)
|
||||||
|
print(f\"{spider.name} ({spider.engine}) detected\")
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from enum import Enum
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
|
||||||
|
class SpiderEngine(str, Enum):
|
||||||
|
"""Well-known search engine crawlers."""
|
||||||
|
|
||||||
|
GOOGLE = "Google"
|
||||||
|
BING = "Bing"
|
||||||
|
YAHOO = "Yahoo"
|
||||||
|
DUCKDUCKGO = "DuckDuckGo"
|
||||||
|
BINGBOT = "Bing" # alias kept for naming clarity — maps to Bing engine
|
||||||
|
YANDEX = "Yandex"
|
||||||
|
BAIDU = "Baidu"
|
||||||
|
SOCK = "Sogou"
|
||||||
|
FACEBOT = "Facebook"
|
||||||
|
TWITTERBOT = "Twitter"
|
||||||
|
APPLEBOT = "Apple"
|
||||||
|
PETLEO = "PetleO"
|
||||||
|
IA_archiver = "IA Archiver"
|
||||||
|
OTHER = "Other"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class SpiderInfo:
|
||||||
|
"""Identified spider details.
|
||||||
|
|
||||||
|
Attributes:
|
||||||
|
engine: Human-friendly name of the search engine / platform.
|
||||||
|
name: Exact bot name from the User-Agent (e.g. 'Googlebot').
|
||||||
|
category: High-level category — 'search', 'social', or 'other'.
|
||||||
|
"""
|
||||||
|
|
||||||
|
engine: str
|
||||||
|
name: str
|
||||||
|
category: str # "search" | "social" | "other"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def is_search_spider(self) -> bool:
|
||||||
|
return self.category == "search"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def is_social_bot(self) -> bool:
|
||||||
|
return self.category == "social"
|
||||||
|
|
||||||
|
|
||||||
|
# ── Spider patterns ──────────────────────────────────────────
|
||||||
|
# Each tuple: (compiled-regex, engine-label, category)
|
||||||
|
# Order matters: more specific patterns first to avoid false positives.
|
||||||
|
|
||||||
|
_SPIDER_PATTERNS: list[tuple[re.Pattern[str], str, str]] = [
|
||||||
|
# ── Google family ─────────────────────────────────────
|
||||||
|
# Main crawler (desktop)
|
||||||
|
(re.compile(r"Googlebot/[\d.]+", re.IGNORECASE), "Google", "search"),
|
||||||
|
# Googlebot for mobile
|
||||||
|
(re.compile(r"Googlebot-Mobile", re.IGNORECASE), "Google", "search"),
|
||||||
|
# Google Images
|
||||||
|
(re.compile(r"Googlebot-Image", re.IGNORECASE), "Google", "search"),
|
||||||
|
# Google News
|
||||||
|
(re.compile(r"Googlebot-News", re.IGNORECASE), "Google", "search"),
|
||||||
|
# Google Ads/AdSense
|
||||||
|
(re.compile(r"Mediapartners-Google", re.IGNORECASE), "Google", "search"),
|
||||||
|
# Google Storebot
|
||||||
|
(re.compile(r"Storebot-Google", re.IGNORECASE), "Google", "search"),
|
||||||
|
# Google Other (GoogleOther)
|
||||||
|
(re.compile(r"Google-AdSense", re.IGNORECASE), "Google", "search"),
|
||||||
|
|
||||||
|
# ── Bing / Microsoft ──────────────────────────────────
|
||||||
|
(re.compile(r"bingbot/[\d.]+", re.IGNORECASE), "Bing", "search"),
|
||||||
|
# MSNBot (legacy Bing crawler)
|
||||||
|
(re.compile(r"msnbot/[\d.]+", re.IGNORECASE), "Bing", "search"),
|
||||||
|
# MSNBot Media
|
||||||
|
(re.compile(r"msnbot-media/[\d.]+", re.IGNORECASE), "Bing", "search"),
|
||||||
|
# bingadsbot
|
||||||
|
(re.compile(r"bingadsbot/[\d.]+", re.IGNORECASE), "Bing", "search"),
|
||||||
|
# WebPreview / Slurp variants from Microsoft
|
||||||
|
(re.compile(r"webpreview", re.IGNORECASE), "Bing", "search"),
|
||||||
|
|
||||||
|
# ── Yahoo / Slurp ─────────────────────────────────────
|
||||||
|
(re.compile(r"Yahoo! Slurp", re.IGNORECASE), "Yahoo", "search"),
|
||||||
|
(re.compile(r"Yahoo! Slurp China", re.IGNORECASE), "Yahoo", "search"),
|
||||||
|
(re.compile(r"Yahoo Deer Spider", re.IGNORECASE), "Yahoo", "search"),
|
||||||
|
(re.compile(r"YandexBot", re.IGNORECASE), "Yandex", "search"),
|
||||||
|
|
||||||
|
# ── DuckDuckGo ────────────────────────────────────────
|
||||||
|
(re.compile(r"DuckDuckBot/[\d.]+", re.IGNORECASE), "DuckDuckGo", "search"),
|
||||||
|
|
||||||
|
# ── Yandex (RU) ───────────────────────────────────────
|
||||||
|
(re.compile(r"YandexBot/[\d.]+", re.IGNORECASE), "Yandex", "search"),
|
||||||
|
(re.compile(r"YandexImages/[\d.]+", re.IGNORECASE), "Yandex", "search"),
|
||||||
|
(re.compile(r"YandexAccessibilityBot", re.IGNORECASE), "Yandex", "search"),
|
||||||
|
(re.compile(r"YandexBlogs", re.IGNORECASE), "Yandex", "search"),
|
||||||
|
(re.compile(r"YandexDirect", re.IGNORECASE), "Yandex", "search"),
|
||||||
|
(re.compile(r"YandexFavicons", re.IGNORECASE), "Yandex", "search"),
|
||||||
|
(re.compile(r"YandexMarket", re.IGNORECASE), "Yandex", "search"),
|
||||||
|
(re.compile(r"YandexMobileBot", re.IGNORECASE), "Yandex", "search"),
|
||||||
|
(re.compile(r"YandexMetrika", re.IGNORECASE), "Yandex", "search"),
|
||||||
|
(re.compile(r"YandexSpravochnik", re.IGNORECASE), "Yandex", "search"),
|
||||||
|
(re.compile(r"YandexSitelinks", re.IGNORECASE), "Yandex", "search"),
|
||||||
|
(re.compile(r"YandexTracker", re.IGNORECASE), "Yandex", "search"),
|
||||||
|
(re.compile(r"YandexVertis", re.IGNORECASE), "Yandex", "search"),
|
||||||
|
(re.compile(r"YandexWebmaster", re.IGNORECASE), "Yandex", "search"),
|
||||||
|
|
||||||
|
# ── Baidu (CN) ────────────────────────────────────────
|
||||||
|
(re.compile(r"Baiduspider/[\d.]+", re.IGNORECASE), "Baidu", "search"),
|
||||||
|
(re.compile(r"Baiduboxapp", re.IGNORECASE), "Baidu", "search"),
|
||||||
|
|
||||||
|
# ── Sogou (CN) ────────────────────────────────────────
|
||||||
|
(re.compile(r"Sogou [Ss]pider", re.IGNORECASE), "Sogou", "search"),
|
||||||
|
(re.compile(r"Sogou web spider", re.IGNORECASE), "Sogou", "search"),
|
||||||
|
|
||||||
|
# ── Apple ─────────────────────────────────────────────
|
||||||
|
(re.compile(r"Applebot/[\d.]+", re.IGNORECASE), "Apple", "search"),
|
||||||
|
(re.compile(r"Applebot-Extended", re.IGNORECASE), "Apple", "search"),
|
||||||
|
|
||||||
|
# ── PetleO (Japan) ────────────────────────────────────
|
||||||
|
(re.compile(r"petalbot", re.IGNORECASE), "PetleO", "search"),
|
||||||
|
|
||||||
|
# ── Internet Archive ──────────────────────────────────
|
||||||
|
(re.compile(r"ia_archiver", re.IGNORECASE), "IA Archiver", "other"),
|
||||||
|
(re.compile(r"archive\.org_bot", re.IGNORECASE), "IA Archiver", "other"),
|
||||||
|
|
||||||
|
# ── Social media bots ─────────────────────────────────
|
||||||
|
(re.compile(r"facebookexternalhit/[\d.]+", re.IGNORECASE), "Facebook", "social"),
|
||||||
|
(re.compile(r"facebookplatform", re.IGNORECASE), "Facebook", "social"),
|
||||||
|
(re.compile(r"Twitterbot/[\d.]+", re.IGNORECASE), "Twitter", "social"),
|
||||||
|
(re.compile(r"LinkedInBot/[\d.]+", re.IGNORECASE), "LinkedIn", "social"),
|
||||||
|
(re.compile(r"W3C_Validator", re.IGNORECASE), "W3C", "other"),
|
||||||
|
(re.compile(r"WhatsApp/[\d.]+", re.IGNORECASE), "WhatsApp", "social"),
|
||||||
|
|
||||||
|
# ── Generic / catch-all bot signatures ────────────────────────
|
||||||
|
# These are last to reduce false positives on real browsers that
|
||||||
|
# happen to include the word "bot" in extension names.
|
||||||
|
# Standard bot naming convention: SomeBot/X.Y
|
||||||
|
(re.compile(r"Bot/[\d.]", re.IGNORECASE), "Other", "other"),
|
||||||
|
# Semicolon-delimited bot markers: "; bot" or ";bot"
|
||||||
|
(re.compile(r";\s*bot\s*;?", re.IGNORECASE), "Other", "other"),
|
||||||
|
# Spider anywhere in the UA
|
||||||
|
(re.compile(r"spider", re.IGNORECASE), "Other", "other"),
|
||||||
|
# Crawler anywhere in the UA
|
||||||
|
(re.compile(r"crawler", re.IGNORECASE), "Other", "other"),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
# ── Public API ───────────────────────────────────────────────
|
||||||
|
|
||||||
|
def is_spider(user_agent: str) -> bool:
|
||||||
|
"""Return True if the User-Agent string belongs to a known spider/bot.
|
||||||
|
|
||||||
|
This is a fast check — use `get_spider_info()` when you need details.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
user_agent: The raw User-Agent header value.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True if a spider pattern matches, False for normal browsers.
|
||||||
|
"""
|
||||||
|
if not user_agent:
|
||||||
|
return False
|
||||||
|
for pattern, _, _ in _SPIDER_PATTERNS:
|
||||||
|
if pattern.search(user_agent):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def get_spider_info(user_agent: str) -> Optional[SpiderInfo]:
|
||||||
|
"""Identify a spider from a User-Agent string.
|
||||||
|
|
||||||
|
Returns the first matching SpiderInfo or None if the UA does not
|
||||||
|
belong to a recognized bot.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
user_agent: The raw User-Agent header value.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
SpiderInfo with engine/name/category, or None for regular browsers.
|
||||||
|
"""
|
||||||
|
if not user_agent:
|
||||||
|
return None
|
||||||
|
for pattern, engine, category in _SPIDER_PATTERNS:
|
||||||
|
match = pattern.search(user_agent)
|
||||||
|
if match:
|
||||||
|
return SpiderInfo(engine=engine, name=match.group(0), category=category)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def is_search_spider(user_agent: str) -> bool:
|
||||||
|
"""Return True only for search-engine crawlers (not social bots).
|
||||||
|
|
||||||
|
Useful when you need SSR/pre-rendering only for SEO-relevant crawlers.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
user_agent: The raw User-Agent header value.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True only if a search-engine spider pattern matches.
|
||||||
|
"""
|
||||||
|
info = get_spider_info(user_agent)
|
||||||
|
return info.is_search_spider if info else False
|
||||||
|
|
||||||
|
|
||||||
|
# ── FastAPI middleware helper ────────────────────────────────
|
||||||
|
|
||||||
|
def get_user_agent(headers: dict) -> str:
|
||||||
|
"""Extract the User-Agent string from a headers dict (case-insensitive).
|
||||||
|
|
||||||
|
Args:
|
||||||
|
headers: A dict-like mapping of header names to values.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The User-Agent value, empty string if not present.
|
||||||
|
"""
|
||||||
|
for key, value in headers.items():
|
||||||
|
if key.lower() == "user-agent":
|
||||||
|
return value
|
||||||
|
return ""
|
||||||
@@ -0,0 +1,289 @@
|
|||||||
|
"""Unit tests for spider/bot User-Agent detection."""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from app.spider_detection import (
|
||||||
|
get_spider_info,
|
||||||
|
get_user_agent,
|
||||||
|
is_search_spider,
|
||||||
|
is_spider,
|
||||||
|
)
|
||||||
|
|
||||||
|
# ──────────────────────────────────────────────────────────────
|
||||||
|
# Known spider User-Agents (should return True)
|
||||||
|
# ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
SPIDER_UA = [
|
||||||
|
# Google family
|
||||||
|
("Googlebot (desktop)", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"),
|
||||||
|
("Googlebot Mobile", "Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)"),
|
||||||
|
("Googlebot Image", "Mozilla/5.0 (compatible; Googlebot-Image/1.0; +http://www.google.com/bot.html)"),
|
||||||
|
("Googlebot News", "Mozilla/5.0 (compatible; Googlebot-News; +http://www.google.com/bot.html)"),
|
||||||
|
("Google Ads", "Mozilla/5.0 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)"),
|
||||||
|
("Storebot Google", "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Mobile Safari/537.36 (compatible; Storebot-Google/1.0)"),
|
||||||
|
("Google AdSense", "Google-AdSense-TopContent"),
|
||||||
|
|
||||||
|
# Bing family
|
||||||
|
("Bingbot", "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)"),
|
||||||
|
("MSNBot", "Mozilla/5.0 (compatible; msnbot/2.0b; +http://search.msn.com/msnbot.htm)"),
|
||||||
|
("MSNBot Media", "Mozilla/5.0 (compatible; msnbot-media/1.1; +http://search.msn.com/msnbot.htm)"),
|
||||||
|
("Bing Ads Bot", "Mozilla/5.0 (compatible; bingadsbot/2.0; +http://www.bing.com/bingbot.htm)"),
|
||||||
|
|
||||||
|
# Yahoo
|
||||||
|
("Yahoo Slurp", "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"),
|
||||||
|
("Yahoo Deer Spider", "Mozilla/5.0 (compatible; Yahoo! Deer Spider; http://help.yahoo.com/help/us/ysearch/crawlers)"),
|
||||||
|
|
||||||
|
# DuckDuckGo
|
||||||
|
("DuckDuckBot", "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)"),
|
||||||
|
|
||||||
|
# Yandex
|
||||||
|
("YandexBot", "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)"),
|
||||||
|
("Yandex Images", "Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)"),
|
||||||
|
("Yandex Mobile Bot", "Mozilla/5.0 (compatible; YandexMobileBot/3.0; +http://yandex.com/bots)"),
|
||||||
|
("Yandex Metrika", "Mozilla/5.0 (compatible; YandexMetrika/7.0; +http://yandex.com/bots)"),
|
||||||
|
("Yandex Direct", "Mozilla/5.0 compatible; YandexDirect/3.0; +http://yandex.com/bots)"),
|
||||||
|
|
||||||
|
# Baidu
|
||||||
|
("Baiduspider", "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"),
|
||||||
|
("Baiduboxapp", "Mozilla/5.0 (compatible; Baiduboxapp; +http://www.baidu.com/search/spider.html)"),
|
||||||
|
|
||||||
|
# Sogou
|
||||||
|
("Sogou Spider", "Mozilla/5.0 (compatible; Sogou web spider/4.0; +http://www.sogou.com/docs/help/webmasters.htm)"),
|
||||||
|
("Sogou Spider (alt)", "Sogou Spider/4.0"),
|
||||||
|
|
||||||
|
# Apple
|
||||||
|
("Applebot", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1)"),
|
||||||
|
("Applebot-Extended", "Mozilla/5.0 (compatible; Applebot-Extended/0.2; +https://www.apple.com/go/applebot-extended)"),
|
||||||
|
|
||||||
|
# PetleO (PetalBot)
|
||||||
|
("PetalBot", "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.83 Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/)"),
|
||||||
|
|
||||||
|
# Internet Archive
|
||||||
|
("IA Archiver", "Mozilla/5.0 (compatible; archive.org_bot +http://archive.org/details/archive.org_bot)"),
|
||||||
|
("ia_archiver", "ia_archiver-web.common-crawl"),
|
||||||
|
|
||||||
|
# Social bots
|
||||||
|
("Facebook External Hit", "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)"),
|
||||||
|
("Facebook Platform", "facebookplatform"),
|
||||||
|
("Twitterbot", "Twitterbot/1.0"),
|
||||||
|
("LinkedInBot", "LinkedInBot/1.0 (compatible; Mozilla/5.0; Apache-HttpClient +https://www.linkedin.com)"),
|
||||||
|
("WhatsApp", "WhatsApp/2.21.1"),
|
||||||
|
|
||||||
|
# Generic bot/crawler signatures (case variants)
|
||||||
|
("Generic bot (semicolon)", "Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)"),
|
||||||
|
("Generic spider", "Mozilla/5.0 (compatible; Spider/1.0)"),
|
||||||
|
("Generic crawler", "MyCrawler/1.0"),
|
||||||
|
]
|
||||||
|
|
||||||
|
# ──────────────────────────────────────────────────────────────
|
||||||
|
# Known browser User-Agents (should return False)
|
||||||
|
# ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
BROWSER_UA = [
|
||||||
|
("Chrome (Windows)", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"),
|
||||||
|
("Chrome (Linux)", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"),
|
||||||
|
("Chrome (macOS)", "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"),
|
||||||
|
("Chrome (Android)", "Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36"),
|
||||||
|
("Chrome (iPhone)", "Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/120.0.6099.119 Mobile/15E148 Safari/604.1"),
|
||||||
|
|
||||||
|
("Firefox (Windows)", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0"),
|
||||||
|
("Firefox (Linux)", "Mozilla/5.0 (X11; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0"),
|
||||||
|
("Firefox (Android)", "Mozilla/5.0 (Android 13; Mobile; rv:121.0) Gecko/121.0 Firefox/121.0"),
|
||||||
|
|
||||||
|
("Safari (macOS)", "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_2) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15"),
|
||||||
|
("Safari (iPhone)", "Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1"),
|
||||||
|
("Safari (iPad)", "Mozilla/5.0 (iPad; CPU OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1"),
|
||||||
|
|
||||||
|
("Edge (Windows)", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.2210.91"),
|
||||||
|
|
||||||
|
("Opera", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 OPR/105.0.0.0"),
|
||||||
|
|
||||||
|
("Samsung Internet", "Mozilla/5.0 (Linux; Android 13; SM-S901B) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/24.0 Chrome/120.0.0.0 Mobile Safari/537.36"),
|
||||||
|
]
|
||||||
|
|
||||||
|
# ──────────────────────────────────────────────────────────────
|
||||||
|
# Tests
|
||||||
|
# ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestIsSpider:
|
||||||
|
"""is_spider() should return True for known spider UAs and False for browsers."""
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("name,ua", SPIDER_UA)
|
||||||
|
def test_known_spiders_detected(self, name: str, ua: str):
|
||||||
|
assert is_spider(ua) is True, f"{name} should be detected as a spider"
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("name,ua", BROWSER_UA)
|
||||||
|
def test_browsers_not_detected(self, name: str, ua: str):
|
||||||
|
assert is_spider(ua) is False, f"{name} should NOT be detected as a spider"
|
||||||
|
|
||||||
|
def test_empty_string(self):
|
||||||
|
assert is_spider("") is False
|
||||||
|
|
||||||
|
def test_none_like_missing_header(self):
|
||||||
|
assert is_spider(" ") is False
|
||||||
|
|
||||||
|
|
||||||
|
class TestGetSpiderInfo:
|
||||||
|
"""get_spider_info() should return SpiderInfo for spiders, None for browsers."""
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("name,ua", SPIDER_UA)
|
||||||
|
def test_spider_returns_info(self, name: str, ua: str):
|
||||||
|
info = get_spider_info(ua)
|
||||||
|
assert info is not None, f"{name}: expected SpiderInfo, got None"
|
||||||
|
assert info.name, f"{name}: SpiderInfo.name should not be empty"
|
||||||
|
assert info.engine, f"{name}: SpiderInfo.engine should not be empty"
|
||||||
|
assert info.category in ("search", "social", "other"), (
|
||||||
|
f"{name}: unknown category {info.category!r}"
|
||||||
|
)
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("name,ua", BROWSER_UA)
|
||||||
|
def test_browser_returns_none(self, name: str, ua: str):
|
||||||
|
assert get_spider_info(ua) is None, f"{name} should return None"
|
||||||
|
|
||||||
|
def test_empty_string(self):
|
||||||
|
assert get_spider_info("") is None
|
||||||
|
|
||||||
|
|
||||||
|
class TestIsSearchSpider:
|
||||||
|
"""is_search_spider() should distinguish search crawlers from social bots."""
|
||||||
|
|
||||||
|
SEARCH_SPIDER_UA = [
|
||||||
|
"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
|
||||||
|
"Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",
|
||||||
|
"DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)",
|
||||||
|
"Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)",
|
||||||
|
"Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",
|
||||||
|
"Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)",
|
||||||
|
]
|
||||||
|
|
||||||
|
SOCIAL_BOT_UA = [
|
||||||
|
"facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)",
|
||||||
|
"Twitterbot/1.0",
|
||||||
|
"LinkedInBot/1.0 (compatible; Mozilla/5.0; Apache-HttpClient +https://www.linkedin.com)",
|
||||||
|
"WhatsApp/2.21.1",
|
||||||
|
]
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("ua", SEARCH_SPIDER_UA)
|
||||||
|
def test_search_spiders_return_true(self, ua: str):
|
||||||
|
assert is_search_spider(ua) is True
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("ua", SOCIAL_BOT_UA)
|
||||||
|
def test_social_bots_return_false(self, ua: str):
|
||||||
|
assert is_search_spider(ua) is False
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("name,ua", BROWSER_UA)
|
||||||
|
def test_browsers_return_false(self, name: str, ua: str):
|
||||||
|
assert is_search_spider(ua) is False, f"{name} should not be a search spider"
|
||||||
|
|
||||||
|
|
||||||
|
class TestSpiderInfoDataclass:
|
||||||
|
"""SpiderInfo properties should be correct."""
|
||||||
|
|
||||||
|
def test_is_search_spider_property(self):
|
||||||
|
info = get_spider_info("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")
|
||||||
|
assert info is not None
|
||||||
|
assert info.is_search_spider is True
|
||||||
|
assert info.is_social_bot is False
|
||||||
|
|
||||||
|
def test_is_social_bot_property(self):
|
||||||
|
info = get_spider_info("facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)")
|
||||||
|
assert info is not None
|
||||||
|
assert info.is_social_bot is True
|
||||||
|
assert info.is_search_spider is False
|
||||||
|
|
||||||
|
def test_spider_info_frozen(self):
|
||||||
|
info = get_spider_info("Mozilla/5.0 (compatible; bingbot/2.0)")
|
||||||
|
assert info is not None
|
||||||
|
with pytest.raises(Exception): # FrozenInstanceError in Python 3.11+, AttributeError before
|
||||||
|
info.engine = "modified" # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
class TestGetUserAgent:
|
||||||
|
"""get_user_agent() should extract UA from headers dict case-insensitively."""
|
||||||
|
|
||||||
|
def test_standard_case(self):
|
||||||
|
headers = {"User-Agent": "test-agent"}
|
||||||
|
assert get_user_agent(headers) == "test-agent"
|
||||||
|
|
||||||
|
def test_lower_case_key(self):
|
||||||
|
headers = {"user-agent": "test-agent"}
|
||||||
|
assert get_user_agent(headers) == "test-agent"
|
||||||
|
|
||||||
|
def test_upper_case_key(self):
|
||||||
|
headers = {"USER-AGENT": "test-agent"}
|
||||||
|
assert get_user_agent(headers) == "test-agent"
|
||||||
|
|
||||||
|
def test_missing_header(self):
|
||||||
|
headers = {"Content-Type": "text/html"}
|
||||||
|
assert get_user_agent(headers) == ""
|
||||||
|
|
||||||
|
def test_empty_headers(self):
|
||||||
|
assert get_user_agent({}) == ""
|
||||||
|
|
||||||
|
def test_combined_with_is_spider(self):
|
||||||
|
headers = {
|
||||||
|
"Host": "example.com",
|
||||||
|
"User-Agent": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
|
||||||
|
"Accept": "text/html",
|
||||||
|
}
|
||||||
|
ua = get_user_agent(headers)
|
||||||
|
assert is_spider(ua) is True
|
||||||
|
|
||||||
|
|
||||||
|
class TestCaseInsensitivity:
|
||||||
|
"""Spider detection patterns should be case-insensitive."""
|
||||||
|
|
||||||
|
def test_googlebot_uppercase(self):
|
||||||
|
assert is_spider("GOOGLEBOT/2.1") is True
|
||||||
|
|
||||||
|
def test_googlebot_mixed_case(self):
|
||||||
|
assert is_spider("googleBot/2.1") is True
|
||||||
|
|
||||||
|
def test_bingbot_uppercase(self):
|
||||||
|
assert is_spider("BINGBOT/2.0") is True
|
||||||
|
|
||||||
|
def test_baiduspider_lowercase(self):
|
||||||
|
assert is_spider("baiduspider/2.0") is True
|
||||||
|
|
||||||
|
def test_facebook_mixed(self):
|
||||||
|
assert is_spider("FaceBookExternalHit/1.1") is True
|
||||||
|
|
||||||
|
|
||||||
|
class TestSpecificSpiderDetails:
|
||||||
|
"""Verify engine and category are correctly assigned for specific bots."""
|
||||||
|
|
||||||
|
def test_googlebot_engine(self):
|
||||||
|
info = get_spider_info("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")
|
||||||
|
assert info is not None
|
||||||
|
assert info.engine == "Google"
|
||||||
|
assert info.category == "search"
|
||||||
|
|
||||||
|
def test_bingbot_engine(self):
|
||||||
|
info = get_spider_info("Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)")
|
||||||
|
assert info is not None
|
||||||
|
assert info.engine == "Bing"
|
||||||
|
assert info.category == "search"
|
||||||
|
|
||||||
|
def test_facebook_category(self):
|
||||||
|
info = get_spider_info("facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)")
|
||||||
|
assert info is not None
|
||||||
|
assert info.engine == "Facebook"
|
||||||
|
assert info.category == "social"
|
||||||
|
|
||||||
|
def test_twitter_category(self):
|
||||||
|
info = get_spider_info("Twitterbot/1.0")
|
||||||
|
assert info is not None
|
||||||
|
assert info.engine == "Twitter"
|
||||||
|
assert info.category == "social"
|
||||||
|
|
||||||
|
def test_yandex_engine(self):
|
||||||
|
info = get_spider_info("Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)")
|
||||||
|
assert info is not None
|
||||||
|
assert info.engine == "Yandex"
|
||||||
|
assert info.category == "search"
|
||||||
|
|
||||||
|
def test_apple_engine(self):
|
||||||
|
info = get_spider_info("Mozilla/5.0 (compatible; Applebot/0.1; +https://www.apple.com/go/applebot)")
|
||||||
|
assert info is not None
|
||||||
|
assert info.engine == "Apple"
|
||||||
|
assert info.category == "search"
|
||||||
+6
-3
@@ -78,9 +78,8 @@ http {
|
|||||||
}
|
}
|
||||||
|
|
||||||
# Proxy all page requests to the Angular SSR server.
|
# Proxy all page requests to the Angular SSR server.
|
||||||
# The SSR server (AngularNodeAppEngine) handles both RenderMode.Server
|
# The SSR server handles both spider SSR rendering and serves CSR shells
|
||||||
# routes (full on-demand rendering with fresh API data) and serves
|
# to regular users, with fresh station config injected at runtime.
|
||||||
# prerendered static HTML for RenderMode.Prerender routes.
|
|
||||||
location / {
|
location / {
|
||||||
proxy_pass http://ssr_backend;
|
proxy_pass http://ssr_backend;
|
||||||
proxy_set_header Host $host;
|
proxy_set_header Host $host;
|
||||||
@@ -92,6 +91,10 @@ http {
|
|||||||
# SSR responses should not be cached by intermediate proxies
|
# SSR responses should not be cached by intermediate proxies
|
||||||
proxy_cache off;
|
proxy_cache off;
|
||||||
add_header X-Powered-By "Angular SSR" always;
|
add_header X-Powered-By "Angular SSR" always;
|
||||||
|
|
||||||
|
# Prevent browsers from caching HTML pages — content changes frequently
|
||||||
|
add_header Cache-Control "no-cache, no-store, must-revalidate" always;
|
||||||
|
add_header Pragma "no-cache" always;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
+186
-20
@@ -16,45 +16,204 @@ vi.mock('node:http', () => ({
|
|||||||
default: { createServer: vi.fn() },
|
default: { createServer: vi.fn() },
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
vi.mock('node:fs', () => ({
|
||||||
|
default: { readFileSync: vi.fn(() => '<!DOCTYPE html><html>static shell</html>') },
|
||||||
|
readFileSync: vi.fn(() => '<!DOCTYPE html><html>static shell</html>'),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('node:path', () => ({
|
||||||
|
default: { join: (...args: string[]) => args.join('/'), dirname: () => '/app/server' },
|
||||||
|
join: (...args: string[]) => args.join('/'),
|
||||||
|
dirname: () => '/app/server',
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Mock the engine manifest that only exists after ng build
|
||||||
|
vi.mock('./angular-app-engine-manifest.mjs', () => ({
|
||||||
|
default: {},
|
||||||
|
}));
|
||||||
|
|
||||||
describe('SSR server — server.ts', () => {
|
describe('SSR server — server.ts', () => {
|
||||||
it('should export a default request handler', async () => {
|
it('should export a request handler', async () => {
|
||||||
const mod = await import('./server');
|
const mod = await import('./server');
|
||||||
expect(mod.default).toBeDefined();
|
expect(mod.requestHandler).toBeDefined();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should not export PRERENDERED_ROUTES (all routes now use RenderMode.Server)', async () => {
|
it('should have spider detection imports and serveCsrShell', async () => {
|
||||||
const mod = await import('./server');
|
|
||||||
// The server.ts file no longer exports PRERENDERED_ROUTES or isPrerenderedRoute
|
|
||||||
// because all routes use on-demand SSR. Verify the file doesn't reference them.
|
|
||||||
const fs = require('fs');
|
const fs = require('fs');
|
||||||
const content = fs.readFileSync('./server.ts', 'utf-8');
|
const content = fs.readFileSync('./server.ts', 'utf-8');
|
||||||
expect(content).not.toContain('PRERENDERED_ROUTES');
|
expect(content).toContain('isSpider');
|
||||||
expect(content).not.toContain('isPrerenderedRoute');
|
expect(content).toContain('getUserAgent');
|
||||||
expect(content).not.toContain('AngularAppEngine');
|
expect(content).toContain('serveCsrShell');
|
||||||
|
expect(content).toContain('spider_detection');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should have a clean server with lazy engine bootstrapping', async () => {
|
it('should gate SSR behind spider detection', async () => {
|
||||||
const fs = require('fs');
|
const fs = require('fs');
|
||||||
const content = fs.readFileSync('./server.ts', 'utf-8');
|
const content = fs.readFileSync('./server.ts', 'utf-8');
|
||||||
expect(content).toContain('getEngine');
|
// Verify the spider detection gate exists
|
||||||
expect(content).toContain('AngularNodeAppEngine');
|
expect(content).toContain("if (!isSpider(userAgent))");
|
||||||
expect(content).toContain('listen(4200');
|
expect(content).toContain('serveCsrShell(res)');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should inject fresh station config as window.__STATION_CONFIG', async () => {
|
||||||
|
const fs = require('fs');
|
||||||
|
const content = fs.readFileSync('./server.ts', 'utf-8');
|
||||||
|
expect(content).toContain('window.__STATION_CONFIG');
|
||||||
|
expect(content).toContain('buildConfigScript');
|
||||||
|
expect(content).toContain('fetchStationConfig');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should inject inline theme CSS custom properties', async () => {
|
||||||
|
const fs = require('fs');
|
||||||
|
const content = fs.readFileSync('./server.ts', 'utf-8');
|
||||||
|
expect(content).toContain('buildThemeStyle');
|
||||||
|
expect(content).toContain('COLOR_MAP');
|
||||||
|
expect(content).toContain('--color-primary');
|
||||||
|
expect(content).toContain(':root{');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should serveCsrShell be async and accept (res)', async () => {
|
||||||
|
const fs = require('fs');
|
||||||
|
const content = fs.readFileSync('./server.ts', 'utf-8');
|
||||||
|
expect(content).toMatch(/async function serveCsrShell\s*\(\s*res/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should fetch station config from SSR_API_BASE_URL with 2s timeout', async () => {
|
||||||
|
const fs = require('fs');
|
||||||
|
const content = fs.readFileSync('./server.ts', 'utf-8');
|
||||||
|
expect(content).toContain('SSR_API_BASE_URL');
|
||||||
|
expect(content).toContain('api/station-config');
|
||||||
|
expect(content).toContain('setTimeout(2000');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should set no-cache headers on CSR shell responses', async () => {
|
||||||
|
const fs = require('fs');
|
||||||
|
const content = fs.readFileSync('./server.ts', 'utf-8');
|
||||||
|
expect(content).toContain("Cache-Control");
|
||||||
|
expect(content).toContain("no-cache, no-store, must-revalidate");
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('app.server.routes.ts — RenderMode.Server for all routes', () => {
|
describe('spider_detection.ts — TypeScript spider detection', () => {
|
||||||
it('should use RenderMode.Server for all public routes (fresh data on every request)', async () => {
|
it('should detect Googlebot', async () => {
|
||||||
|
const { isSpider } = await import('./spider_detection');
|
||||||
|
expect(isSpider('Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)')).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should detect Bingbot', async () => {
|
||||||
|
const { isSpider } = await import('./spider_detection');
|
||||||
|
expect(isSpider('Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)')).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should detect DuckDuckBot', async () => {
|
||||||
|
const { isSpider } = await import('./spider_detection');
|
||||||
|
expect(isSpider('DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)')).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should detect YandexBot', async () => {
|
||||||
|
const { isSpider } = await import('./spider_detection');
|
||||||
|
expect(isSpider('Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)')).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should detect Baiduspider', async () => {
|
||||||
|
const { isSpider } = await import('./spider_detection');
|
||||||
|
expect(isSpider('Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)')).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should detect Applebot', async () => {
|
||||||
|
const { isSpider } = await import('./spider_detection');
|
||||||
|
expect(isSpider('Mozilla/5.0 (compatible; Applebot/0.1; +https://www.apple.com/go/applebot)')).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should detect Facebook bot', async () => {
|
||||||
|
const { isSpider } = await import('./spider_detection');
|
||||||
|
expect(isSpider('facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)')).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should detect Twitterbot', async () => {
|
||||||
|
const { isSpider } = await import('./spider_detection');
|
||||||
|
expect(isSpider('Twitterbot/1.0')).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should NOT detect Chrome', async () => {
|
||||||
|
const { isSpider } = await import('./spider_detection');
|
||||||
|
expect(isSpider('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should NOT detect Firefox', async () => {
|
||||||
|
const { isSpider } = await import('./spider_detection');
|
||||||
|
expect(isSpider('Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should NOT detect Safari', async () => {
|
||||||
|
const { isSpider } = await import('./spider_detection');
|
||||||
|
expect(isSpider('Mozilla/5.0 (Macintosh; Intel Mac OS X 14_2) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should NOT detect Edge', async () => {
|
||||||
|
const { isSpider } = await import('./spider_detection');
|
||||||
|
expect(isSpider('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.2210.91')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should NOT detect mobile Safari', async () => {
|
||||||
|
const { isSpider } = await import('./spider_detection');
|
||||||
|
expect(isSpider('Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should handle empty string', async () => {
|
||||||
|
const { isSpider } = await import('./spider_detection');
|
||||||
|
expect(isSpider('')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should handle null/undefined', async () => {
|
||||||
|
const { isSpider } = await import('./spider_detection');
|
||||||
|
expect(isSpider('')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('isSearchSpider should distinguish search from social bots', async () => {
|
||||||
|
const { isSearchSpider, isSpider } = await import('./spider_detection');
|
||||||
|
expect(isSpider('facebookexternalhit/1.1')).toBe(true);
|
||||||
|
expect(isSearchSpider('facebookexternalhit/1.1')).toBe(false);
|
||||||
|
expect(isSearchSpider('Mozilla/5.0 (compatible; Googlebot/2.1)')).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('getSpiderInfo should return SpiderInfo for known bots', async () => {
|
||||||
|
const { getSpiderInfo } = await import('./spider_detection');
|
||||||
|
const info = getSpiderInfo('Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)');
|
||||||
|
expect(info).not.toBeNull();
|
||||||
|
expect(info?.engine).toBe('Google');
|
||||||
|
expect(info?.category).toBe('search');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('getSpiderInfo should return null for browsers', async () => {
|
||||||
|
const { getSpiderInfo } = await import('./spider_detection');
|
||||||
|
expect(getSpiderInfo('Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0')).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('getUserAgent should extract UA from request headers', async () => {
|
||||||
|
const { getUserAgent } = await import('./spider_detection');
|
||||||
|
const headers: Record<string, string | string[] | undefined> = {
|
||||||
|
'user-agent': 'TestBot/1.0',
|
||||||
|
'host': 'example.com',
|
||||||
|
};
|
||||||
|
expect(getUserAgent(headers)).toBe('TestBot/1.0');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('getUserAgent should handle missing UA header', async () => {
|
||||||
|
const { getUserAgent } = await import('./spider_detection');
|
||||||
|
expect(getUserAgent({ host: 'example.com' })).toBe('');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('app.server.routes.ts — RenderMode configuration', () => {
|
||||||
|
it('should use RenderMode.Server for all routes (SSR server handles routing)', async () => {
|
||||||
const fs = require('fs');
|
const fs = require('fs');
|
||||||
const content = fs.readFileSync('./src/app/app.server.routes.ts', 'utf-8');
|
const content = fs.readFileSync('./src/app/app.server.routes.ts', 'utf-8');
|
||||||
|
|
||||||
// Every route should use RenderMode.Server
|
|
||||||
const prerenderMatches = content.match(/RenderMode\.Prerender/g);
|
|
||||||
expect(prerenderMatches).toBeNull();
|
|
||||||
|
|
||||||
// Should have RenderMode.Server entries
|
// Should have RenderMode.Server entries
|
||||||
const serverMatches = content.match(/RenderMode\.Server/g);
|
const serverMatches = content.match(/RenderMode\.Server/g);
|
||||||
expect(serverMatches).not.toBeNull();
|
expect(serverMatches).not.toBeNull();
|
||||||
expect(serverMatches.length).toBeGreaterThanOrEqual(8);
|
expect(serverMatches.length).toBeGreaterThanOrEqual(7);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should have no Prerender routes (eliminates stale baked-in DEFAULT_CONFIG)', async () => {
|
it('should have no Prerender routes (eliminates stale baked-in DEFAULT_CONFIG)', async () => {
|
||||||
@@ -155,6 +314,13 @@ describe('nginx.conf.template — SSR proxy chain', () => {
|
|||||||
expect(content).toContain('location /api/');
|
expect(content).toContain('location /api/');
|
||||||
expect(content).toContain('__API_UPSTREAM__');
|
expect(content).toContain('__API_UPSTREAM__');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('should add Cache-Control no-cache headers on HTML page responses', () => {
|
||||||
|
const fs = require('fs');
|
||||||
|
const content = fs.readFileSync('./nginx.conf.template', 'utf-8');
|
||||||
|
expect(content).toContain('add_header Cache-Control "no-cache, no-store, must-revalidate" always');
|
||||||
|
expect(content).toContain('add_header Pragma "no-cache" always');
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('main.server.ts — SSR API base URL injection', () => {
|
describe('main.server.ts — SSR API base URL injection', () => {
|
||||||
|
|||||||
@@ -4,7 +4,11 @@ import {
|
|||||||
isMainModule,
|
isMainModule,
|
||||||
writeResponseToNodeResponse,
|
writeResponseToNodeResponse,
|
||||||
} from '@angular/ssr/node';
|
} from '@angular/ssr/node';
|
||||||
|
import * as fs from 'node:fs';
|
||||||
import * as http from 'node:http';
|
import * as http from 'node:http';
|
||||||
|
import * as https from 'node:https';
|
||||||
|
import * as path from 'node:path';
|
||||||
|
import { isSpider, getUserAgent } from './spider_detection';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Manually import and register the Angular app engine manifest.
|
* Manually import and register the Angular app engine manifest.
|
||||||
@@ -20,6 +24,226 @@ import { ɵsetAngularAppEngineManifest } from '@angular/ssr';
|
|||||||
|
|
||||||
const appEngine = new AngularNodeAppEngine();
|
const appEngine = new AngularNodeAppEngine();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Station config shape — mirrors the backend API response.
|
||||||
|
* Only the color fields are used for theme injection; the rest are
|
||||||
|
* passed through verbatim for the client-side StationConfigService.
|
||||||
|
*/
|
||||||
|
interface StationConfig {
|
||||||
|
color_primary: string;
|
||||||
|
color_primary_light: string;
|
||||||
|
color_primary_dark: string;
|
||||||
|
color_primary_muted: string;
|
||||||
|
color_accent: string;
|
||||||
|
color_accent_light: string;
|
||||||
|
color_accent_dark: string;
|
||||||
|
color_accent_muted: string;
|
||||||
|
color_background: string;
|
||||||
|
color_text: string;
|
||||||
|
color_success: string;
|
||||||
|
color_danger: string;
|
||||||
|
color_info: string;
|
||||||
|
color_white: string;
|
||||||
|
color_light: string;
|
||||||
|
color_medium: string;
|
||||||
|
color_black: string;
|
||||||
|
[key: string]: unknown;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** CSS custom property names mapped from StationConfig color fields. */
|
||||||
|
const COLOR_MAP: ReadonlyArray<{ field: keyof StationConfig; prop: string }> = [
|
||||||
|
{ field: 'color_primary', prop: '--color-primary' },
|
||||||
|
{ field: 'color_primary_light', prop: '--color-primary-light' },
|
||||||
|
{ field: 'color_primary_dark', prop: '--color-primary-dark' },
|
||||||
|
{ field: 'color_primary_muted', prop: '--color-primary-muted' },
|
||||||
|
{ field: 'color_accent', prop: '--color-accent' },
|
||||||
|
{ field: 'color_accent_light', prop: '--color-accent-light' },
|
||||||
|
{ field: 'color_accent_dark', prop: '--color-accent-dark' },
|
||||||
|
{ field: 'color_accent_muted', prop: '--color-accent-muted' },
|
||||||
|
{ field: 'color_background', prop: '--color-cream' },
|
||||||
|
{ field: 'color_text', prop: '--color-dark' },
|
||||||
|
{ field: 'color_success', prop: '--color-success' },
|
||||||
|
{ field: 'color_danger', prop: '--color-danger' },
|
||||||
|
{ field: 'color_info', prop: '--color-info' },
|
||||||
|
{ field: 'color_white', prop: '--color-white' },
|
||||||
|
{ field: 'color_light', prop: '--color-light' },
|
||||||
|
{ field: 'color_medium', prop: '--color-medium' },
|
||||||
|
{ field: 'color_black', prop: '--color-black' },
|
||||||
|
];
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resolve the static CSR shell path (index.csr.html).
|
||||||
|
*
|
||||||
|
* IMPORTANT: We MUST prefer index.csr.html over index.html.
|
||||||
|
* The Angular build with prerender config produces both:
|
||||||
|
* - index.html: prerendered at build time with stale DEFAULT_CONFIG baked in
|
||||||
|
* - index.csr.html: pure CSR shell with empty <app-root> — Angular hydrates client-side
|
||||||
|
*
|
||||||
|
* Serving index.html to regular users causes the flash: the prerendered DOM
|
||||||
|
* shows stale build-time content until Angular re-renders with fresh data.
|
||||||
|
* Even with window.__STATION_CONFIG injection, the prerendered innerHTML inside
|
||||||
|
* <app-root> still shows old content on the first paint.
|
||||||
|
*
|
||||||
|
* Production (Docker): browser files are copied to /usr/share/nginx/html/
|
||||||
|
* Local dev (ng build): browser files live in dist/radio-station/browser/
|
||||||
|
* Try each candidate in order and return the first that exists.
|
||||||
|
*/
|
||||||
|
function browserIndexPath(): string {
|
||||||
|
// Production path — Dockerfile copies browser output here
|
||||||
|
// CSR shell first — avoid prerendered index.html with stale content
|
||||||
|
const productionPaths = [
|
||||||
|
'/usr/share/nginx/html/index.csr.html',
|
||||||
|
'/usr/share/nginx/html/index.html',
|
||||||
|
'/var/www/html/index.csr.html',
|
||||||
|
'/var/www/html/index.html',
|
||||||
|
];
|
||||||
|
|
||||||
|
// Local dev path — relative to the compiled server directory
|
||||||
|
const dir = typeof __dirname !== 'undefined' ? __dirname : path.dirname(new URL(import.meta.url).pathname);
|
||||||
|
const localPaths = [
|
||||||
|
path.join(dir, 'browser', 'index.csr.html'),
|
||||||
|
path.join(dir, 'browser', 'index.html'),
|
||||||
|
path.join(dir, '..', 'browser', 'index.csr.html'),
|
||||||
|
path.join(dir, '..', 'browser', 'index.html'),
|
||||||
|
];
|
||||||
|
|
||||||
|
const allCandidates = [...productionPaths, ...localPaths];
|
||||||
|
for (const candidate of allCandidates) {
|
||||||
|
try {
|
||||||
|
if (fs.existsSync(candidate)) return candidate;
|
||||||
|
} catch {
|
||||||
|
// ignore — keep trying
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Return the first candidate so the error path has a sensible value to log
|
||||||
|
return allCandidates[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build an inline `<script>` that seeds `window.__STATION_CONFIG` so the
|
||||||
|
* client-side StationConfigService can read fresh config synchronously
|
||||||
|
* during bootstrap — no async API round-trip needed, no DEFAULT_CONFIG flash.
|
||||||
|
*/
|
||||||
|
function buildConfigScript(config: StationConfig): string {
|
||||||
|
const json = JSON.stringify(config);
|
||||||
|
return `<script>window.__STATION_CONFIG=${json};</script>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Build a gradient CSS value from three color stops. */
|
||||||
|
function gradient(from: string, mid: string, to: string): string {
|
||||||
|
return `linear-gradient(135deg, ${from} 0%, ${mid} 40%, ${to} 100%)`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build inline CSS custom properties on `:root` so the first paint carries
|
||||||
|
* the correct color scheme — eliminates the color flash that occurs when
|
||||||
|
* ThemeService.apply() runs asynchronously after the API call.
|
||||||
|
*
|
||||||
|
* Includes both flat color properties AND derived gradient variables
|
||||||
|
* (--sky-gradient, --sunset-gradient, --mixed-gradient) so components
|
||||||
|
* using var() for backgrounds render correctly on the very first paint.
|
||||||
|
*/
|
||||||
|
function buildThemeStyle(config: StationConfig): string {
|
||||||
|
const props: string[] = [];
|
||||||
|
|
||||||
|
for (const { field, prop } of COLOR_MAP) {
|
||||||
|
const value = (config as any)[field];
|
||||||
|
if (value && typeof value === 'string') {
|
||||||
|
props.push(`${prop}: ${value}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Derived gradients — mirror the same formulas ThemeService.apply() uses.
|
||||||
|
// These must be injected server-side; otherwise components referencing
|
||||||
|
// var(--sky-gradient) etc. fall back to their SCSS build-time defaults
|
||||||
|
// on first paint, causing a visible color flash.
|
||||||
|
const cp = (field: keyof StationConfig): string => (config as any)[field] || '';
|
||||||
|
props.push(`--sky-gradient: ${gradient(cp('color_primary_dark'), cp('color_primary'), cp('color_primary_light'))}`);
|
||||||
|
props.push(`--sunset-gradient: ${gradient(cp('color_accent_dark'), cp('color_accent'), cp('color_accent_light'))}`);
|
||||||
|
props.push(`--mixed-gradient: linear-gradient(135deg, ${cp('color_primary_dark')} 0%, ${cp('color_primary')} 50%, ${cp('color_accent')} 100%)`);
|
||||||
|
|
||||||
|
if (props.length === 0) return '';
|
||||||
|
|
||||||
|
const css = `:root{${props.join(';')}}`;
|
||||||
|
return `<style>${css}</style>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetch fresh station config from the backend API.
|
||||||
|
* This runs server-side — it goes through the nginx proxy chain to the
|
||||||
|
* upstream FastAPI backend. On failure we fall back to serving the shell
|
||||||
|
* unmodified (client-side fetch picks up the slack).
|
||||||
|
*/
|
||||||
|
async function fetchStationConfig(): Promise<StationConfig | null> {
|
||||||
|
// SSR_API_BASE_URL points at nginx (localhost:80) which proxies /api/ to
|
||||||
|
// the upstream FastAPI backend. This avoids cross-origin issues and reuses
|
||||||
|
// the same proxy chain the SSR engine itself would use.
|
||||||
|
const baseUrl = process.env['SSR_API_BASE_URL'] || 'http://localhost:80';
|
||||||
|
const url = `${baseUrl}/api/station-config`;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const client = baseUrl.startsWith('https') ? https : http;
|
||||||
|
const data = await new Promise<string>((resolve, reject) => {
|
||||||
|
const req = client.get(url, {
|
||||||
|
headers: { 'Accept': 'application/json' },
|
||||||
|
}, (res) => {
|
||||||
|
const chunks: Buffer[] = [];
|
||||||
|
res.on('data', (chunk: Buffer) => chunks.push(chunk));
|
||||||
|
res.on('end', () => resolve(Buffer.concat(chunks).toString('utf-8')));
|
||||||
|
res.on('error', reject);
|
||||||
|
});
|
||||||
|
req.on('error', reject);
|
||||||
|
req.setTimeout(2000, () => { req.destroy(new Error('Station config fetch timed out')); });
|
||||||
|
});
|
||||||
|
return JSON.parse(data) as StationConfig;
|
||||||
|
} catch (err) {
|
||||||
|
console.warn(`[SSR] Failed to fetch station config from ${url}: ${err}`);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Serve the CSR app shell, enhanced with fresh station config injected at
|
||||||
|
* runtime so the client bootstraps with real data — no DEFAULT_CONFIG flash.
|
||||||
|
*
|
||||||
|
* Strategy:
|
||||||
|
* 1. Read CSR shell HTML from disk
|
||||||
|
* 2. Fetch live station config from the backend API
|
||||||
|
* 3. Inject window.__STATION_CONFIG as an inline <script> before </head>
|
||||||
|
* 4. Inject theme CSS custom properties as an inline <style> before </head>
|
||||||
|
* 5. The client-side StationConfigService reads window.__STATION_CONFIG
|
||||||
|
* synchronously on construction — it skips the API call when seeded.
|
||||||
|
*/
|
||||||
|
async function serveCsrShell(res: http.ServerResponse): Promise<void> {
|
||||||
|
const indexPath = browserIndexPath();
|
||||||
|
|
||||||
|
let html: string;
|
||||||
|
try {
|
||||||
|
html = fs.readFileSync(indexPath, 'utf-8');
|
||||||
|
} catch {
|
||||||
|
res.statusCode = 500;
|
||||||
|
res.end('Internal server error — index.html not found');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Attempt to fetch fresh config to inject into the shell.
|
||||||
|
const config = await fetchStationConfig();
|
||||||
|
|
||||||
|
let enhanced = html;
|
||||||
|
if (config) {
|
||||||
|
const script = buildConfigScript(config);
|
||||||
|
const style = buildThemeStyle(config);
|
||||||
|
const injection = `${style}\n${script}`;
|
||||||
|
enhanced = html.replace('</head>', `${injection}</head>`);
|
||||||
|
}
|
||||||
|
|
||||||
|
res.setHeader('Content-Type', 'text/html; charset=utf-8');
|
||||||
|
res.setHeader('X-Render-Mode', 'CSR');
|
||||||
|
res.setHeader('Cache-Control', 'no-cache, no-store, must-revalidate');
|
||||||
|
res.setHeader('Pragma', 'no-cache');
|
||||||
|
res.end(enhanced);
|
||||||
|
}
|
||||||
|
|
||||||
const requestHandler = createNodeRequestHandler(async (req, res, next) => {
|
const requestHandler = createNodeRequestHandler(async (req, res, next) => {
|
||||||
try {
|
try {
|
||||||
const response = await appEngine.handle(req);
|
const response = await appEngine.handle(req);
|
||||||
@@ -39,6 +263,17 @@ if (isMainModule(import.meta.url)) {
|
|||||||
const server = http.createServer();
|
const server = http.createServer();
|
||||||
server.on('request', async (req, res) => {
|
server.on('request', async (req, res) => {
|
||||||
try {
|
try {
|
||||||
|
// ── Spider detection gate ──────────────────────────
|
||||||
|
// Only search engine spiders get full SSR rendering.
|
||||||
|
// Regular users receive the CSR app shell from disk,
|
||||||
|
// enhanced with fresh station config injected at runtime.
|
||||||
|
const userAgent = getUserAgent(req.headers);
|
||||||
|
if (!isSpider(userAgent)) {
|
||||||
|
await serveCsrShell(res);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Spider — render full HTML via Angular SSR with live API data
|
||||||
await requestHandler(req, res, () => {
|
await requestHandler(req, res, () => {
|
||||||
res.statusCode = 404;
|
res.statusCode = 404;
|
||||||
res.end('Not Found');
|
res.end('Not Found');
|
||||||
|
|||||||
@@ -0,0 +1,147 @@
|
|||||||
|
/**
|
||||||
|
* Search engine spider / bot detection based on User-Agent headers.
|
||||||
|
*
|
||||||
|
* TypeScript mirror of backend/app/spider_detection.py.
|
||||||
|
* Used by the SSR server to decide whether to render full HTML (for spiders)
|
||||||
|
* or serve the static CSR app shell (for regular users).
|
||||||
|
*
|
||||||
|
* Usage:
|
||||||
|
* import { isSpider, getSpiderInfo } from './spider_detection';
|
||||||
|
*
|
||||||
|
* if (isSpider(userAgent)) {
|
||||||
|
* // Full SSR rendering with fresh API data
|
||||||
|
* } else {
|
||||||
|
* // Serve static CSR app shell
|
||||||
|
* }
|
||||||
|
*/
|
||||||
|
|
||||||
|
export class SpiderEngine {
|
||||||
|
public static readonly GOOGLE = 'Google';
|
||||||
|
public static readonly BING = 'Bing';
|
||||||
|
public static readonly YAHOO = 'Yahoo';
|
||||||
|
public static readonly DUCKDUCKGO = 'DuckDuckGo';
|
||||||
|
public static readonly YANDEX = 'Yandex';
|
||||||
|
public static readonly BAIDU = 'Baidu';
|
||||||
|
public static readonly SOGOU = 'Sogou';
|
||||||
|
public static readonly APPLE = 'Apple';
|
||||||
|
public static readonly PETALBOT = 'PetalBot';
|
||||||
|
public static readonly IA_ARCHIVER = 'IA Archiver';
|
||||||
|
public static readonly FACEBOOK = 'Facebook';
|
||||||
|
public static readonly TWITTER = 'Twitter';
|
||||||
|
public static readonly LINKEDIN = 'LinkedIn';
|
||||||
|
public static readonly WHATSAPP = 'WhatsApp';
|
||||||
|
public static readonly OTHER = 'Other';
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SpiderInfo {
|
||||||
|
engine: string;
|
||||||
|
name: string;
|
||||||
|
category: 'search' | 'social' | 'other';
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Spider patterns ──────────────────────────────────────────
|
||||||
|
// Each tuple: [compiled-regex, engine-label, category]
|
||||||
|
// Order matters: more specific patterns first to avoid false positives.
|
||||||
|
|
||||||
|
type SpiderPattern = [RegExp, string, 'search' | 'social' | 'other'];
|
||||||
|
|
||||||
|
const SPIDER_PATTERNS: SpiderPattern[] = [
|
||||||
|
// ── Google family ─────────────────────────────────────
|
||||||
|
[/(?:Googlebot\/[\d.]+)/i, 'Google', 'search'],
|
||||||
|
[/(?:Googlebot-Mobile)/i, 'Google', 'search'],
|
||||||
|
[/(?:Googlebot-Image)/i, 'Google', 'search'],
|
||||||
|
[/(?:Googlebot-News)/i, 'Google', 'search'],
|
||||||
|
[/(?:Mediapartners-Google)/i, 'Google', 'search'],
|
||||||
|
[/(?:Storebot-Google)/i, 'Google', 'search'],
|
||||||
|
[/(?:Google-AdSense)/i, 'Google', 'search'],
|
||||||
|
|
||||||
|
// ── Bing / Microsoft ──────────────────────────────────
|
||||||
|
[/(?:bingbot\/[\d.]+)/i, 'Bing', 'search'],
|
||||||
|
[/(?:msnbot\/[\d.]+)/i, 'Bing', 'search'],
|
||||||
|
[/(?:msnbot-media\/[\d.]+)/i, 'Bing', 'search'],
|
||||||
|
[/(?:bingadsbot\/[\d.]+)/i, 'Bing', 'search'],
|
||||||
|
[/(?:webpreview)/i, 'Bing', 'search'],
|
||||||
|
|
||||||
|
// ── Yahoo / Slurp ─────────────────────────────────────
|
||||||
|
[/(?:Yahoo!\s*Slurp)/i, 'Yahoo', 'search'],
|
||||||
|
[/(?:Yahoo!\s*Deer\s*Spider)/i, 'Yahoo', 'search'],
|
||||||
|
[/(?:YandexBot)/i, 'Yandex', 'search'],
|
||||||
|
|
||||||
|
// ── DuckDuckGo ────────────────────────────────────────
|
||||||
|
[/(?:DuckDuckBot\/[\d.]+)/i, 'DuckDuckGo', 'search'],
|
||||||
|
|
||||||
|
// ── Yandex (RU) ───────────────────────────────────────
|
||||||
|
[/(?:YandexBot\/[\d.]+)/i, 'Yandex', 'search'],
|
||||||
|
[/(?:YandexImages\/[\d.]+)/i, 'Yandex', 'search'],
|
||||||
|
[/(?:YandexAccessibilityBot)/i, 'Yandex', 'search'],
|
||||||
|
[/(?:YandexMobileBot)/i, 'Yandex', 'search'],
|
||||||
|
[/(?:YandexMetrika)/i, 'Yandex', 'search'],
|
||||||
|
[/(?:YandexDirect)/i, 'Yandex', 'search'],
|
||||||
|
[/(?:YandexMobileBot)/i, 'Yandex', 'search'],
|
||||||
|
|
||||||
|
// ── Baidu (CN) ────────────────────────────────────────
|
||||||
|
[/(?:Baiduspider\/[\d.]+)/i, 'Baidu', 'search'],
|
||||||
|
[/(?:Baiduboxapp)/i, 'Baidu', 'search'],
|
||||||
|
|
||||||
|
// ── Sogou (CN) ────────────────────────────────────────
|
||||||
|
[/(?:Sogou\s*Spider)/i, 'Sogou', 'search'],
|
||||||
|
[/(?:Sogou\s*web\s*spider)/i, 'Sogou', 'search'],
|
||||||
|
|
||||||
|
// ── Apple ─────────────────────────────────────────────
|
||||||
|
[/(?:Applebot\/[\d.]+)/i, 'Apple', 'search'],
|
||||||
|
[/(?:Applebot-Extended)/i, 'Apple', 'search'],
|
||||||
|
|
||||||
|
// ── PetalBot (China) ──────────────────────────────────
|
||||||
|
[/(?:petalbot)/i, 'PetalBot', 'search'],
|
||||||
|
|
||||||
|
// ── Internet Archive ──────────────────────────────────
|
||||||
|
[/(?:ia_archiver)/i, 'IA Archiver', 'other'],
|
||||||
|
[/(?:archive\.org_bot)/i, 'IA Archiver', 'other'],
|
||||||
|
|
||||||
|
// ── Social media bots ─────────────────────────────────
|
||||||
|
[/(?:facebookexternalhit\/[\d.]+)/i, 'Facebook', 'social'],
|
||||||
|
[/(?:facebookplatform)/i, 'Facebook', 'social'],
|
||||||
|
[/(?:Twitterbot\/[\d.]+)/i, 'Twitter', 'social'],
|
||||||
|
[/(?:LinkedInBot\/[\d.]+)/i, 'LinkedIn', 'social'],
|
||||||
|
[/(?:WhatsApp\/[\d.]+)/i, 'WhatsApp', 'social'],
|
||||||
|
[/(?:W3C_Validator)/i, 'W3C', 'other'],
|
||||||
|
|
||||||
|
// ── Generic / catch-all bot signatures ────────────────
|
||||||
|
[/(?:Bot\/[\d.])/i, 'Other', 'other'],
|
||||||
|
[/(?:;\s*bot\s*;?)/i, 'Other', 'other'],
|
||||||
|
[/(?:spider)/i, 'Other', 'other'],
|
||||||
|
[/(?:crawler)/i, 'Other', 'other'],
|
||||||
|
];
|
||||||
|
|
||||||
|
// ── Public API ───────────────────────────────────────────────
|
||||||
|
|
||||||
|
export function isSpider(userAgent: string): boolean {
|
||||||
|
if (!userAgent) return false;
|
||||||
|
for (const [pattern] of SPIDER_PATTERNS) {
|
||||||
|
if (pattern.test(userAgent)) return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getSpiderInfo(userAgent: string): SpiderInfo | null {
|
||||||
|
if (!userAgent) return null;
|
||||||
|
for (const [pattern, engine, category] of SPIDER_PATTERNS) {
|
||||||
|
const match = userAgent.match(pattern);
|
||||||
|
if (match) {
|
||||||
|
return { engine, name: match[0], category };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isSearchSpider(userAgent: string): boolean {
|
||||||
|
const info = getSpiderInfo(userAgent);
|
||||||
|
return info?.category === 'search';
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getUserAgent(headers: Record<string, string | string[] | undefined>): string {
|
||||||
|
const ua = headers['user-agent'] || headers['User-Agent'];
|
||||||
|
if (typeof ua === 'string') return ua;
|
||||||
|
if (Array.isArray(ua) && ua.length > 0) return ua[0];
|
||||||
|
return '';
|
||||||
|
}
|
||||||
@@ -213,7 +213,7 @@
|
|||||||
.about-cta {
|
.about-cta {
|
||||||
text-align: center;
|
text-align: center;
|
||||||
padding: $spacing-2xl $spacing-lg;
|
padding: $spacing-2xl $spacing-lg;
|
||||||
background: $mixed-gradient;
|
background: var(--mixed-gradient);
|
||||||
border-radius: $radius-xl;
|
border-radius: $radius-xl;
|
||||||
color: var(--color-white);
|
color: var(--color-white);
|
||||||
|
|
||||||
|
|||||||
+7
-4
@@ -54,14 +54,17 @@ export class App implements OnInit, OnDestroy {
|
|||||||
}
|
}
|
||||||
|
|
||||||
ngOnInit(): void {
|
ngOnInit(): void {
|
||||||
|
// Load station config on BOTH server and browser. On the server this
|
||||||
|
// fetches fresh data from the API so SSR renders with real colors,
|
||||||
|
// assets, and content — not DEFAULT_CONFIG baked at build time.
|
||||||
|
this.stationConfig.load().catch(() => {});
|
||||||
|
|
||||||
// These DOM operations only run in the browser.
|
// These DOM operations only run in the browser.
|
||||||
if (!isPlatformBrowser(this.platformId)) return;
|
if (!isPlatformBrowser(this.platformId)) return;
|
||||||
|
|
||||||
// Load runtime config.json and station config asynchronously in the
|
// Load runtime config.json asynchronously in the background — do not
|
||||||
// background — do not block rendering. The app already renders with
|
// block rendering.
|
||||||
// defaults from the SSR/prerendered HTML.
|
|
||||||
loadAppConfigAsync(this.http).catch(() => {});
|
loadAppConfigAsync(this.http).catch(() => {});
|
||||||
this.stationConfig.load().catch(() => {});
|
|
||||||
|
|
||||||
// Start watching router for automatic SEO metadata updates per route.
|
// Start watching router for automatic SEO metadata updates per route.
|
||||||
this.seoService.watchRouter();
|
this.seoService.watchRouter();
|
||||||
|
|||||||
@@ -34,7 +34,7 @@
|
|||||||
|
|
||||||
.event-date {
|
.event-date {
|
||||||
flex-shrink: 0;
|
flex-shrink: 0;
|
||||||
background: $mixed-gradient;
|
background: var(--mixed-gradient);
|
||||||
color: var(--color-white);
|
color: var(--color-white);
|
||||||
display: flex;
|
display: flex;
|
||||||
flex-direction: column;
|
flex-direction: column;
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
.login-section {
|
.login-section {
|
||||||
@include flex-center;
|
@include flex-center;
|
||||||
min-height: calc(100vh - 80px);
|
min-height: calc(100vh - 80px);
|
||||||
background: $sky-gradient;
|
background: var(--sky-gradient);
|
||||||
padding: $spacing-xl;
|
padding: $spacing-xl;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -21,13 +21,13 @@
|
|||||||
|
|
||||||
h1 {
|
h1 {
|
||||||
font-family: $font-heading;
|
font-family: $font-heading;
|
||||||
color: $primary-blue;
|
color: var(--color-primary);
|
||||||
margin: $spacing-sm 0 $spacing-xs;
|
margin: $spacing-sm 0 $spacing-xs;
|
||||||
font-size: 1.75rem;
|
font-size: 1.75rem;
|
||||||
}
|
}
|
||||||
|
|
||||||
p {
|
p {
|
||||||
color: $neutral-medium;
|
color: var(--color-medium);
|
||||||
font-size: 0.9rem;
|
font-size: 0.9rem;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -49,14 +49,14 @@
|
|||||||
display: block;
|
display: block;
|
||||||
font-size: 0.875rem;
|
font-size: 0.875rem;
|
||||||
font-weight: 600;
|
font-weight: 600;
|
||||||
color: $neutral-dark;
|
color: var(--color-dark);
|
||||||
margin-bottom: $spacing-xs;
|
margin-bottom: $spacing-xs;
|
||||||
}
|
}
|
||||||
|
|
||||||
input {
|
input {
|
||||||
width: 100%;
|
width: 100%;
|
||||||
padding: $spacing-sm $spacing-md;
|
padding: $spacing-sm $spacing-md;
|
||||||
border: 1px solid $neutral-light;
|
border: 1px solid var(--color-light);
|
||||||
border-radius: $radius-md;
|
border-radius: $radius-md;
|
||||||
font-size: 1rem;
|
font-size: 1rem;
|
||||||
transition: border-color $transition-fast;
|
transition: border-color $transition-fast;
|
||||||
@@ -64,7 +64,7 @@
|
|||||||
|
|
||||||
&:focus {
|
&:focus {
|
||||||
outline: none;
|
outline: none;
|
||||||
border-color: $primary-blue;
|
border-color: var(--color-primary);
|
||||||
box-shadow: 0 0 0 3px rgba($primary-blue, 0.15);
|
box-shadow: 0 0 0 3px rgba($primary-blue, 0.15);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -72,7 +72,7 @@
|
|||||||
|
|
||||||
.login-error {
|
.login-error {
|
||||||
background: rgba($danger-red, 0.1);
|
background: rgba($danger-red, 0.1);
|
||||||
color: $danger-red;
|
color: var(--color-danger);
|
||||||
padding: $spacing-sm $spacing-md;
|
padding: $spacing-sm $spacing-md;
|
||||||
border-radius: $radius-md;
|
border-radius: $radius-md;
|
||||||
font-size: 0.875rem;
|
font-size: 0.875rem;
|
||||||
@@ -81,7 +81,7 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
.btn-login {
|
.btn-login {
|
||||||
@include button-style($primary-blue, $neutral-white, $primary-blue-light);
|
@include button-style(var(--color-primary), var(--color-white), var(--color-primary-light));
|
||||||
width: 100%;
|
width: 100%;
|
||||||
padding: $spacing-sm $spacing-lg;
|
padding: $spacing-sm $spacing-lg;
|
||||||
margin-top: $spacing-sm;
|
margin-top: $spacing-sm;
|
||||||
|
|||||||
@@ -74,7 +74,18 @@ export class StationConfigService {
|
|||||||
private themeService = inject(ThemeService);
|
private themeService = inject(ThemeService);
|
||||||
private baseUrl = `${getAppConfig().apiBaseUrl}/api/station-config`;
|
private baseUrl = `${getAppConfig().apiBaseUrl}/api/station-config`;
|
||||||
|
|
||||||
readonly config = signal<StationConfig>(DEFAULT_CONFIG);
|
/**
|
||||||
|
* Read pre-seeded config injected by the server at runtime.
|
||||||
|
* The SSR server fetches fresh station config and injects it as
|
||||||
|
* `window.__STATION_CONFIG` before the Angular bundle loads.
|
||||||
|
* This eliminates the DEFAULT_CONFIG flash: the client bootstraps
|
||||||
|
* with real data instead of hardcoded defaults.
|
||||||
|
*/
|
||||||
|
readonly config = signal<StationConfig>(
|
||||||
|
typeof window !== 'undefined' && (window as any).__STATION_CONFIG
|
||||||
|
? (window as any).__STATION_CONFIG as StationConfig
|
||||||
|
: DEFAULT_CONFIG,
|
||||||
|
);
|
||||||
|
|
||||||
/** Load station config from the API. Falls back to defaults on error. */
|
/** Load station config from the API. Falls back to defaults on error. */
|
||||||
async load(): Promise<void> {
|
async load(): Promise<void> {
|
||||||
|
|||||||
+18
-1
@@ -1,10 +1,12 @@
|
|||||||
import { APP_ID } from '@angular/core';
|
import { APP_ID, APP_INITIALIZER, PLATFORM_ID } from '@angular/core';
|
||||||
|
import { isPlatformBrowser } from '@angular/common';
|
||||||
import { bootstrapApplication, BootstrapContext } from '@angular/platform-browser';
|
import { bootstrapApplication, BootstrapContext } from '@angular/platform-browser';
|
||||||
import { provideServerRendering, withRoutes } from '@angular/ssr';
|
import { provideServerRendering, withRoutes } from '@angular/ssr';
|
||||||
|
|
||||||
import { appConfig } from './app/app.config';
|
import { appConfig } from './app/app.config';
|
||||||
import { App } from './app/app';
|
import { App } from './app/app';
|
||||||
import { APP_CONFIG, AppConfig } from './app/services/app-config.service';
|
import { APP_CONFIG, AppConfig } from './app/services/app-config.service';
|
||||||
|
import { StationConfigService } from './app/services/station-config.service';
|
||||||
import { serverRoutes } from './app/app.server.routes';
|
import { serverRoutes } from './app/app.server.routes';
|
||||||
|
|
||||||
// During SSR, the API base URL must be absolute (HttpClient runs in Node.js).
|
// During SSR, the API base URL must be absolute (HttpClient runs in Node.js).
|
||||||
@@ -19,6 +21,21 @@ const serverAppConfig = {
|
|||||||
{ provide: APP_ID, useValue: 'server' },
|
{ provide: APP_ID, useValue: 'server' },
|
||||||
{ provide: APP_CONFIG, useValue: ssrAppConfig },
|
{ provide: APP_CONFIG, useValue: ssrAppConfig },
|
||||||
provideServerRendering(withRoutes(serverRoutes)),
|
provideServerRendering(withRoutes(serverRoutes)),
|
||||||
|
// Block server rendering until station config is fetched from the API.
|
||||||
|
// This ensures SSR renders with real DB data (colors, assets, content)
|
||||||
|
// instead of DEFAULT_CONFIG baked at build time. In the browser, this
|
||||||
|
// is a no-op — config loads async in ngOnInit after first paint.
|
||||||
|
{
|
||||||
|
provide: APP_INITIALIZER,
|
||||||
|
multi: true,
|
||||||
|
deps: [StationConfigService, PLATFORM_ID],
|
||||||
|
useFactory: (svc: StationConfigService, platformId: object) => {
|
||||||
|
if (!isPlatformBrowser(platformId)) {
|
||||||
|
return () => svc.load().catch(() => {});
|
||||||
|
}
|
||||||
|
return () => {};
|
||||||
|
},
|
||||||
|
},
|
||||||
],
|
],
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
+7
-25
@@ -3,33 +3,15 @@
|
|||||||
// ========== CSS Custom Properties (exposed to the browser) ==========
|
// ========== CSS Custom Properties (exposed to the browser) ==========
|
||||||
// These map SCSS variables to CSS custom properties so Angular components
|
// These map SCSS variables to CSS custom properties so Angular components
|
||||||
// can consume them via var() at runtime.
|
// can consume them via var() at runtime.
|
||||||
|
//
|
||||||
|
// COLOR variables (color_*, gradient_*) are NOT set here anymore.
|
||||||
|
// They are injected at runtime by the SSR server as an inline <style> block
|
||||||
|
// so the first paint always carries the correct station colors.
|
||||||
|
// Client-side ThemeService.apply() keeps them in sync after Angular boots.
|
||||||
|
//
|
||||||
|
// Non-color variables (fonts, radii, shadows) remain below as build-time defaults.
|
||||||
|
|
||||||
:root {
|
:root {
|
||||||
// Primary
|
|
||||||
--color-primary: #{$primary-blue};
|
|
||||||
--color-primary-light: #{$primary-blue-light};
|
|
||||||
--color-primary-dark: #{$primary-blue-dark};
|
|
||||||
--color-primary-muted: #{$primary-blue-muted};
|
|
||||||
|
|
||||||
// Accent
|
|
||||||
--color-accent: #{$accent-orange};
|
|
||||||
--color-accent-light: #{$accent-orange-light};
|
|
||||||
--color-accent-dark: #{$accent-orange-dark};
|
|
||||||
--color-accent-muted: #{$accent-orange-muted};
|
|
||||||
|
|
||||||
// Neutrals
|
|
||||||
--color-white: #{$neutral-white};
|
|
||||||
--color-cream: #{$neutral-cream};
|
|
||||||
--color-light: #{$neutral-light};
|
|
||||||
--color-medium: #{$neutral-medium};
|
|
||||||
--color-dark: #{$neutral-dark};
|
|
||||||
--color-black: #{$neutral-black};
|
|
||||||
|
|
||||||
// Semantic
|
|
||||||
--color-success: #{$success-green};
|
|
||||||
--color-danger: #{$danger-red};
|
|
||||||
--color-info: #{$info-blue};
|
|
||||||
|
|
||||||
// Typography
|
// Typography
|
||||||
--font-primary: #{$font-primary};
|
--font-primary: #{$font-primary};
|
||||||
--font-heading: #{$font-heading};
|
--font-heading: #{$font-heading};
|
||||||
|
|||||||
+2
-1
@@ -9,7 +9,8 @@
|
|||||||
},
|
},
|
||||||
"include": [
|
"include": [
|
||||||
"src/**/*.ts",
|
"src/**/*.ts",
|
||||||
"server.ts"
|
"server.ts",
|
||||||
|
"spider_detection.ts"
|
||||||
],
|
],
|
||||||
"exclude": [
|
"exclude": [
|
||||||
"src/**/*.spec.ts"
|
"src/**/*.spec.ts"
|
||||||
|
|||||||
Reference in new issue
Block a user