Hermes Robot
c5d80f4caa
CI Pipeline / backend-test (push) Successful in 12m23s
CI Pipeline / frontend-test (push) Successful in 24s
CI Pipeline / frontend-build (push) Successful in 31s
CI Pipeline / backend-lint (push) Successful in 10s
CI Pipeline / backend-test (pull_request) Successful in 17m40s
CI Pipeline / frontend-test (pull_request) Successful in 24s
CI Pipeline / frontend-build (pull_request) Successful in 31s
CI Pipeline / backend-lint (pull_request) Successful in 10s
CI Pipeline / quality-gate (push) Successful in 3s
CI Pipeline / quality-gate (pull_request) Successful in 3s
Add spider detection gate to SSR server so that search engine crawlers get full server-side rendering with fresh API data, while regular users receive the static CSR app shell from disk. - backend/app/spider_detection.py: Python spider detection module - spider_detection.ts: TypeScript mirror of spider detection patterns - server.ts: SSR gate — isSpider() check before dispatching to Angular SSR engine; non-spiders get static index.html - server.spec.ts: Added spider detection unit tests - tsconfig.app.json: Include spider_detection.ts in compilation - backend/app/models.py: Fix SQLAlchemy FK resolution (user_models import)
290 lines
14 KiB
Python
290 lines
14 KiB
Python
"""Unit tests for spider/bot User-Agent detection."""
|
|
|
|
import pytest
|
|
from app.spider_detection import (
|
|
get_spider_info,
|
|
get_user_agent,
|
|
is_search_spider,
|
|
is_spider,
|
|
)
|
|
|
|
# ──────────────────────────────────────────────────────────────
|
|
# Known spider User-Agents (should return True)
|
|
# ──────────────────────────────────────────────────────────────
|
|
|
|
SPIDER_UA = [
|
|
# Google family
|
|
("Googlebot (desktop)", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"),
|
|
("Googlebot Mobile", "Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)"),
|
|
("Googlebot Image", "Mozilla/5.0 (compatible; Googlebot-Image/1.0; +http://www.google.com/bot.html)"),
|
|
("Googlebot News", "Mozilla/5.0 (compatible; Googlebot-News; +http://www.google.com/bot.html)"),
|
|
("Google Ads", "Mozilla/5.0 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)"),
|
|
("Storebot Google", "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Mobile Safari/537.36 (compatible; Storebot-Google/1.0)"),
|
|
("Google AdSense", "Google-AdSense-TopContent"),
|
|
|
|
# Bing family
|
|
("Bingbot", "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)"),
|
|
("MSNBot", "Mozilla/5.0 (compatible; msnbot/2.0b; +http://search.msn.com/msnbot.htm)"),
|
|
("MSNBot Media", "Mozilla/5.0 (compatible; msnbot-media/1.1; +http://search.msn.com/msnbot.htm)"),
|
|
("Bing Ads Bot", "Mozilla/5.0 (compatible; bingadsbot/2.0; +http://www.bing.com/bingbot.htm)"),
|
|
|
|
# Yahoo
|
|
("Yahoo Slurp", "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"),
|
|
("Yahoo Deer Spider", "Mozilla/5.0 (compatible; Yahoo! Deer Spider; http://help.yahoo.com/help/us/ysearch/crawlers)"),
|
|
|
|
# DuckDuckGo
|
|
("DuckDuckBot", "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)"),
|
|
|
|
# Yandex
|
|
("YandexBot", "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)"),
|
|
("Yandex Images", "Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)"),
|
|
("Yandex Mobile Bot", "Mozilla/5.0 (compatible; YandexMobileBot/3.0; +http://yandex.com/bots)"),
|
|
("Yandex Metrika", "Mozilla/5.0 (compatible; YandexMetrika/7.0; +http://yandex.com/bots)"),
|
|
("Yandex Direct", "Mozilla/5.0 compatible; YandexDirect/3.0; +http://yandex.com/bots)"),
|
|
|
|
# Baidu
|
|
("Baiduspider", "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"),
|
|
("Baiduboxapp", "Mozilla/5.0 (compatible; Baiduboxapp; +http://www.baidu.com/search/spider.html)"),
|
|
|
|
# Sogou
|
|
("Sogou Spider", "Mozilla/5.0 (compatible; Sogou web spider/4.0; +http://www.sogou.com/docs/help/webmasters.htm)"),
|
|
("Sogou Spider (alt)", "Sogou Spider/4.0"),
|
|
|
|
# Apple
|
|
("Applebot", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1)"),
|
|
("Applebot-Extended", "Mozilla/5.0 (compatible; Applebot-Extended/0.2; +https://www.apple.com/go/applebot-extended)"),
|
|
|
|
# PetleO (PetalBot)
|
|
("PetalBot", "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.83 Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/)"),
|
|
|
|
# Internet Archive
|
|
("IA Archiver", "Mozilla/5.0 (compatible; archive.org_bot +http://archive.org/details/archive.org_bot)"),
|
|
("ia_archiver", "ia_archiver-web.common-crawl"),
|
|
|
|
# Social bots
|
|
("Facebook External Hit", "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)"),
|
|
("Facebook Platform", "facebookplatform"),
|
|
("Twitterbot", "Twitterbot/1.0"),
|
|
("LinkedInBot", "LinkedInBot/1.0 (compatible; Mozilla/5.0; Apache-HttpClient +https://www.linkedin.com)"),
|
|
("WhatsApp", "WhatsApp/2.21.1"),
|
|
|
|
# Generic bot/crawler signatures (case variants)
|
|
("Generic bot (semicolon)", "Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)"),
|
|
("Generic spider", "Mozilla/5.0 (compatible; Spider/1.0)"),
|
|
("Generic crawler", "MyCrawler/1.0"),
|
|
]
|
|
|
|
# ──────────────────────────────────────────────────────────────
|
|
# Known browser User-Agents (should return False)
|
|
# ──────────────────────────────────────────────────────────────
|
|
|
|
BROWSER_UA = [
|
|
("Chrome (Windows)", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"),
|
|
("Chrome (Linux)", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"),
|
|
("Chrome (macOS)", "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"),
|
|
("Chrome (Android)", "Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36"),
|
|
("Chrome (iPhone)", "Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/120.0.6099.119 Mobile/15E148 Safari/604.1"),
|
|
|
|
("Firefox (Windows)", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0"),
|
|
("Firefox (Linux)", "Mozilla/5.0 (X11; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0"),
|
|
("Firefox (Android)", "Mozilla/5.0 (Android 13; Mobile; rv:121.0) Gecko/121.0 Firefox/121.0"),
|
|
|
|
("Safari (macOS)", "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_2) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15"),
|
|
("Safari (iPhone)", "Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1"),
|
|
("Safari (iPad)", "Mozilla/5.0 (iPad; CPU OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1"),
|
|
|
|
("Edge (Windows)", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.2210.91"),
|
|
|
|
("Opera", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 OPR/105.0.0.0"),
|
|
|
|
("Samsung Internet", "Mozilla/5.0 (Linux; Android 13; SM-S901B) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/24.0 Chrome/120.0.0.0 Mobile Safari/537.36"),
|
|
]
|
|
|
|
# ──────────────────────────────────────────────────────────────
|
|
# Tests
|
|
# ──────────────────────────────────────────────────────────────
|
|
|
|
|
|
class TestIsSpider:
|
|
"""is_spider() should return True for known spider UAs and False for browsers."""
|
|
|
|
@pytest.mark.parametrize("name,ua", SPIDER_UA)
|
|
def test_known_spiders_detected(self, name: str, ua: str):
|
|
assert is_spider(ua) is True, f"{name} should be detected as a spider"
|
|
|
|
@pytest.mark.parametrize("name,ua", BROWSER_UA)
|
|
def test_browsers_not_detected(self, name: str, ua: str):
|
|
assert is_spider(ua) is False, f"{name} should NOT be detected as a spider"
|
|
|
|
def test_empty_string(self):
|
|
assert is_spider("") is False
|
|
|
|
def test_none_like_missing_header(self):
|
|
assert is_spider(" ") is False
|
|
|
|
|
|
class TestGetSpiderInfo:
|
|
"""get_spider_info() should return SpiderInfo for spiders, None for browsers."""
|
|
|
|
@pytest.mark.parametrize("name,ua", SPIDER_UA)
|
|
def test_spider_returns_info(self, name: str, ua: str):
|
|
info = get_spider_info(ua)
|
|
assert info is not None, f"{name}: expected SpiderInfo, got None"
|
|
assert info.name, f"{name}: SpiderInfo.name should not be empty"
|
|
assert info.engine, f"{name}: SpiderInfo.engine should not be empty"
|
|
assert info.category in ("search", "social", "other"), (
|
|
f"{name}: unknown category {info.category!r}"
|
|
)
|
|
|
|
@pytest.mark.parametrize("name,ua", BROWSER_UA)
|
|
def test_browser_returns_none(self, name: str, ua: str):
|
|
assert get_spider_info(ua) is None, f"{name} should return None"
|
|
|
|
def test_empty_string(self):
|
|
assert get_spider_info("") is None
|
|
|
|
|
|
class TestIsSearchSpider:
|
|
"""is_search_spider() should distinguish search crawlers from social bots."""
|
|
|
|
SEARCH_SPIDER_UA = [
|
|
"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
|
|
"Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",
|
|
"DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)",
|
|
"Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)",
|
|
"Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",
|
|
"Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)",
|
|
]
|
|
|
|
SOCIAL_BOT_UA = [
|
|
"facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)",
|
|
"Twitterbot/1.0",
|
|
"LinkedInBot/1.0 (compatible; Mozilla/5.0; Apache-HttpClient +https://www.linkedin.com)",
|
|
"WhatsApp/2.21.1",
|
|
]
|
|
|
|
@pytest.mark.parametrize("ua", SEARCH_SPIDER_UA)
|
|
def test_search_spiders_return_true(self, ua: str):
|
|
assert is_search_spider(ua) is True
|
|
|
|
@pytest.mark.parametrize("ua", SOCIAL_BOT_UA)
|
|
def test_social_bots_return_false(self, ua: str):
|
|
assert is_search_spider(ua) is False
|
|
|
|
@pytest.mark.parametrize("name,ua", BROWSER_UA)
|
|
def test_browsers_return_false(self, name: str, ua: str):
|
|
assert is_search_spider(ua) is False, f"{name} should not be a search spider"
|
|
|
|
|
|
class TestSpiderInfoDataclass:
|
|
"""SpiderInfo properties should be correct."""
|
|
|
|
def test_is_search_spider_property(self):
|
|
info = get_spider_info("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")
|
|
assert info is not None
|
|
assert info.is_search_spider is True
|
|
assert info.is_social_bot is False
|
|
|
|
def test_is_social_bot_property(self):
|
|
info = get_spider_info("facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)")
|
|
assert info is not None
|
|
assert info.is_social_bot is True
|
|
assert info.is_search_spider is False
|
|
|
|
def test_spider_info_frozen(self):
|
|
info = get_spider_info("Mozilla/5.0 (compatible; bingbot/2.0)")
|
|
assert info is not None
|
|
with pytest.raises(Exception): # FrozenInstanceError in Python 3.11+, AttributeError before
|
|
info.engine = "modified" # type: ignore
|
|
|
|
|
|
class TestGetUserAgent:
|
|
"""get_user_agent() should extract UA from headers dict case-insensitively."""
|
|
|
|
def test_standard_case(self):
|
|
headers = {"User-Agent": "test-agent"}
|
|
assert get_user_agent(headers) == "test-agent"
|
|
|
|
def test_lower_case_key(self):
|
|
headers = {"user-agent": "test-agent"}
|
|
assert get_user_agent(headers) == "test-agent"
|
|
|
|
def test_upper_case_key(self):
|
|
headers = {"USER-AGENT": "test-agent"}
|
|
assert get_user_agent(headers) == "test-agent"
|
|
|
|
def test_missing_header(self):
|
|
headers = {"Content-Type": "text/html"}
|
|
assert get_user_agent(headers) == ""
|
|
|
|
def test_empty_headers(self):
|
|
assert get_user_agent({}) == ""
|
|
|
|
def test_combined_with_is_spider(self):
|
|
headers = {
|
|
"Host": "example.com",
|
|
"User-Agent": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
|
|
"Accept": "text/html",
|
|
}
|
|
ua = get_user_agent(headers)
|
|
assert is_spider(ua) is True
|
|
|
|
|
|
class TestCaseInsensitivity:
|
|
"""Spider detection patterns should be case-insensitive."""
|
|
|
|
def test_googlebot_uppercase(self):
|
|
assert is_spider("GOOGLEBOT/2.1") is True
|
|
|
|
def test_googlebot_mixed_case(self):
|
|
assert is_spider("googleBot/2.1") is True
|
|
|
|
def test_bingbot_uppercase(self):
|
|
assert is_spider("BINGBOT/2.0") is True
|
|
|
|
def test_baiduspider_lowercase(self):
|
|
assert is_spider("baiduspider/2.0") is True
|
|
|
|
def test_facebook_mixed(self):
|
|
assert is_spider("FaceBookExternalHit/1.1") is True
|
|
|
|
|
|
class TestSpecificSpiderDetails:
|
|
"""Verify engine and category are correctly assigned for specific bots."""
|
|
|
|
def test_googlebot_engine(self):
|
|
info = get_spider_info("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")
|
|
assert info is not None
|
|
assert info.engine == "Google"
|
|
assert info.category == "search"
|
|
|
|
def test_bingbot_engine(self):
|
|
info = get_spider_info("Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)")
|
|
assert info is not None
|
|
assert info.engine == "Bing"
|
|
assert info.category == "search"
|
|
|
|
def test_facebook_category(self):
|
|
info = get_spider_info("facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)")
|
|
assert info is not None
|
|
assert info.engine == "Facebook"
|
|
assert info.category == "social"
|
|
|
|
def test_twitter_category(self):
|
|
info = get_spider_info("Twitterbot/1.0")
|
|
assert info is not None
|
|
assert info.engine == "Twitter"
|
|
assert info.category == "social"
|
|
|
|
def test_yandex_engine(self):
|
|
info = get_spider_info("Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)")
|
|
assert info is not None
|
|
assert info.engine == "Yandex"
|
|
assert info.category == "search"
|
|
|
|
def test_apple_engine(self):
|
|
info = get_spider_info("Mozilla/5.0 (compatible; Applebot/0.1; +https://www.apple.com/go/applebot)")
|
|
assert info is not None
|
|
assert info.engine == "Apple"
|
|
assert info.category == "search"
|