fix: switch sitemap/robots to dynamic generation + Nginx proxy
CI Pipeline / backend-test (push) Successful in 8m58s
CI Pipeline / frontend-test (push) Successful in 29s
CI Pipeline / frontend-build (push) Successful in 31s
CI Pipeline / backend-lint (push) Successful in 10s
CI Pipeline / backend-test (pull_request) Successful in 8m56s
CI Pipeline / frontend-test (pull_request) Successful in 28s
CI Pipeline / frontend-build (pull_request) Successful in 32s
CI Pipeline / backend-lint (pull_request) Successful in 11s
CI Pipeline / quality-gate (push) Successful in 3s
CI Pipeline / quality-gate (pull_request) Successful in 3s

The reviewer noted the static sitemap.xml had a hardcoded domain in a
white-label app.  Instead of writing files to disk, we now serve
sitemap.xml and robots.txt as dynamic API endpoints that read
KMTN_WEBSITE_URL at runtime.

- Nginx proxies /sitemap.xml and /robots.txt to the backend API so
  search crawlers reach them at root paths.
- Removed the startup filesystem-write that would fail in Docker
  (backend can't write to Nginx container's static dir).
- Simplified sitemap_export.py to pure generation functions (no I/O).
- Added /api/sitemap/regenerate admin endpoint for consistency.
This commit is contained in:
Hermes Robot committed 2026-08-04 20:06:00 +00:00
1 parent 5484c761b4
commit 3dead2921a
4 files changed
+67 -99

No files matched your search

+8 -72
View File
@@ -1,24 +1,13 @@
"""Sitemap & robots.txt export: generate tenant-aware SEO files.
"""Sitemap & robots.txt generation: tenant-aware SEO content.
Because this is a white-label app where each tenant has its own domain
(set via KMTN_WEBSITE_URL), the sitemap and robots.txt are generated
at runtime from the station config — one static write per startup,
plus an optional admin-triggered regeneration.
Follows the same pattern as theme_export.py:
- Atomic write (tmp + os.replace)
- Written to STATIC_FILES_DIR so Nginx serves them
- Uses WEBSITE_URL for per-tenant domain resolution
at runtime — no static files are needed. The FastAPI /api/sitemap/*
endpoints call these functions; Nginx proxies /sitemap.xml and
/robots.txt to those endpoints so search-engine crawlers hit the right
content at the root URL.
"""
import logging
from datetime import datetime, timezone
from pathlib import Path
from app.config import settings
logger = logging.getLogger(__name__)
# Public routes included in the sitemap with their change frequency and priority.
SEOMAP_ROUTES = [
{"path": "/", "changefreq": "daily", "priority": 1.0},
@@ -49,8 +38,9 @@ def build_sitemap_xml(base_url: str) -> str:
)
def build_robots_txt(sitemap_url: str) -> str:
def build_robots_txt(base_url: str) -> str:
"""Build a robots.txt string that references the correct sitemap URL."""
sitemap = base_url.rstrip("/") + "/sitemap.xml"
return (
"User-agent: *\n"
"Allow: /\n"
@@ -58,59 +48,5 @@ def build_robots_txt(sitemap_url: str) -> str:
"Disallow: /admin\n"
"Disallow: /login\n"
"\n"
f"Sitemap: {sitemap_url}/sitemap.xml\n"
f"Sitemap: {sitemap}\n"
)
def get_sitemap_status() -> dict:
"""Check if sitemap.xml and robots.txt exist on disk and return metadata."""
base = Path(settings.STATIC_FILES_DIR)
result = {}
for name in ("sitemap.xml", "robots.txt"):
path = base / name
if not path.exists():
result[name] = {"exists": False, "path": str(path)}
continue
stat = path.stat()
result[name] = {
"exists": True,
"path": str(path),
"size": stat.st_size,
"modified": datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat(),
}
return result
def write_sitemap_files() -> dict:
"""Write sitemap.xml and robots.txt to the static files directory.
Uses atomic write (write to .tmp, then os.replace) to prevent
Nginx from serving a partially-written file.
Returns a status dict with file paths.
"""
base_url = settings.WEBSITE_URL
static_dir = Path(settings.STATIC_FILES_DIR)
static_dir.mkdir(parents=True, exist_ok=True)
# sitemap.xml
sitemap_content = build_sitemap_xml(base_url)
sitemap_path = static_dir / "sitemap.xml"
tmp_path = sitemap_path.with_suffix(".tmp")
tmp_path.write_text(sitemap_content, encoding="utf-8")
tmp_path.replace(sitemap_path)
logger.info("Wrote sitemap.xml (%d bytes) to %s, base_url=%s", len(sitemap_content), sitemap_path, base_url)
# robots.txt
robots_content = build_robots_txt(base_url)
robots_path = static_dir / "robots.txt"
tmp_path = robots_path.with_suffix(".tmp")
tmp_path.write_text(robots_content, encoding="utf-8")
tmp_path.replace(robots_path)
logger.info("Wrote robots.txt (%d bytes) to %s", len(robots_content), robots_path)
return {
"sitemap.xml": str(sitemap_path),
"robots.txt": str(robots_path),
"base_url": base_url,
}