fix: switch sitemap/robots to dynamic generation + Nginx proxy
CI Pipeline / backend-test (push) Successful in 8m58s
CI Pipeline / frontend-test (push) Successful in 29s
CI Pipeline / frontend-build (push) Successful in 31s
CI Pipeline / backend-lint (push) Successful in 10s
CI Pipeline / backend-test (pull_request) Successful in 8m56s
CI Pipeline / frontend-test (pull_request) Successful in 28s
CI Pipeline / frontend-build (pull_request) Successful in 32s
CI Pipeline / backend-lint (pull_request) Successful in 11s
CI Pipeline / quality-gate (push) Successful in 3s
CI Pipeline / quality-gate (pull_request) Successful in 3s
CI Pipeline / backend-test (push) Successful in 8m58s
CI Pipeline / frontend-test (push) Successful in 29s
CI Pipeline / frontend-build (push) Successful in 31s
CI Pipeline / backend-lint (push) Successful in 10s
CI Pipeline / backend-test (pull_request) Successful in 8m56s
CI Pipeline / frontend-test (pull_request) Successful in 28s
CI Pipeline / frontend-build (pull_request) Successful in 32s
CI Pipeline / backend-lint (pull_request) Successful in 11s
CI Pipeline / quality-gate (push) Successful in 3s
CI Pipeline / quality-gate (pull_request) Successful in 3s
The reviewer noted the static sitemap.xml had a hardcoded domain in a white-label app. Instead of writing files to disk, we now serve sitemap.xml and robots.txt as dynamic API endpoints that read KMTN_WEBSITE_URL at runtime. - Nginx proxies /sitemap.xml and /robots.txt to the backend API so search crawlers reach them at root paths. - Removed the startup filesystem-write that would fail in Docker (backend can't write to Nginx container's static dir). - Simplified sitemap_export.py to pure generation functions (no I/O). - Added /api/sitemap/regenerate admin endpoint for consistency.
This commit is contained in:
1 parent
5484c761b4
commit
3dead2921a
4 files changed
+67
-99
No files matched your search
@@ -1,24 +1,13 @@
|
||||
"""Sitemap & robots.txt export: generate tenant-aware SEO files.
|
||||
"""Sitemap & robots.txt generation: tenant-aware SEO content.
|
||||
|
||||
Because this is a white-label app where each tenant has its own domain
|
||||
(set via KMTN_WEBSITE_URL), the sitemap and robots.txt are generated
|
||||
at runtime from the station config — one static write per startup,
|
||||
plus an optional admin-triggered regeneration.
|
||||
|
||||
Follows the same pattern as theme_export.py:
|
||||
- Atomic write (tmp + os.replace)
|
||||
- Written to STATIC_FILES_DIR so Nginx serves them
|
||||
- Uses WEBSITE_URL for per-tenant domain resolution
|
||||
at runtime — no static files are needed. The FastAPI /api/sitemap/*
|
||||
endpoints call these functions; Nginx proxies /sitemap.xml and
|
||||
/robots.txt to those endpoints so search-engine crawlers hit the right
|
||||
content at the root URL.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Public routes included in the sitemap with their change frequency and priority.
|
||||
SEOMAP_ROUTES = [
|
||||
{"path": "/", "changefreq": "daily", "priority": 1.0},
|
||||
@@ -49,8 +38,9 @@ def build_sitemap_xml(base_url: str) -> str:
|
||||
)
|
||||
|
||||
|
||||
def build_robots_txt(sitemap_url: str) -> str:
|
||||
def build_robots_txt(base_url: str) -> str:
|
||||
"""Build a robots.txt string that references the correct sitemap URL."""
|
||||
sitemap = base_url.rstrip("/") + "/sitemap.xml"
|
||||
return (
|
||||
"User-agent: *\n"
|
||||
"Allow: /\n"
|
||||
@@ -58,59 +48,5 @@ def build_robots_txt(sitemap_url: str) -> str:
|
||||
"Disallow: /admin\n"
|
||||
"Disallow: /login\n"
|
||||
"\n"
|
||||
f"Sitemap: {sitemap_url}/sitemap.xml\n"
|
||||
f"Sitemap: {sitemap}\n"
|
||||
)
|
||||
|
||||
|
||||
def get_sitemap_status() -> dict:
|
||||
"""Check if sitemap.xml and robots.txt exist on disk and return metadata."""
|
||||
base = Path(settings.STATIC_FILES_DIR)
|
||||
result = {}
|
||||
for name in ("sitemap.xml", "robots.txt"):
|
||||
path = base / name
|
||||
if not path.exists():
|
||||
result[name] = {"exists": False, "path": str(path)}
|
||||
continue
|
||||
stat = path.stat()
|
||||
result[name] = {
|
||||
"exists": True,
|
||||
"path": str(path),
|
||||
"size": stat.st_size,
|
||||
"modified": datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat(),
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def write_sitemap_files() -> dict:
|
||||
"""Write sitemap.xml and robots.txt to the static files directory.
|
||||
|
||||
Uses atomic write (write to .tmp, then os.replace) to prevent
|
||||
Nginx from serving a partially-written file.
|
||||
|
||||
Returns a status dict with file paths.
|
||||
"""
|
||||
base_url = settings.WEBSITE_URL
|
||||
static_dir = Path(settings.STATIC_FILES_DIR)
|
||||
static_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# sitemap.xml
|
||||
sitemap_content = build_sitemap_xml(base_url)
|
||||
sitemap_path = static_dir / "sitemap.xml"
|
||||
tmp_path = sitemap_path.with_suffix(".tmp")
|
||||
tmp_path.write_text(sitemap_content, encoding="utf-8")
|
||||
tmp_path.replace(sitemap_path)
|
||||
logger.info("Wrote sitemap.xml (%d bytes) to %s, base_url=%s", len(sitemap_content), sitemap_path, base_url)
|
||||
|
||||
# robots.txt
|
||||
robots_content = build_robots_txt(base_url)
|
||||
robots_path = static_dir / "robots.txt"
|
||||
tmp_path = robots_path.with_suffix(".tmp")
|
||||
tmp_path.write_text(robots_content, encoding="utf-8")
|
||||
tmp_path.replace(robots_path)
|
||||
logger.info("Wrote robots.txt (%d bytes) to %s", len(robots_content), robots_path)
|
||||
|
||||
return {
|
||||
"sitemap.xml": str(sitemap_path),
|
||||
"robots.txt": str(robots_path),
|
||||
"base_url": base_url,
|
||||
}
|
||||
Reference in new issue
Block a user