fix: address reviewer feedback — dynamic sitemap/robots + wildcard CI branches
CI Pipeline / backend-test (push) Successful in 8m12s
CI Pipeline / frontend-test (push) Successful in 28s
CI Pipeline / frontend-build (push) Successful in 32s
CI Pipeline / backend-lint (push) Successful in 9s
CI Pipeline / backend-test (pull_request) Successful in 8m43s
CI Pipeline / frontend-test (pull_request) Successful in 29s
CI Pipeline / frontend-build (pull_request) Successful in 32s
CI Pipeline / backend-lint (pull_request) Successful in 9s
CI Pipeline / quality-gate (push) Successful in 3s
CI Pipeline / quality-gate (pull_request) Successful in 2s
CI Pipeline / backend-test (push) Successful in 8m12s
CI Pipeline / frontend-test (push) Successful in 28s
CI Pipeline / frontend-build (push) Successful in 32s
CI Pipeline / backend-lint (push) Successful in 9s
CI Pipeline / backend-test (pull_request) Successful in 8m43s
CI Pipeline / frontend-test (pull_request) Successful in 29s
CI Pipeline / frontend-build (pull_request) Successful in 32s
CI Pipeline / backend-lint (pull_request) Successful in 9s
CI Pipeline / quality-gate (push) Successful in 3s
CI Pipeline / quality-gate (pull_request) Successful in 2s
- CI workflow: replace hardcoded branch list with ['**'] wildcard so pipeline runs on every branch push, not just 4 named branches. - Sitemap/robots.txt: moved from static files to dynamic generation. New sitemap_export.py module (follows theme_export.py pattern) uses KMTN_WEBSITE_URL to generate tenant-aware sitemap.xml and robots.txt on app startup. Atomic writes to STATIC_FILES_DIR so Nginx serves them. - Added /api/sitemap endpoint with /status (public) and /regenerate (admin-only) for runtime control. Addresses reviewer comments on PR #4: 1. 'more general way to ensure this runs on any branch on push' 2. 'whitelabel app - this file may need to be dynamically generated'
This commit is contained in:
1 parent
a7d994a122
commit
5484c761b4
6 files changed
+165
-37
No files matched your search
@@ -0,0 +1,38 @@
|
||||
"""Sitemap & robots.txt API: status check, regenerate endpoint.
|
||||
|
||||
Follows the same pattern as the theme API — public GET for status,
|
||||
admin-only POST to force regeneration.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
|
||||
from app.auth import get_current_admin_user
|
||||
from app.sitemap_export import get_sitemap_status, write_sitemap_files
|
||||
from app.user_models import User
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
async def sitemap_status() -> Dict[str, Any]:
|
||||
"""Return sitemap/robots file status. Public endpoint."""
|
||||
return get_sitemap_status()
|
||||
|
||||
|
||||
@router.post("/regenerate", status_code=200)
|
||||
async def regenerate_sitemap(
|
||||
_: User = Depends(get_current_admin_user),
|
||||
) -> Dict[str, Any]:
|
||||
"""Force regenerate sitemap.xml and robots.txt. Admin only.
|
||||
|
||||
Returns the paths of the newly written files.
|
||||
"""
|
||||
try:
|
||||
return write_sitemap_files()
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to generate sitemap: {e}",
|
||||
)
|
||||
+10
-1
@@ -10,7 +10,7 @@ from app.config import settings
|
||||
from app.database import async_session, engine
|
||||
from app.models import Base, Show, StationConfig, HistoryEntry, TeamMember, CommunityHighlight, Underwriter
|
||||
from app.user_models import User
|
||||
from app.api import events, auth, station_config, history, team, community, shows, storage, underwriters, stats, mobile_builds, theme
|
||||
from app.api import events, auth, station_config, history, team, community, shows, storage, underwriters, stats, mobile_builds, theme, sitemap
|
||||
|
||||
|
||||
def _cleanup_orphaned_sequences(sync_conn):
|
||||
@@ -183,6 +183,14 @@ async def lifespan(app: FastAPI):
|
||||
except Exception as e:
|
||||
print(f" WARNING: Failed to write theme.json on startup: {e}")
|
||||
|
||||
# Write sitemap.xml + robots.txt on startup (tenant-aware)
|
||||
try:
|
||||
from app.sitemap_export import write_sitemap_files
|
||||
info = write_sitemap_files()
|
||||
print(f" ✓ Sitemap + robots.txt written (base_url={info['base_url']})")
|
||||
except Exception as e:
|
||||
print(f" WARNING: Failed to write sitemap on startup: {e}")
|
||||
|
||||
# Check GeoLite2 database
|
||||
geo_db = settings.GEOLITE2_DB_PATH
|
||||
if os.path.exists(geo_db):
|
||||
@@ -235,6 +243,7 @@ app.include_router(underwriters.router, prefix="/api/underwriters", tags=["under
|
||||
app.include_router(stats.router, prefix="/api/stats", tags=["stats"])
|
||||
app.include_router(mobile_builds.router, prefix="/api/mobile-builds", tags=["mobile-builds"])
|
||||
app.include_router(theme.router, prefix="/api/theme", tags=["theme"])
|
||||
app.include_router(sitemap.router, prefix="/api/sitemap", tags=["sitemap"])
|
||||
|
||||
# Admin user management (available in all environments)
|
||||
from app.api import users
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
"""Sitemap & robots.txt export: generate tenant-aware SEO files.
|
||||
|
||||
Because this is a white-label app where each tenant has its own domain
|
||||
(set via KMTN_WEBSITE_URL), the sitemap and robots.txt are generated
|
||||
at runtime from the station config — one static write per startup,
|
||||
plus an optional admin-triggered regeneration.
|
||||
|
||||
Follows the same pattern as theme_export.py:
|
||||
- Atomic write (tmp + os.replace)
|
||||
- Written to STATIC_FILES_DIR so Nginx serves them
|
||||
- Uses WEBSITE_URL for per-tenant domain resolution
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Public routes included in the sitemap with their change frequency and priority.
|
||||
SEOMAP_ROUTES = [
|
||||
{"path": "/", "changefreq": "daily", "priority": 1.0},
|
||||
{"path": "/about", "changefreq": "weekly", "priority": 0.8},
|
||||
{"path": "/schedule", "changefreq": "weekly", "priority": 0.8},
|
||||
{"path": "/events", "changefreq": "daily", "priority": 0.8},
|
||||
{"path": "/contact", "changefreq": "monthly", "priority": 0.5},
|
||||
]
|
||||
|
||||
|
||||
def build_sitemap_xml(base_url: str) -> str:
|
||||
"""Build a valid sitemap.xml string for the given base URL."""
|
||||
url = base_url.rstrip("/")
|
||||
entries = []
|
||||
for route in SEOMAP_ROUTES:
|
||||
entries.append(
|
||||
f" <url>\n"
|
||||
f" <loc>{url}{route['path']}</loc>\n"
|
||||
f" <changefreq>{route['changefreq']}</changefreq>\n"
|
||||
f" <priority>{route['priority']}</priority>\n"
|
||||
f" </url>"
|
||||
)
|
||||
return (
|
||||
'<?xml version="1.0" encoding="UTF-8"?>\n'
|
||||
'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n'
|
||||
+ "\n".join(entries) + "\n"
|
||||
"</urlset>\n"
|
||||
)
|
||||
|
||||
|
||||
def build_robots_txt(sitemap_url: str) -> str:
|
||||
"""Build a robots.txt string that references the correct sitemap URL."""
|
||||
return (
|
||||
"User-agent: *\n"
|
||||
"Allow: /\n"
|
||||
"\n"
|
||||
"Disallow: /admin\n"
|
||||
"Disallow: /login\n"
|
||||
"\n"
|
||||
f"Sitemap: {sitemap_url}/sitemap.xml\n"
|
||||
)
|
||||
|
||||
|
||||
def get_sitemap_status() -> dict:
|
||||
"""Check if sitemap.xml and robots.txt exist on disk and return metadata."""
|
||||
base = Path(settings.STATIC_FILES_DIR)
|
||||
result = {}
|
||||
for name in ("sitemap.xml", "robots.txt"):
|
||||
path = base / name
|
||||
if not path.exists():
|
||||
result[name] = {"exists": False, "path": str(path)}
|
||||
continue
|
||||
stat = path.stat()
|
||||
result[name] = {
|
||||
"exists": True,
|
||||
"path": str(path),
|
||||
"size": stat.st_size,
|
||||
"modified": datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat(),
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def write_sitemap_files() -> dict:
|
||||
"""Write sitemap.xml and robots.txt to the static files directory.
|
||||
|
||||
Uses atomic write (write to .tmp, then os.replace) to prevent
|
||||
Nginx from serving a partially-written file.
|
||||
|
||||
Returns a status dict with file paths.
|
||||
"""
|
||||
base_url = settings.WEBSITE_URL
|
||||
static_dir = Path(settings.STATIC_FILES_DIR)
|
||||
static_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# sitemap.xml
|
||||
sitemap_content = build_sitemap_xml(base_url)
|
||||
sitemap_path = static_dir / "sitemap.xml"
|
||||
tmp_path = sitemap_path.with_suffix(".tmp")
|
||||
tmp_path.write_text(sitemap_content, encoding="utf-8")
|
||||
tmp_path.replace(sitemap_path)
|
||||
logger.info("Wrote sitemap.xml (%d bytes) to %s, base_url=%s", len(sitemap_content), sitemap_path, base_url)
|
||||
|
||||
# robots.txt
|
||||
robots_content = build_robots_txt(base_url)
|
||||
robots_path = static_dir / "robots.txt"
|
||||
tmp_path = robots_path.with_suffix(".tmp")
|
||||
tmp_path.write_text(robots_content, encoding="utf-8")
|
||||
tmp_path.replace(robots_path)
|
||||
logger.info("Wrote robots.txt (%d bytes) to %s", len(robots_content), robots_path)
|
||||
|
||||
return {
|
||||
"sitemap.xml": str(sitemap_path),
|
||||
"robots.txt": str(robots_path),
|
||||
"base_url": base_url,
|
||||
}
|
||||
Reference in new issue
Block a user