From 5484c761b4a7206ce33fc7d181ac9a9750168bf2 Mon Sep 17 00:00:00 2001 From: Hermes Robot Date: Tue, 4 Aug 2026 19:54:48 +0000 Subject: [PATCH] =?UTF-8?q?fix:=20address=20reviewer=20feedback=20?= =?UTF-8?q?=E2=80=94=20dynamic=20sitemap/robots=20+=20wildcard=20CI=20bran?= =?UTF-8?q?ches?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - CI workflow: replace hardcoded branch list with ['**'] wildcard so pipeline runs on every branch push, not just 4 named branches. - Sitemap/robots.txt: moved from static files to dynamic generation. New sitemap_export.py module (follows theme_export.py pattern) uses KMTN_WEBSITE_URL to generate tenant-aware sitemap.xml and robots.txt on app startup. Atomic writes to STATIC_FILES_DIR so Nginx serves them. - Added /api/sitemap endpoint with /status (public) and /regenerate (admin-only) for runtime control. Addresses reviewer comments on PR #4: 1. 'more general way to ensure this runs on any branch on push' 2. 'whitelabel app - this file may need to be dynamically generated' --- .gitea/workflows/ci.yaml | 2 +- backend/app/api/sitemap.py | 38 +++++++++++ backend/app/main.py | 11 +++- backend/app/sitemap_export.py | 116 ++++++++++++++++++++++++++++++++++ public/robots.txt | 7 -- public/sitemap.xml | 28 -------- 6 files changed, 165 insertions(+), 37 deletions(-) create mode 100644 backend/app/api/sitemap.py create mode 100644 backend/app/sitemap_export.py delete mode 100644 public/robots.txt delete mode 100644 public/sitemap.xml diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 68c9a5c..5fe1d3e 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -2,7 +2,7 @@ name: CI Pipeline on: push: - branches: [main, develop, feature/tests-and-ci, feature/multi-admin-roles, fix/seo-improvements] + branches: ['**'] pull_request: branches: [main] diff --git a/backend/app/api/sitemap.py b/backend/app/api/sitemap.py new file mode 100644 index 0000000..34b315e --- /dev/null +++ b/backend/app/api/sitemap.py @@ -0,0 +1,38 @@ +"""Sitemap & robots.txt API: status check, regenerate endpoint. + +Follows the same pattern as the theme API — public GET for status, +admin-only POST to force regeneration. +""" + +from typing import Any, Dict + +from fastapi import APIRouter, Depends, HTTPException, status + +from app.auth import get_current_admin_user +from app.sitemap_export import get_sitemap_status, write_sitemap_files +from app.user_models import User + +router = APIRouter() + + +@router.get("/status") +async def sitemap_status() -> Dict[str, Any]: + """Return sitemap/robots file status. Public endpoint.""" + return get_sitemap_status() + + +@router.post("/regenerate", status_code=200) +async def regenerate_sitemap( + _: User = Depends(get_current_admin_user), +) -> Dict[str, Any]: + """Force regenerate sitemap.xml and robots.txt. Admin only. + + Returns the paths of the newly written files. + """ + try: + return write_sitemap_files() + except Exception as e: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to generate sitemap: {e}", + ) diff --git a/backend/app/main.py b/backend/app/main.py index a551719..2485dfd 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -10,7 +10,7 @@ from app.config import settings from app.database import async_session, engine from app.models import Base, Show, StationConfig, HistoryEntry, TeamMember, CommunityHighlight, Underwriter from app.user_models import User -from app.api import events, auth, station_config, history, team, community, shows, storage, underwriters, stats, mobile_builds, theme +from app.api import events, auth, station_config, history, team, community, shows, storage, underwriters, stats, mobile_builds, theme, sitemap def _cleanup_orphaned_sequences(sync_conn): @@ -183,6 +183,14 @@ async def lifespan(app: FastAPI): except Exception as e: print(f" WARNING: Failed to write theme.json on startup: {e}") + # Write sitemap.xml + robots.txt on startup (tenant-aware) + try: + from app.sitemap_export import write_sitemap_files + info = write_sitemap_files() + print(f" ✓ Sitemap + robots.txt written (base_url={info['base_url']})") + except Exception as e: + print(f" WARNING: Failed to write sitemap on startup: {e}") + # Check GeoLite2 database geo_db = settings.GEOLITE2_DB_PATH if os.path.exists(geo_db): @@ -235,6 +243,7 @@ app.include_router(underwriters.router, prefix="/api/underwriters", tags=["under app.include_router(stats.router, prefix="/api/stats", tags=["stats"]) app.include_router(mobile_builds.router, prefix="/api/mobile-builds", tags=["mobile-builds"]) app.include_router(theme.router, prefix="/api/theme", tags=["theme"]) +app.include_router(sitemap.router, prefix="/api/sitemap", tags=["sitemap"]) # Admin user management (available in all environments) from app.api import users diff --git a/backend/app/sitemap_export.py b/backend/app/sitemap_export.py new file mode 100644 index 0000000..7661921 --- /dev/null +++ b/backend/app/sitemap_export.py @@ -0,0 +1,116 @@ +"""Sitemap & robots.txt export: generate tenant-aware SEO files. + +Because this is a white-label app where each tenant has its own domain +(set via KMTN_WEBSITE_URL), the sitemap and robots.txt are generated +at runtime from the station config — one static write per startup, +plus an optional admin-triggered regeneration. + +Follows the same pattern as theme_export.py: +- Atomic write (tmp + os.replace) +- Written to STATIC_FILES_DIR so Nginx serves them +- Uses WEBSITE_URL for per-tenant domain resolution +""" + +import logging +from datetime import datetime, timezone +from pathlib import Path + +from app.config import settings + +logger = logging.getLogger(__name__) + +# Public routes included in the sitemap with their change frequency and priority. +SEOMAP_ROUTES = [ + {"path": "/", "changefreq": "daily", "priority": 1.0}, + {"path": "/about", "changefreq": "weekly", "priority": 0.8}, + {"path": "/schedule", "changefreq": "weekly", "priority": 0.8}, + {"path": "/events", "changefreq": "daily", "priority": 0.8}, + {"path": "/contact", "changefreq": "monthly", "priority": 0.5}, +] + + +def build_sitemap_xml(base_url: str) -> str: + """Build a valid sitemap.xml string for the given base URL.""" + url = base_url.rstrip("/") + entries = [] + for route in SEOMAP_ROUTES: + entries.append( + f" \n" + f" {url}{route['path']}\n" + f" {route['changefreq']}\n" + f" {route['priority']}\n" + f" " + ) + return ( + '\n' + '\n' + + "\n".join(entries) + "\n" + "\n" + ) + + +def build_robots_txt(sitemap_url: str) -> str: + """Build a robots.txt string that references the correct sitemap URL.""" + return ( + "User-agent: *\n" + "Allow: /\n" + "\n" + "Disallow: /admin\n" + "Disallow: /login\n" + "\n" + f"Sitemap: {sitemap_url}/sitemap.xml\n" + ) + + +def get_sitemap_status() -> dict: + """Check if sitemap.xml and robots.txt exist on disk and return metadata.""" + base = Path(settings.STATIC_FILES_DIR) + result = {} + for name in ("sitemap.xml", "robots.txt"): + path = base / name + if not path.exists(): + result[name] = {"exists": False, "path": str(path)} + continue + stat = path.stat() + result[name] = { + "exists": True, + "path": str(path), + "size": stat.st_size, + "modified": datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat(), + } + return result + + +def write_sitemap_files() -> dict: + """Write sitemap.xml and robots.txt to the static files directory. + + Uses atomic write (write to .tmp, then os.replace) to prevent + Nginx from serving a partially-written file. + + Returns a status dict with file paths. + """ + base_url = settings.WEBSITE_URL + static_dir = Path(settings.STATIC_FILES_DIR) + static_dir.mkdir(parents=True, exist_ok=True) + + # sitemap.xml + sitemap_content = build_sitemap_xml(base_url) + sitemap_path = static_dir / "sitemap.xml" + tmp_path = sitemap_path.with_suffix(".tmp") + tmp_path.write_text(sitemap_content, encoding="utf-8") + tmp_path.replace(sitemap_path) + logger.info("Wrote sitemap.xml (%d bytes) to %s, base_url=%s", len(sitemap_content), sitemap_path, base_url) + + # robots.txt + robots_content = build_robots_txt(base_url) + robots_path = static_dir / "robots.txt" + tmp_path = robots_path.with_suffix(".tmp") + tmp_path.write_text(robots_content, encoding="utf-8") + tmp_path.replace(robots_path) + logger.info("Wrote robots.txt (%d bytes) to %s", len(robots_content), robots_path) + + return { + "sitemap.xml": str(sitemap_path), + "robots.txt": str(robots_path), + "base_url": base_url, + } diff --git a/public/robots.txt b/public/robots.txt deleted file mode 100644 index 27c712f..0000000 --- a/public/robots.txt +++ /dev/null @@ -1,7 +0,0 @@ -User-agent: * -Allow: / - -Disallow: /admin -Disallow: /login - -Sitemap: https://kmountainflower.org/sitemap.xml diff --git a/public/sitemap.xml b/public/sitemap.xml deleted file mode 100644 index d7169f9..0000000 --- a/public/sitemap.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - - https://kmountainflower.org/ - daily - 1.0 - - - https://kmountainflower.org/about - weekly - 0.8 - - - https://kmountainflower.org/schedule - weekly - 0.8 - - - https://kmountainflower.org/events - daily - 0.8 - - - https://kmountainflower.org/contact - monthly - 0.5 - -