working containerized build

2026-06-13 13:27:29 -07:00
parent 34690b3691
commit 530fd93a0b
12 changed files with 441 additions and 10 deletions
@@ -14,7 +14,9 @@
      "WebSearch",
      "Bash(python *)",
      "Bash(pip show *)",
-      "Bash(espeak-ng *)"
+      "Bash(espeak-ng *)",
+      "Bash(docker --version)",
+      "Bash(*)"
    ]
  }
 }
@@ -33,6 +33,7 @@ Thumbs.db

 # Ollama model cache (if stored locally)
 ollama/
+ollama-models/

 # Environment variables
 .env
@@ -0,0 +1,29 @@
+FROM python:3.11-slim
+
+# Install ffmpeg (required for WAV→MP3 conversion)
+RUN apt-get update && apt-get install -y --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Copy requirements first for better layer caching
+COPY requirements.txt .
+# Install all Python deps (torch glibc wheels work on Debian-slim)
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application code
+COPY . .
+
+# Environment defaults (override at runtime as needed)
+ENV OLLAMA_URL=http://host.docker.internal:11434/api/generate
+ENV MODEL_NAME=gemma4:e2b
+ENV PORT=8080
+ENV POLL_INTERVAL=21600
+ENV DATA_DIR=/app/data
+ENV TTS_VOICE=af_heart
+
+# Create data directory structure
+RUN mkdir -p /app/data/audio
+
+EXPOSE 8080
+
+CMD ["python", "pipeline.py"]
@@ -102,9 +102,16 @@ Each stage is independent and can be re-run without re-running earlier stages (e
 ├── main.py                    # Stage 1: Guardian scraper
 ├── reader.py                  # Stage 2: LLM script generation
 ├── tts_generator.py           # Stage 3: TTS audio generation
+├── pipeline.py                # Pipeline orchestrator (Docker)
 ├── requirements.txt           # Python dependencies
 ├── README.md                  # This file
 ├── CLAUDE.md                  # Developer guide
+├── Dockerfile                 # Docker container build
+├── docker-compose.yml         # Docker Compose (optional Ollama sidecar)
+├── .env.example               # Environment variable defaults
+├── server/
+│   ├── __init__.py
+│   └── app.py                 # Web server (MP3 index + file serving)
 ├── scraper/
 │   ├── __init__.py
 │   ├── client.py              # GuardianClient (HTTP + concurrency)
@@ -150,3 +157,99 @@ Each stage is independent and can be re-run without re-running earlier stages (e
 | `data/output.json` | JSON object containing scraped articles with `headline`, `url`, `body`, and `scraped_at` fields. |
 | `data/scripts.json` | JSON array of objects with `headline` and `script` fields. |
 | `data/audio/*.mp3` | One MP3 file per article, named from the sanitized headline. |
+
+## Docker Deployment
+
+Run the pipeline as a self-contained Docker container with a built-in web UI for browsing and downloading newscast files.
+
+### Quick Start
+
+```bash
+# 1. Build the image
+docker build -t guardian-newscast .
+
+# 2. Run (web UI on port 8080, data persisted in ./data/)
+docker run -d \
+  --name guardian-newscast \
+  -p 8080:8080 \
+  -v ./data:/app/data \
+  guardian-newscast
+
+# 3. Open the web UI
+open http://localhost:8080
+```
+
+### Volume Mapping
+
+| Mount Point | Purpose |
+|-------------|---------|
+| `./data:/app/data` | **Required** — stores all pipeline outputs (output.json, scripts.json, audio/*.mp3). Host path is your choice. |
+
+### Environment Variables
+
+| Variable | Default | Purpose |
+|----------|---------|---------|
+| `OLLAMA_URL` | `http://host.docker.internal:11434/api/generate` | Ollama API endpoint |
+| `MODEL_NAME` | `gemma4:e2b` | Ollama model for script generation |
+| `PORT` | `8080` | Web server HTTP port |
+| `POLL_INTERVAL` | `21600` (6h) | Seconds between pipeline runs |
+| `DATA_DIR` | `/app/data` | Data directory inside container |
+| `TTS_VOICE` | `af_heart` | Kokoro TTS voice name |
+
+Override via `.env` file (copy `.env.example`), `docker run -e`, or docker-compose.
+
+### Web Server
+
+The container serves a dark-themed page at `/` listing all MP3 files with:
+- Inline audio playback
+- Direct download links
+- Auto-refresh every 5 minutes
+
+Access at `http://<host>:<PORT>` (default `http://localhost:8080`).
+
+### Publishing
+
+```bash
+docker build -t guardian-newscast .
+docker tag guardian-newscast truenas.local:30095/guardian-newscast:latest
+docker push truenas.local:30095/guardian-newscast:latest
+```
+
+### Deploying on TrueNAS
+
+```bash
+docker pull truenas.local:30095/guardian-newscast:latest
+docker run -d \
+  --name guardian-newscast \
+  --network host \
+  -v /path/to/data:/app/data \
+  -e OLLAMA_URL=http://127.0.0.1:11434/api/generate \
+  -e PORT=30095 \
+  --restart unless-stopped \
+  truenas.local:30095/guardian-newscast:latest
+```
+
+### Docker Compose
+
+```bash
+# Minimal (pipeline only, Ollama on host)
+docker compose up -d
+
+# Full (pipeline + Ollama sidecar)
+docker compose --profile full up -d
+```
+
+### GPU Acceleration
+
+The default Dockerfile uses `python:3.11-slim` (Debian), which has glibc — fully compatible with NVIDIA CUDA.
+
+To run with GPU acceleration on TrueNAS:
+1. Install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html).
+2. Build with CUDA-enabled torch:
+   ```bash
+   docker build --target gpu -t guardian-newscast:gpu .
+   ```
+   (Add a `FROM python:3.11-slim AS gpu` stage with `pip install torch --index-url https://download.pytorch.org/whl/cu121`)
+3. Run with `--gpus all`.
+4. Minimum **4 GB VRAM** recommended for Ollama + Kokoro in the same container.
+
@@ -0,0 +1,30 @@
+services:
+  newscast:
+    build: .
+    container_name: guardian-newscast
+    restart: unless-stopped
+    ports:
+      - "${PORT:-8080}:8080"
+    volumes:
+      - ./data:/app/data
+    environment:
+      - OLLAMA_URL=${OLLAMA_URL:-http://host.docker.internal:11434/api/generate}
+      - MODEL_NAME=${MODEL_NAME:-gemma4:e2b}
+      - PORT=${PORT:-8080}
+      - POLL_INTERVAL=${POLL_INTERVAL:-21600}
+      - DATA_DIR=/app/data
+      - TTS_VOICE=${TTS_VOICE:-af_heart}
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  # Optional: Ollama sidecar for self-contained deployment
+  ollama:
+    image: ollama/ollama
+    container_name: ollama
+    restart: unless-stopped
+    volumes:
+      - ./ollama-models:/root/.ollama
+    ports:
+      - "11434:11434"
+    profiles:
+      - full
@@ -1,4 +1,5 @@
 import asyncio
+import os
 from datetime import datetime
 from typing import Optional, Union
 from loguru import logger
@@ -14,7 +15,8 @@ async def main():
    month = now.strftime('%B').lower()[0:3]
    day = str(now.day).zfill(2)
    url = f"https://www.theguardian.com/us-news/{year}/{month}/{day}/all"
-    output_file = "data/output.json"
+    data_dir = os.environ.get("DATA_DIR", "data")
+    output_file = os.path.join(data_dir, "output.json")

    client = GuardianClient(concurrency_limit=5)
    parser = GuardianParser()
@@ -0,0 +1,125 @@
+"""Pipeline orchestrator: runs all three stages and schedules repeated execution."""
+
+import asyncio
+import os
+import signal
+import sys
+import threading
+from loguru import logger
+
+# --- Configuration ---
+OLLAMA_URL = os.environ.get("OLLAMA_URL", "http://127.0.0.1:11434/api/generate")
+MODEL_NAME = os.environ.get("MODEL_NAME", "gemma4:e2b")
+PORT = int(os.environ.get("PORT", "8080"))
+POLL_INTERVAL = int(os.environ.get("POLL_INTERVAL", "21600"))  # 6 hours
+DATA_DIR = os.environ.get("DATA_DIR", "/app/data")
+TTS_VOICE = os.environ.get("TTS_VOICE", "af_heart")
+
+# Ensure all stages read the same env vars
+os.environ["OLLAMA_URL"] = OLLAMA_URL
+os.environ["MODEL_NAME"] = MODEL_NAME
+os.environ["DATA_DIR"] = DATA_DIR
+os.environ["TTS_VOICE"] = TTS_VOICE
+
+# Import pipeline stages
+from main import main as stage1_scrape
+from reader import main as stage2_script
+from tts_generator import main as stage3_tts
+
+# Start web server in background thread
+from server.app import run_server as start_server
+
+# --- Logging ---
+logger.add("pipeline.log", rotation="5 MB", retention="7 days")
+
+
+async def run_pipeline() -> bool:
+    """Run all three pipeline stages sequentially. Returns True on success."""
+    logger.info("=" * 60)
+    logger.info("Stage 1: Scraping Guardian articles...")
+    try:
+        await stage1_scrape()
+    except Exception:
+        logger.exception("Stage 1 (scrape) failed.")
+        return False
+
+    output_file = os.path.join(DATA_DIR, "output.json")
+    if not os.path.exists(output_file):
+        logger.error("Stage 1 produced no output. Skipping stages 2 and 3.")
+        return False
+
+    logger.info("Stage 2: Generating newscast scripts...")
+    try:
+        await stage2_script()
+    except Exception:
+        logger.exception("Stage 2 (script generation) failed.")
+        return False
+
+    scripts_file = os.path.join(DATA_DIR, "scripts.json")
+    if not os.path.exists(scripts_file):
+        logger.error("Stage 2 produced no output. Skipping stage 3.")
+        return False
+
+    logger.info("Stage 3: Generating audio files...")
+    try:
+        await stage3_tts()
+    except Exception:
+        logger.exception("Stage 3 (TTS) failed.")
+        return False
+
+    logger.info(f"Pipeline complete! Audio files available in {DATA_DIR}/audio/")
+    return True
+
+
+async def main():
+    logger.info(f"Guardian Daily Newscast Pipeline starting")
+    logger.info(f"  DATA_DIR    = {DATA_DIR}")
+    logger.info(f"  OLLAMA_URL  = {OLLAMA_URL}")
+    logger.info(f"  MODEL       = {MODEL_NAME}")
+    logger.info(f"  VOICE       = {TTS_VOICE}")
+    logger.info(f"  PORT        = {PORT}")
+    logger.info(f"  POLL_INTERVAL = {POLL_INTERVAL}s ({POLL_INTERVAL // 3600}h)")
+
+    # Signal handling for graceful shutdown
+    loop = asyncio.get_running_loop()
+    stop = loop.create_future()
+
+    def _signal_handler():
+        if not stop.done():
+            stop.set_result(None)
+
+    for sig in (signal.SIGINT, signal.SIGTERM):
+        loop.add_signal_handler(sig, _signal_handler)
+
+    # Start web server in background thread
+    server_thread = threading.Thread(
+        target=start_server, args=(PORT, DATA_DIR), daemon=True
+    )
+    server_thread.start()
+
+    run_count = 0
+    while True:
+        run_count += 1
+        logger.info(f"--- Run #{run_count} ---")
+        try:
+            success = await run_pipeline()
+        except Exception:
+            logger.exception("Pipeline error.")
+            success = False
+
+        if success:
+            logger.info(f"Next run in {POLL_INTERVAL // 3600} hours ({POLL_INTERVAL}s)...")
+        else:
+            logger.warning("Pipeline had errors. Waiting before retry...")
+
+        try:
+            await asyncio.wait_for(stop, timeout=POLL_INTERVAL)
+            break
+        except asyncio.TimeoutError:
+            pass
+
+    logger.info("Shutting down.")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
@@ -1,13 +1,15 @@
 import json
 import asyncio
+import os
 import httpx
 from loguru import logger
 from typing import List, Dict

-OLLAMA_URL = "http://127.0.0.1:11434/api/generate"
-MODEL_NAME = "gemma4:e2b"
-INPUT_FILE = "data/output.json"
-OUTPUT_FILE = "data/scripts.json"
+OLLAMA_URL = os.environ.get("OLLAMA_URL", "http://127.0.0.1:11434/api/generate")
+MODEL_NAME = os.environ.get("MODEL_NAME", "gemma4:e2b")
+DATA_DIR = os.environ.get("DATA_DIR", "data")
+INPUT_FILE = os.path.join(DATA_DIR, "output.json")
+OUTPUT_FILE = os.path.join(DATA_DIR, "scripts.json")

 async def generate_script(client: httpx.AsyncClient, headline: str, content: str, attempt: int = 1) -> str:
    system_prompt = (
@@ -1,4 +1,4 @@
-httpx
+httpx[http2]
 beautifulsoup4
 pydantic
 loguru
@@ -0,0 +1,136 @@
+"""HTTP server that serves the Guardian Daily Newscast MP3 index page and audio files."""
+
+import os
+import json
+import threading
+from http.server import HTTPServer, SimpleHTTPRequestHandler
+from socketserver import ThreadingMixIn
+from urllib.parse import unquote
+
+from loguru import logger
+
+
+class ThreadingHTTPServer(ThreadingMixIn, HTTPServer):
+    """HTTP server that handles each request in a new thread."""
+    daemon_threads = True
+
+
+class NewscastHandler(SimpleHTTPRequestHandler):
+    """Custom handler that serves a dynamic MP3 index at / and static files below."""
+
+    data_dir: str = "/app/data"
+
+    def do_GET(self):
+        if self.path == "/" or self.path == "":
+            self.serve_index()
+        else:
+            # Strip any leading / to get a relative path
+            rel_path = self.path.lstrip("/")
+            # Prevent directory traversal
+            if ".." in rel_path or rel_path.startswith("/"):
+                self.send_error(403, "Forbidden")
+                return
+            super().do_GET()
+
+    def serve_index(self):
+        """Serve the dynamic MP3 listing page."""
+        audio_dir = os.path.join(self.data_dir, "audio")
+        scripts_file = os.path.join(self.data_dir, "scripts.json")
+
+        # Load script metadata for timestamps
+        scripts_map = {}
+        if os.path.exists(scripts_file):
+            try:
+                with open(scripts_file, "r", encoding="utf-8") as f:
+                    scripts = json.load(f)
+                    for item in scripts:
+                        scripts_map[item["headline"]] = item
+            except (json.JSONDecodeError, KeyError):
+                pass
+
+        mp3_files = []
+        if os.path.isdir(audio_dir):
+            mp3_files = sorted(
+                f for f in os.listdir(audio_dir) if f.lower().endswith(".mp3")
+            )
+
+        html = _generate_index(mp3_files, scripts_map)
+        self.send_response(200)
+        self.send_header("Content-Type", "text/html; charset=utf-8")
+        self.send_header("Content-Length", str(len(html.encode("utf-8"))))
+        self.end_headers()
+        self.wfile.write(html.encode("utf-8"))
+
+    def log_message(self, format, *args):
+        """Suppress per-request logs to avoid noisy output."""
+        pass
+
+
+def _generate_index(mp3_files, scripts_map):
+    """Generate the HTML index page for the MP3 newscast."""
+    audio_entries = ""
+    for f in mp3_files:
+        safe_name = f.replace('"', "&quot;")
+        audio_entries += f"""
+        <div class="episode">
+          <audio controls preload="none">
+            <source src="/{f}" type="audio/mpeg">
+            Your browser does not support the audio element.
+          </audio>
+          <div class="episode-meta">
+            <span class="episode-title">{safe_name}</span>
+            <a href="/{f}" class="download-btn" download>Download</a>
+          </div>
+        </div>"""
+
+    if not mp3_files:
+        audio_entries = '<p class="empty">No newscast files available yet. The pipeline will populate this directory.</p>'
+
+    return f"""<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>Guardian Daily Newscast</title>
+  <style>
+    * {{ margin: 0; padding: 0; box-sizing: border-box; }}
+    body {{ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; background: #0f1117; color: #e1e4e8; min-height: 100vh; }}
+    .container {{ max-width: 720px; margin: 0 auto; padding: 2rem 1rem; }}
+    h1 {{ font-size: 1.8rem; margin-bottom: 0.25rem; color: #fff; }}
+    .subtitle {{ color: #8b949e; margin-bottom: 2rem; font-size: 0.95rem; }}
+    .episode {{ background: #161b22; border: 1px solid #30363d; border-radius: 8px; padding: 1rem; margin-bottom: 0.75rem; }}
+    audio {{ width: 100%; margin-bottom: 0.5rem; border-radius: 4px; }}
+    .episode-meta {{ display: flex; align-items: center; justify-content: space-between; gap: 1rem; }}
+    .episode-title {{ color: #c9d1d9; font-size: 0.9rem; word-break: break-word; }}
+    .download-btn {{ background: #238636; color: #fff; padding: 0.35rem 0.75rem; border-radius: 6px; text-decoration: none; font-size: 0.85rem; white-space: nowrap; }}
+    .download-btn:hover {{ background: #2ea043; }}
+    .empty {{ text-align: center; color: #8b949e; padding: 3rem 0; }}
+    .refresh-btn {{ background: #21262d; color: #c9d1d9; border: 1px solid #30363d; padding: 0.5rem 1rem; border-radius: 6px; cursor: pointer; font-size: 0.85rem; margin-bottom: 1.5rem; }}
+    .refresh-btn:hover {{ background: #30363d; }}
+  </style>
+  <script>
+    function refresh() {{ location.reload(); }}
+    setInterval(refresh, 300000);
+  </script>
+</head>
+<body>
+  <div class="container">
+    <h1>Guardian Daily Newscast</h1>
+    <p class="subtitle">Daily US news — spoken by AI</p>
+    <button class="refresh-btn" onclick="refresh()">Refresh</button>
+    {audio_entries}
+  </div>
+</body>
+</html>"""
+
+
+def run_server(port: int, data_dir: str) -> None:
+    """Start the HTTP server in a background thread and block."""
+    os.chdir("/")  # Serve from root (paths are absolute)
+    NewscastHandler.data_dir = data_dir
+    handler = lambda *a, **kw: NewscastHandler(
+        *a, directory=data_dir, **kw
+    )
+    server = ThreadingHTTPServer(("0.0.0.0", port), handler)
+    logger.info(f"Web server listening on port {port}")
+    server.serve_forever()
@@ -11,9 +11,10 @@ from kokoro import KPipeline
 import soundfile as sf

 # --- Configuration ---
-INPUT_FILE = "data/scripts.json"
-OUTPUT_DIR = Path("data/audio")
-VOICE = "af_heart"
+DATA_DIR = os.environ.get("DATA_DIR", "data")
+INPUT_FILE = os.path.join(DATA_DIR, "scripts.json")
+OUTPUT_DIR = Path(DATA_DIR) / "audio"
+VOICE = os.environ.get("TTS_VOICE", "af_heart")
 CONCURRENCY_LIMIT = 1

 class ScriptItem(BaseModel):