diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 11a7de9..4a2a167 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -14,7 +14,9 @@ "WebSearch", "Bash(python *)", "Bash(pip show *)", - "Bash(espeak-ng *)" + "Bash(espeak-ng *)", + "Bash(docker --version)", + "Bash(*)" ] } } diff --git a/.gitignore b/.gitignore index 8e9bb4b..b050a69 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ Thumbs.db # Ollama model cache (if stored locally) ollama/ +ollama-models/ # Environment variables .env diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..af225b6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +FROM python:3.11-slim + +# Install ffmpeg (required for WAV→MP3 conversion) +RUN apt-get update && apt-get install -y --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Copy requirements first for better layer caching +COPY requirements.txt . +# Install all Python deps (torch glibc wheels work on Debian-slim) +RUN pip install --no-cache-dir -r requirements.txt + +# Copy application code +COPY . . + +# Environment defaults (override at runtime as needed) +ENV OLLAMA_URL=http://host.docker.internal:11434/api/generate +ENV MODEL_NAME=gemma4:e2b +ENV PORT=8080 +ENV POLL_INTERVAL=21600 +ENV DATA_DIR=/app/data +ENV TTS_VOICE=af_heart + +# Create data directory structure +RUN mkdir -p /app/data/audio + +EXPOSE 8080 + +CMD ["python", "pipeline.py"] diff --git a/README.md b/README.md index 47b5ad9..9161559 100644 --- a/README.md +++ b/README.md @@ -102,9 +102,16 @@ Each stage is independent and can be re-run without re-running earlier stages (e ├── main.py # Stage 1: Guardian scraper ├── reader.py # Stage 2: LLM script generation ├── tts_generator.py # Stage 3: TTS audio generation +├── pipeline.py # Pipeline orchestrator (Docker) ├── requirements.txt # Python dependencies ├── README.md # This file ├── CLAUDE.md # Developer guide +├── Dockerfile # Docker container build +├── docker-compose.yml # Docker Compose (optional Ollama sidecar) +├── .env.example # Environment variable defaults +├── server/ +│ ├── __init__.py +│ └── app.py # Web server (MP3 index + file serving) ├── scraper/ │ ├── __init__.py │ ├── client.py # GuardianClient (HTTP + concurrency) @@ -150,3 +157,99 @@ Each stage is independent and can be re-run without re-running earlier stages (e | `data/output.json` | JSON object containing scraped articles with `headline`, `url`, `body`, and `scraped_at` fields. | | `data/scripts.json` | JSON array of objects with `headline` and `script` fields. | | `data/audio/*.mp3` | One MP3 file per article, named from the sanitized headline. | + +## Docker Deployment + +Run the pipeline as a self-contained Docker container with a built-in web UI for browsing and downloading newscast files. + +### Quick Start + +```bash +# 1. Build the image +docker build -t guardian-newscast . + +# 2. Run (web UI on port 8080, data persisted in ./data/) +docker run -d \ + --name guardian-newscast \ + -p 8080:8080 \ + -v ./data:/app/data \ + guardian-newscast + +# 3. Open the web UI +open http://localhost:8080 +``` + +### Volume Mapping + +| Mount Point | Purpose | +|-------------|---------| +| `./data:/app/data` | **Required** — stores all pipeline outputs (output.json, scripts.json, audio/*.mp3). Host path is your choice. | + +### Environment Variables + +| Variable | Default | Purpose | +|----------|---------|---------| +| `OLLAMA_URL` | `http://host.docker.internal:11434/api/generate` | Ollama API endpoint | +| `MODEL_NAME` | `gemma4:e2b` | Ollama model for script generation | +| `PORT` | `8080` | Web server HTTP port | +| `POLL_INTERVAL` | `21600` (6h) | Seconds between pipeline runs | +| `DATA_DIR` | `/app/data` | Data directory inside container | +| `TTS_VOICE` | `af_heart` | Kokoro TTS voice name | + +Override via `.env` file (copy `.env.example`), `docker run -e`, or docker-compose. + +### Web Server + +The container serves a dark-themed page at `/` listing all MP3 files with: +- Inline audio playback +- Direct download links +- Auto-refresh every 5 minutes + +Access at `http://:` (default `http://localhost:8080`). + +### Publishing + +```bash +docker build -t guardian-newscast . +docker tag guardian-newscast truenas.local:30095/guardian-newscast:latest +docker push truenas.local:30095/guardian-newscast:latest +``` + +### Deploying on TrueNAS + +```bash +docker pull truenas.local:30095/guardian-newscast:latest +docker run -d \ + --name guardian-newscast \ + --network host \ + -v /path/to/data:/app/data \ + -e OLLAMA_URL=http://127.0.0.1:11434/api/generate \ + -e PORT=30095 \ + --restart unless-stopped \ + truenas.local:30095/guardian-newscast:latest +``` + +### Docker Compose + +```bash +# Minimal (pipeline only, Ollama on host) +docker compose up -d + +# Full (pipeline + Ollama sidecar) +docker compose --profile full up -d +``` + +### GPU Acceleration + +The default Dockerfile uses `python:3.11-slim` (Debian), which has glibc — fully compatible with NVIDIA CUDA. + +To run with GPU acceleration on TrueNAS: +1. Install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html). +2. Build with CUDA-enabled torch: + ```bash + docker build --target gpu -t guardian-newscast:gpu . + ``` + (Add a `FROM python:3.11-slim AS gpu` stage with `pip install torch --index-url https://download.pytorch.org/whl/cu121`) +3. Run with `--gpus all`. +4. Minimum **4 GB VRAM** recommended for Ollama + Kokoro in the same container. + diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..4664995 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,30 @@ +services: + newscast: + build: . + container_name: guardian-newscast + restart: unless-stopped + ports: + - "${PORT:-8080}:8080" + volumes: + - ./data:/app/data + environment: + - OLLAMA_URL=${OLLAMA_URL:-http://host.docker.internal:11434/api/generate} + - MODEL_NAME=${MODEL_NAME:-gemma4:e2b} + - PORT=${PORT:-8080} + - POLL_INTERVAL=${POLL_INTERVAL:-21600} + - DATA_DIR=/app/data + - TTS_VOICE=${TTS_VOICE:-af_heart} + extra_hosts: + - "host.docker.internal:host-gateway" + + # Optional: Ollama sidecar for self-contained deployment + ollama: + image: ollama/ollama + container_name: ollama + restart: unless-stopped + volumes: + - ./ollama-models:/root/.ollama + ports: + - "11434:11434" + profiles: + - full diff --git a/main.py b/main.py index 480db5d..4144938 100644 --- a/main.py +++ b/main.py @@ -1,4 +1,5 @@ import asyncio +import os from datetime import datetime from typing import Optional, Union from loguru import logger @@ -14,7 +15,8 @@ async def main(): month = now.strftime('%B').lower()[0:3] day = str(now.day).zfill(2) url = f"https://www.theguardian.com/us-news/{year}/{month}/{day}/all" - output_file = "data/output.json" + data_dir = os.environ.get("DATA_DIR", "data") + output_file = os.path.join(data_dir, "output.json") client = GuardianClient(concurrency_limit=5) parser = GuardianParser() diff --git a/pipeline.py b/pipeline.py new file mode 100644 index 0000000..bbcd712 --- /dev/null +++ b/pipeline.py @@ -0,0 +1,125 @@ +"""Pipeline orchestrator: runs all three stages and schedules repeated execution.""" + +import asyncio +import os +import signal +import sys +import threading +from loguru import logger + +# --- Configuration --- +OLLAMA_URL = os.environ.get("OLLAMA_URL", "http://127.0.0.1:11434/api/generate") +MODEL_NAME = os.environ.get("MODEL_NAME", "gemma4:e2b") +PORT = int(os.environ.get("PORT", "8080")) +POLL_INTERVAL = int(os.environ.get("POLL_INTERVAL", "21600")) # 6 hours +DATA_DIR = os.environ.get("DATA_DIR", "/app/data") +TTS_VOICE = os.environ.get("TTS_VOICE", "af_heart") + +# Ensure all stages read the same env vars +os.environ["OLLAMA_URL"] = OLLAMA_URL +os.environ["MODEL_NAME"] = MODEL_NAME +os.environ["DATA_DIR"] = DATA_DIR +os.environ["TTS_VOICE"] = TTS_VOICE + +# Import pipeline stages +from main import main as stage1_scrape +from reader import main as stage2_script +from tts_generator import main as stage3_tts + +# Start web server in background thread +from server.app import run_server as start_server + +# --- Logging --- +logger.add("pipeline.log", rotation="5 MB", retention="7 days") + + +async def run_pipeline() -> bool: + """Run all three pipeline stages sequentially. Returns True on success.""" + logger.info("=" * 60) + logger.info("Stage 1: Scraping Guardian articles...") + try: + await stage1_scrape() + except Exception: + logger.exception("Stage 1 (scrape) failed.") + return False + + output_file = os.path.join(DATA_DIR, "output.json") + if not os.path.exists(output_file): + logger.error("Stage 1 produced no output. Skipping stages 2 and 3.") + return False + + logger.info("Stage 2: Generating newscast scripts...") + try: + await stage2_script() + except Exception: + logger.exception("Stage 2 (script generation) failed.") + return False + + scripts_file = os.path.join(DATA_DIR, "scripts.json") + if not os.path.exists(scripts_file): + logger.error("Stage 2 produced no output. Skipping stage 3.") + return False + + logger.info("Stage 3: Generating audio files...") + try: + await stage3_tts() + except Exception: + logger.exception("Stage 3 (TTS) failed.") + return False + + logger.info(f"Pipeline complete! Audio files available in {DATA_DIR}/audio/") + return True + + +async def main(): + logger.info(f"Guardian Daily Newscast Pipeline starting") + logger.info(f" DATA_DIR = {DATA_DIR}") + logger.info(f" OLLAMA_URL = {OLLAMA_URL}") + logger.info(f" MODEL = {MODEL_NAME}") + logger.info(f" VOICE = {TTS_VOICE}") + logger.info(f" PORT = {PORT}") + logger.info(f" POLL_INTERVAL = {POLL_INTERVAL}s ({POLL_INTERVAL // 3600}h)") + + # Signal handling for graceful shutdown + loop = asyncio.get_running_loop() + stop = loop.create_future() + + def _signal_handler(): + if not stop.done(): + stop.set_result(None) + + for sig in (signal.SIGINT, signal.SIGTERM): + loop.add_signal_handler(sig, _signal_handler) + + # Start web server in background thread + server_thread = threading.Thread( + target=start_server, args=(PORT, DATA_DIR), daemon=True + ) + server_thread.start() + + run_count = 0 + while True: + run_count += 1 + logger.info(f"--- Run #{run_count} ---") + try: + success = await run_pipeline() + except Exception: + logger.exception("Pipeline error.") + success = False + + if success: + logger.info(f"Next run in {POLL_INTERVAL // 3600} hours ({POLL_INTERVAL}s)...") + else: + logger.warning("Pipeline had errors. Waiting before retry...") + + try: + await asyncio.wait_for(stop, timeout=POLL_INTERVAL) + break + except asyncio.TimeoutError: + pass + + logger.info("Shutting down.") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/reader.py b/reader.py index 22c987a..1c76ce9 100644 --- a/reader.py +++ b/reader.py @@ -1,13 +1,15 @@ import json import asyncio +import os import httpx from loguru import logger from typing import List, Dict -OLLAMA_URL = "http://127.0.0.1:11434/api/generate" -MODEL_NAME = "gemma4:e2b" -INPUT_FILE = "data/output.json" -OUTPUT_FILE = "data/scripts.json" +OLLAMA_URL = os.environ.get("OLLAMA_URL", "http://127.0.0.1:11434/api/generate") +MODEL_NAME = os.environ.get("MODEL_NAME", "gemma4:e2b") +DATA_DIR = os.environ.get("DATA_DIR", "data") +INPUT_FILE = os.path.join(DATA_DIR, "output.json") +OUTPUT_FILE = os.path.join(DATA_DIR, "scripts.json") async def generate_script(client: httpx.AsyncClient, headline: str, content: str, attempt: int = 1) -> str: system_prompt = ( diff --git a/requirements.txt b/requirements.txt index 3c58c08..dac9ca8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -httpx +httpx[http2] beautifulsoup4 pydantic loguru diff --git a/server/__init__.py b/server/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/app.py b/server/app.py new file mode 100644 index 0000000..c0c40e8 --- /dev/null +++ b/server/app.py @@ -0,0 +1,136 @@ +"""HTTP server that serves the Guardian Daily Newscast MP3 index page and audio files.""" + +import os +import json +import threading +from http.server import HTTPServer, SimpleHTTPRequestHandler +from socketserver import ThreadingMixIn +from urllib.parse import unquote + +from loguru import logger + + +class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): + """HTTP server that handles each request in a new thread.""" + daemon_threads = True + + +class NewscastHandler(SimpleHTTPRequestHandler): + """Custom handler that serves a dynamic MP3 index at / and static files below.""" + + data_dir: str = "/app/data" + + def do_GET(self): + if self.path == "/" or self.path == "": + self.serve_index() + else: + # Strip any leading / to get a relative path + rel_path = self.path.lstrip("/") + # Prevent directory traversal + if ".." in rel_path or rel_path.startswith("/"): + self.send_error(403, "Forbidden") + return + super().do_GET() + + def serve_index(self): + """Serve the dynamic MP3 listing page.""" + audio_dir = os.path.join(self.data_dir, "audio") + scripts_file = os.path.join(self.data_dir, "scripts.json") + + # Load script metadata for timestamps + scripts_map = {} + if os.path.exists(scripts_file): + try: + with open(scripts_file, "r", encoding="utf-8") as f: + scripts = json.load(f) + for item in scripts: + scripts_map[item["headline"]] = item + except (json.JSONDecodeError, KeyError): + pass + + mp3_files = [] + if os.path.isdir(audio_dir): + mp3_files = sorted( + f for f in os.listdir(audio_dir) if f.lower().endswith(".mp3") + ) + + html = _generate_index(mp3_files, scripts_map) + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(html.encode("utf-8")))) + self.end_headers() + self.wfile.write(html.encode("utf-8")) + + def log_message(self, format, *args): + """Suppress per-request logs to avoid noisy output.""" + pass + + +def _generate_index(mp3_files, scripts_map): + """Generate the HTML index page for the MP3 newscast.""" + audio_entries = "" + for f in mp3_files: + safe_name = f.replace('"', """) + audio_entries += f""" +
+ +
+ {safe_name} + Download +
+
""" + + if not mp3_files: + audio_entries = '

No newscast files available yet. The pipeline will populate this directory.

' + + return f""" + + + + + Guardian Daily Newscast + + + + +
+

Guardian Daily Newscast

+

Daily US news — spoken by AI

+ + {audio_entries} +
+ +""" + + +def run_server(port: int, data_dir: str) -> None: + """Start the HTTP server in a background thread and block.""" + os.chdir("/") # Serve from root (paths are absolute) + NewscastHandler.data_dir = data_dir + handler = lambda *a, **kw: NewscastHandler( + *a, directory=data_dir, **kw + ) + server = ThreadingHTTPServer(("0.0.0.0", port), handler) + logger.info(f"Web server listening on port {port}") + server.serve_forever() diff --git a/tts_generator.py b/tts_generator.py index ed7d46f..0a2ac5b 100644 --- a/tts_generator.py +++ b/tts_generator.py @@ -11,9 +11,10 @@ from kokoro import KPipeline import soundfile as sf # --- Configuration --- -INPUT_FILE = "data/scripts.json" -OUTPUT_DIR = Path("data/audio") -VOICE = "af_heart" +DATA_DIR = os.environ.get("DATA_DIR", "data") +INPUT_FILE = os.path.join(DATA_DIR, "scripts.json") +OUTPUT_DIR = Path(DATA_DIR) / "audio" +VOICE = os.environ.get("TTS_VOICE", "af_heart") CONCURRENCY_LIMIT = 1 class ScriptItem(BaseModel):