commit 34690b3691353a37b91bc2aa3b56b93ab2c39af1 Author: kfj001 Date: Fri Jun 12 23:10:33 2026 -0700 initial commit diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 0000000..11a7de9 --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,20 @@ +{ + "permissions": { + "allow": [ + "WebFetch(domain:theguardian.com)", + "Bash(curl *)", + "WebFetch(domain:www.theguardian.com)", + "Bash(python3 *)", + "Bash(pip3 install *)", + "Bash(pkill -f reader.py)", + "Bash(pip list *)", + "Bash(ffmpeg *)", + "Bash(pip3 show *)", + "Bash(ping *)", + "WebSearch", + "Bash(python *)", + "Bash(pip show *)", + "Bash(espeak-ng *)" + ] + } +} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8e9bb4b --- /dev/null +++ b/.gitignore @@ -0,0 +1,39 @@ +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +dist/ +build/ +.eggs/ + +# Virtual environments +venv/ +.venv/ +env/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# Data files (stage outputs) +data/output.json +data/scripts.json +data/audio/*.wav +data/audio/*.mp3 + +# Logs +*.log + +# OS +.DS_Store +Thumbs.db + +# Ollama model cache (if stored locally) +ollama/ + +# Environment variables +.env +.env.* diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..a55c949 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,126 @@ +# CLAUDE.md — Guardian Daily Newscast Pipeline Guide + +## Project Overview + +A Python pipeline that turns The Guardian's daily US news into MP3 newscast audio files. + +Three scripts run in sequence: +1. **main.py** — Scrapes Guardian articles (outputs `data/output.json`) +2. **reader.py** — Generates teleprompter scripts via local LLM (outputs `data/scripts.json`) +3. **tts_generator.py** — Converts scripts to MP3 audio (outputs `data/audio/*.mp3`) + +## Coding Standards + +- **Language**: Python 3.x +- **Style**: PEP 8 compliant. +- **Async**: Use `asyncio` and `httpx.AsyncClient`. Always ensure clients are closed properly (e.g., via `await client.close()`). +- **Logging**: Use `loguru` for all logging. +- **Typing**: Use Python type hints throughout. +- **Data Validation**: Use Pydantic models for any structured data. + +## Architecture & Key Patterns + +### Pipeline Overview + +``` +[Stage 1] main.py (Guardian Scraper) --> data/output.json +[Stage 2] reader.py (LLM Script Gen) --> data/scripts.json +[Stage 3] tts_generator.py (TTS) --> data/audio/*.mp3 +``` + +Each stage reads the previous stage's output file. Stages are independent and can be re-run in any order (but logically must follow 1 → 2 → 3). + +### Stage 1: Guardian Scraper (`main.py` + `scraper/`) + +**Client-Parser-Model Pattern:** + +- `GuardianClient` (`scraper/client.py`): Handles HTTP networking, rate limits, and concurrency via `asyncio.Semaphore`. Implements exponential backoff for 429/5xx responses. +- `GuardianParser` (`scraper/parser.py`): Pure functions for HTML extraction. Body fallback logic: (1) `div[data-gu-context="body"]`, (2) `
`, (3) `
`. +- `Article` / `ScrapeResult` (`scraper/models.py`): Pydantic models for data consistency. +- `utils.py`: Disk I/O (`save_results_to_json`). + +### Stage 2: LLM Script Generator (`reader.py`) + +- Sends each article body to Ollama (`http://127.0.0.1:11434/api/generate`, model `gemma4:e2b`). +- System prompt enforces clean teleprompter-ready text (no stage directions, no speaker labels). +- Retry logic: up to 3 attempts with exponential backoff (10s, 20s). 2s delay between requests. +- Incremental save to `data/scripts.json` during processing. + +### Stage 3: TTS Audio Generator (`tts_generator.py`) + +- `KokoroTTS`: Wraps Kokoro `KPipeline` with voice `af_heart`, outputs WAV at 24kHz. +- `convert_wav_to_mp3`: ffmpeg subprocess (`-q:a 2`). +- `ScriptItem`: Pydantic model validating input from `data/scripts.json`. +- Concurrent processing with `asyncio.Semaphore(CONCURRENCY_LIMIT=1)`. +- Cleans up intermediate WAV files after conversion. + +## Common Tasks + +### Add a New Pipeline Stage + +1. Create a new script (e.g., `stage_name.py`) in the project root. +2. Read the previous stage's output file. +3. Write results to a new output file. +4. Add it to the `&&` chain in README.md Quick Start. +5. Update the pipeline diagram in this document. +6. Add the output file to the Output Files table. + +### Change Target URL (Stage 1) + +Update the `url` variable in `main.py` (line 16). The URL is built dynamically from today's date: + +```python +url = f"https://www.theguardian.com/us-news/{year}/{month}/{day}/all" +``` + +### Modify Concurrency (Stage 1) + +Change the `concurrency_limit` passed to `GuardianClient` in `main.py` (line 19): + +```python +client = GuardianClient(concurrency_limit=5) +``` + +### Adjust TTS Voice or Quality + +- Voice: Change `VOICE = "af_heart"` in `tts_generator.py` (line 16). +- Audio quality: Change `-q:a 2` in the `convert_wav_to_mp3` function (line 62). +- Sample rate: Change `24000` in `sf.write()` (line 51). + +### Change LLM Model (Stage 2) + +Update `MODEL_NAME = "gemma4:e2b"` in `reader.py` (line 8). Ensure the model is pulled in Ollama (`ollama pull `). + +### Add Extraction Fields (Stage 1) + +This is a three-step process: +1. Update the `Article` model in `scraper/models.py`. +2. Update `extract_article_body` (or add a new method) in `scraper/parser.py`. +3. Update the `Article` instantiation in `main.py`'s `scrape_article` function. + +### Adjust Rate Limiting (Stage 1) + +Modify the `wait_time` calculation in `scraper/client.py` (lines 36, 42). + +### Modify Retry Logic (Stage 2) + +Change the retry count (line 40: `if attempt < 3`) and the backoff delay (line 41: `wait = attempt * 10`) in `reader.py`. + +### Update Data Flow Between Stages + +When an output file format changes: +1. Update the output format in the producing script (e.g., `main.py` stage 1). +2. Update the input format in the consuming script (e.g., `reader.py` stage 2). +3. Update type hints, Pydantic models, and JSON parsing in both files. +4. Update this document's pipeline diagram and output tables. + +## Critical Files + +- `main.py`: Stage 1 orchestration (Guardian scraper). +- `reader.py`: Stage 2 orchestration (LLM script generation). +- `tts_generator.py`: Stage 3 orchestration (TTS audio generation). +- `scraper/client.py`: HTTP networking + concurrency control. +- `scraper/parser.py`: HTML extraction (Guardian page structure). +- `scraper/models.py`: Pydantic data schemas (Article, ScrapeResult, ScriptItem). +- `scraper/utils.py`: Disk I/O helpers. +- `requirements.txt`: Python dependencies. diff --git a/README.md b/README.md new file mode 100644 index 0000000..47b5ad9 --- /dev/null +++ b/README.md @@ -0,0 +1,152 @@ +# Guardian Daily Newscast Pipeline + +Turns The Guardian's daily US news headlines into a spoken MP3 newscast. + +Three scripts run in sequence: **scrape** the articles, **generate** teleprompter scripts via a local LLM, **convert** scripts to audio. + +## Prerequisites + +External tools you must have installed before running: + +- **Ollama** — with the `gemma4:e2b` model pulled (`ollama pull gemma4:e2b`). Used in stage 2 to generate newscast scripts. +- **ffmpeg** — used in stage 3 to convert WAV to MP3 (`brew install ffmpeg` on macOS, `apt install ffmpeg` on Ubuntu). + +## Quick Start + +```bash +# 1. Install Python dependencies +pip install -r requirements.txt + +# 2. Run the full pipeline (stages 1, 2, 3 in order) +python main.py && python reader.py && python tts_generator.py + +# 3. Find your newscast files in data/audio/*.mp3 +ls data/audio/ +``` + +## Installation + +```bash +pip install -r requirements.txt +``` + +Dependencies: `httpx`, `beautifulsoup4`, `pydantic`, `loguru`, `kokoro`, `soundfile`, `torch`. + +## The Pipeline + +The project has three distinct stages that **must be run in order**: + +``` +[Stage 1] main.py (Guardian Scraper) + | + v + data/output.json + | + v +[Stage 2] reader.py (LLM Script Generator) + | + v + data/scripts.json + | + v +[Stage 3] tts_generator.py (TTS Audio Generator) + | + v + data/audio/*.mp3 +``` + +### Stage 1: Guardian Scraper (`main.py`) + +- Fetches today's Guardian US news home page (URL built from current date). +- Extracts all article links, then fetches each article's body text concurrently (semaphore-limited to 5). +- **Output:** `data/output.json` — a JSON object with an `articles` array (each entry: `headline`, `url`, `body`, `scraped_at`). + +### Stage 2: LLM Script Generator (`reader.py`) + +- Reads `data/output.json`. +- Sends each article body to a local Ollama instance (`http://127.0.0.1:11434/api/generate`, model `gemma4:e2b`). +- System prompt instructs the model to produce clean teleprompter-ready spoken text (no stage directions, no speaker labels). +- Retries failed requests up to 3 times (exponential backoff: 10s, 20s). 2s delay between each request to avoid overloading Ollama. +- **Output:** `data/scripts.json` — a JSON array of objects (each: `headline`, `script`). + +### Stage 3: TTS Audio Generator (`tts_generator.py`) + +- Reads `data/scripts.json`. +- Uses Kokoro TTS (local, voice `af_heart`, 24kHz) to generate WAV audio. +- Converts WAV to MP3 via ffmpeg (`-q:a 2`). +- Cleans up intermediate WAV files after conversion. +- **Output:** `data/audio/.mp3` files — one MP3 per article. + +## Usage + +Run all three stages in order: + +```bash +python main.py +python reader.py +python tts_generator.py +``` + +Or chain them with `&&`: + +```bash +python main.py && python reader.py && python tts_generator.py +``` + +Each stage is independent and can be re-run without re-running earlier stages (e.g., re-run stage 3 if audio quality needs to change, without re-scraping). + +## Project Structure + +``` +. +├── main.py # Stage 1: Guardian scraper +├── reader.py # Stage 2: LLM script generation +├── tts_generator.py # Stage 3: TTS audio generation +├── requirements.txt # Python dependencies +├── README.md # This file +├── CLAUDE.md # Developer guide +├── scraper/ +│ ├── __init__.py +│ ├── client.py # GuardianClient (HTTP + concurrency) +│ ├── parser.py # GuardianParser (HTML extraction) +│ ├── models.py # Pydantic models (Article, ScrapeResult) +│ └── utils.py # Disk I/O helpers +├── data/ +│ ├── output.json # Stage 1 output (scraped articles) +│ ├── scripts.json # Stage 2 output (generated scripts) +│ └── audio/ # Stage 3 output (MP3 files) +│ ├── article_1.mp3 +│ ├── article_2.mp3 +│ └── ... +└── .claude/ + └── settings.local.json +``` + +## Dependencies + +### Python (via `pip install -r requirements.txt`) + +| Package | Purpose | +|---|---| +| httpx | Async HTTP client for scraping | +| beautifulsoup4 | HTML parsing | +| pydantic | Data validation models | +| loguru | Logging | +| kokoro | Local TTS engine | +| soundfile | WAV audio I/O | +| torch | Required by kokoro | + +### External (not installed via pip) + +| Tool | Requirement | +|---|---| +| Ollama | Running locally, with `gemma4:e2b` | +| ffmpeg | Available on PATH | + +## Output Files + +| File | Description | +|---|---| +| `data/output.json` | JSON object containing scraped articles with `headline`, `url`, `body`, and `scraped_at` fields. | +| `data/scripts.json` | JSON array of objects with `headline` and `script` fields. | +| `data/audio/*.mp3` | One MP3 file per article, named from the sanitized headline. | diff --git a/main.py b/main.py new file mode 100644 index 0000000..480db5d --- /dev/null +++ b/main.py @@ -0,0 +1,76 @@ +import asyncio +from datetime import datetime +from typing import Optional, Union +from loguru import logger +from scraper.client import GuardianClient +from scraper.parser import GuardianParser +from scraper.models import Article, ScrapeResult +from scraper.utils import save_results_to_json + +async def main(): + # Construct dynamic URL for today's headlines + now = datetime.now() + year = now.year + month = now.strftime('%B').lower()[0:3] + day = str(now.day).zfill(2) + url = f"https://www.theguardian.com/us-news/{year}/{month}/{day}/all" + output_file = "data/output.json" + + client = GuardianClient(concurrency_limit=5) + parser = GuardianParser() + + try: + logger.info(f"Starting scraper. Target: {url}") + + # 1. Fetch home page + home_html = await client.fetch(url) + if not home_html: + logger.error("Failed to retrieve home page. Exiting.") + return + + # 2. Extract article links + article_links = parser.extract_article_links(home_html) + logger.info(f"Found {len(article_links)} articles to scrape.") + + # 3. Fetch and parse articles concurrently + tasks = [] + for headline, link in article_links: + tasks.append(scrape_article(client, parser, headline, link)) + + articles = await asyncio.gather(*tasks) + + # Filter out None results (failed fetches) + valid_articles = [a for a in articles if a is not None] + logger.info(f"Successfully scraped {len(valid_articles)}/{len(article_links)} articles.") + + # 4. Save results + result = ScrapeResult( + articles=valid_articles, + total_count=len(valid_articles) + ) + save_results_to_json(result, output_file) + logger.info(f"Results saved to {output_file}") + + except Exception as e: + logger.exception(f"An unexpected error occurred: {e}") + finally: + await client.close() + +async def scrape_article(client: GuardianClient, parser: GuardianParser, headline: str, url: str) -> Optional[Article]: + html = await client.fetch(url) + if not html: + return None + + body = parser.extract_article_body(html) + if not body: + logger.warning(f"Could not extract body for {url}") + return None + + return Article( + headline=headline, + url=url, + body=body + ) + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/reader.py b/reader.py new file mode 100644 index 0000000..22c987a --- /dev/null +++ b/reader.py @@ -0,0 +1,92 @@ +import json +import asyncio +import httpx +from loguru import logger +from typing import List, Dict + +OLLAMA_URL = "http://127.0.0.1:11434/api/generate" +MODEL_NAME = "gemma4:e2b" +INPUT_FILE = "data/output.json" +OUTPUT_FILE = "data/scripts.json" + +async def generate_script(client: httpx.AsyncClient, headline: str, content: str, attempt: int = 1) -> str: + system_prompt = ( + "You are an expert newscaster script writer. Your sole purpose is to convert articles into scripts for a TTS engine. " + "CRITICAL RULE: Provide ONLY the literal words to be spoken by the news anchor. " + "STRICTLY FORBIDDEN: Do not include peripheral annotations, stage directions, or non-spoken text. " + "Specifically, do NOT include:\n" + "- Bracketed text (e.g., [Music], [Pause], [Cut to B-roll])\n" + "- Parenthetical notes (e.g., (smiling), (slowly))\n" + "- Speaker labels or names (e.g., 'Anchor:', 'Reporter:', 'Host:')\n" + "- Scene headings, timestamps, or markdown formatting.\n" + "The output must be a clean stream of spoken text ready for a teleprompter." + ) + prompt = f"Write a detailed, lengthy newscaster's script based on the following text:\n\n```text\n{content}```" + payload = { + "model": MODEL_NAME, + "prompt": prompt, + "system": system_prompt, + "stream": False + } + + try: + logger.info(f"Generating script for: {headline} (Attempt {attempt})") + # High timeout for slow local LLM + response = await client.post(OLLAMA_URL, json=payload, timeout=600.0) + response.raise_for_status() + result = response.json() + return result.get("response", "") + except (httpx.HTTPStatusError, httpx.RequestError) as e: + if attempt < 3: + wait = attempt * 10 + logger.warning(f"Error occurred for {headline}: {e}. Retrying in {wait}s...") + await asyncio.sleep(wait) + return await generate_script(client, headline, content, attempt + 1) + logger.error(f"Max retries reached for {headline}. Final error: {e}") + return "" + except Exception: + logger.exception(f"Unexpected error occurred while requesting script for {headline}") + return "" + +async def main(): + try: + with open(INPUT_FILE, "r", encoding="utf-8") as f: + data = json.load(f) + except (FileNotFoundError, json.JSONDecodeError) as e: + logger.error(f"Failed to read {INPUT_FILE}: {e}") + return + + articles = data.get("articles", []) + if not articles: + logger.warning("No articles found in the input file.") + return + + results = [] + async with httpx.AsyncClient() as client: + for i, article in enumerate(articles): + headline = article.get("headline", "Unknown Headline") + body = article.get("body", "") + + if not body: + logger.warning(f"Skipping article {headline} because body is empty.") + continue + + script = await generate_script(client, headline, body) + if script: + results.append({ + "headline": headline, + "script": script + }) + # Incremental save + with open(OUTPUT_FILE, "w", encoding="utf-8") as f: + json.dump(results, f, indent=4, ensure_ascii=False) + else: + logger.warning(f"Failed to generate script for {headline}") + + # Avoid overloading Ollama + await asyncio.sleep(2) + + logger.info(f"Completed processing. Total scripts written: {len(results)}") + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..3c58c08 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +httpx +beautifulsoup4 +pydantic +loguru +kokoro +soundfile +torch \ No newline at end of file diff --git a/scraper/__init__.py b/scraper/__init__.py new file mode 100644 index 0000000..61f81f2 --- /dev/null +++ b/scraper/__init__.py @@ -0,0 +1 @@ +# Initialize scraper package diff --git a/scraper/client.py b/scraper/client.py new file mode 100644 index 0000000..67a0145 --- /dev/null +++ b/scraper/client.py @@ -0,0 +1,59 @@ +import httpx +import asyncio +from loguru import logger +from typing import Optional + +class GuardianClient: + def __init__(self, concurrency_limit: int = 5): + self.headers = { + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3", + "Accept-Language": "en-US,en;q=0.9", + "Accept-Encoding": "gzip, deflate, br", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "Referer": "https://www.google.com/", + } + self.semaphore = asyncio.Semaphore(concurrency_limit) + self.client = httpx.AsyncClient( + headers=self.headers, + follow_redirects=True, + timeout=httpx.Timeout(10.0, read=30.0), + http2=True + ) + + async def fetch(self, url: str, retries: int = 3) -> Optional[str]: + async with self.semaphore: + for attempt in range(retries): + try: + logger.debug(f"Fetching {url} (Attempt {attempt + 1}/{retries})") + response = await self.client.get(url) + + if response.status_code == 200: + return response.text + + if response.status_code == 429: + wait_time = (attempt + 1) * 2 + logger.warning(f"Rate limited (429) on {url}. Waiting {wait_time}s...") + await asyncio.sleep(wait_time) + continue + + if 500 <= response.status_code < 600: + wait_time = (attempt + 1) * 2 + logger.warning(f"Server error ({response.status_code}) on {url}. Waiting {wait_time}s...") + await asyncio.sleep(wait_time) + continue + + logger.error(f"Failed to fetch {url}: HTTP {response.status_code}") + return None + + except httpx.RequestError as e: + logger.error(f"Request error fetching {url}: {e}") + if attempt == retries - 1: + return None + await asyncio.sleep((attempt + 1) * 2) + + return None + + async def close(self): + await self.client.aclose() diff --git a/scraper/models.py b/scraper/models.py new file mode 100644 index 0000000..95ec41a --- /dev/null +++ b/scraper/models.py @@ -0,0 +1,14 @@ +from pydantic import BaseModel, Field +from datetime import datetime +from typing import Optional + +class Article(BaseModel): + headline: str = Field(..., description="The headline of the article") + url: str = Field(..., description="The URL of the article") + body: str = Field(..., description="The main body text of the article") + scraped_at: datetime = Field(default_factory=datetime.utcnow, description="Timestamp of when the article was scraped") + +class ScrapeResult(BaseModel): + articles: list[Article] + total_count: int + scraped_at: datetime = Field(default_factory=datetime.utcnow) diff --git a/scraper/parser.py b/scraper/parser.py new file mode 100644 index 0000000..60730af --- /dev/null +++ b/scraper/parser.py @@ -0,0 +1,91 @@ +from bs4 import BeautifulSoup +from typing import List, Tuple, Optional +import re + +class GuardianParser: + # Regular expression to identify article URLs + # Guardian articles typically follow /category/article-slug + # Refined pattern: articles usually have more segments or a specific slug format + # Avoid matching simple category paths like /world/middleeast + ARTICLE_URL_PATTERN = re.compile(r'^/(world|politics|sport|business|culture|lifeandstyle|opinion|environment|science|tech)/[a-zA-Z0-9-]+(/[a-zA-Z0-9-]+)*$') + + @staticmethod + def extract_article_links(html: str) -> List[Tuple[str, str]]: + """ + Extracts headlines and URLs from the home page using the data-link-name attribute. + """ + soup = BeautifulSoup(html, 'html.parser') + articles = [] + + # Target links with data-link-name="article" + for a in soup.select('a[data-link-name="article"]'): + href = a.get('href', '') + if not href: + continue + + # Clean href (remove query params) + href = href.split('?')[0] + + # Handle absolute and relative URLs + if href.startswith('http'): + full_url = href + elif href.startswith('/'): + full_url = f"https://www.theguardian.com{href}" + else: + continue + + headline = a.get_text(strip=True) + if headline: + articles.append((headline, full_url)) + + # De-duplicate while preserving order + seen = set() + unique_articles = [] + for headline, url in articles: + if url not in seen: + unique_articles.append((headline, url)) + seen.add(url) + + return unique_articles + + + # De-duplicate while preserving order + seen = set() + unique_articles = [] + for headline, url in articles: + if url not in seen: + unique_articles.append((headline, url)) + seen.add(url) + + return unique_articles + + @staticmethod + def extract_article_body(html: str) -> Optional[str]: + """ + Extracts the main body text from an article page. + """ + soup = BeautifulSoup(html, 'html.parser') + + # Primary target: data-gu-context="body" + body_div = soup.find('div', {'data-gu-context': 'body'}) + if body_div: + # Remove script and style elements + for script_or_style in body_div(['script', 'style']): + script_or_style.decompose() + return body_div.get_text(separator='\n', strip=True) + + # Fallback 1:
tag + article_tag = soup.find('article') + if article_tag: + for script_or_style in article_tag(['script', 'style']): + script_or_style.decompose() + return article_tag.get_text(separator='\n', strip=True) + + # Fallback 2: Main content area + main_content = soup.find('main') + if main_content: + for script_or_style in main_content(['script', 'style']): + script_or_style.decompose() + return main_content.get_text(separator='\n', strip=True) + + return None diff --git a/scraper/utils.py b/scraper/utils.py new file mode 100644 index 0000000..0c18bd1 --- /dev/null +++ b/scraper/utils.py @@ -0,0 +1,16 @@ +import json +import os +from .models import ScrapeResult + +def save_results_to_json(result: ScrapeResult, file_path: str): + """ + Saves the scrape result to a JSON file. + """ + # Ensure the directory exists + os.makedirs(os.path.dirname(file_path), exist_ok=True) + + # Convert Pydantic model to dict + data = result.model_dump(mode='json') + + with open(file_path, 'w', encoding='utf-8') as f: + json.dump(data, f, indent=4, ensure_ascii=False) diff --git a/tts_generator.py b/tts_generator.py new file mode 100644 index 0000000..ed7d46f --- /dev/null +++ b/tts_generator.py @@ -0,0 +1,133 @@ +import asyncio +import json +import os +import io +from pathlib import Path +from typing import List + +from loguru import logger +from pydantic import BaseModel +from kokoro import KPipeline +import soundfile as sf + +# --- Configuration --- +INPUT_FILE = "data/scripts.json" +OUTPUT_DIR = Path("data/audio") +VOICE = "af_heart" +CONCURRENCY_LIMIT = 1 + +class ScriptItem(BaseModel): + headline: str + script: str + +class KokoroTTS: + def __init__(self, lang_code: str = 'a'): + self.pipeline = KPipeline(lang_code=lang_code) + + async def text_to_speech(self, text: str, output_wav: Path): + """ + Generates speech locally using Kokoro and saves the resulting audio to a WAV file. + """ + try: + logger.debug(f"Generating TTS locally for: {text[:50]}...") + + # The pipeline returns a generator + generator = self.pipeline(text, voice=VOICE) + + # We take the first audio segment produced + # For short scripts, it's usually one segment. + # For longer ones, we concatenate. + all_audio = [] + for _, _, audio in generator: + all_audio.append(audio) + + if not all_audio: + raise RuntimeError("No audio data generated by Kokoro") + + import numpy as np + final_audio = np.concatenate(all_audio) + + # Kokoro outputs at 24000 Hz + sf.write(output_wav, final_audio, 24000) + + except Exception as e: + logger.exception(f"Unexpected error during Kokoro TTS: {e}") + raise + +async def convert_wav_to_mp3(wav_path: Path, mp3_path: Path): + """ + Converts a WAV file to MP3 using ffmpeg. + """ + try: + cmd = ["ffmpeg", "-y", "-i", str(wav_path), "-q:a", "2", str(mp3_path)] + process = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE + ) + stdout, stderr = await process.communicate() + + if process.returncode != 0: + logger.error(f"ffmpeg error: {stderr.decode()}") + return False + return True + except Exception as e: + logger.exception(f"Unexpected error during conversion: {e}") + return False + +async def process_article(semaphore: asyncio.Semaphore, client: KokoroTTS, item: ScriptItem): + """ + Processes a single article: combines text, generates audio, and converts to MP3. + """ + async with semaphore: + if not item.headline: + logger.warning("Skipping article processing: headline is empty.") + return + safe_headline = "".join([c if c.isalnum() else "_" for c in item.headline])[:100] + wav_path = OUTPUT_DIR / f"{safe_headline}.wav" + mp3_path = OUTPUT_DIR / f"{safe_headline}.mp3" + + full_text = f"{item.headline}. {item.script}" + + try: + await client.text_to_speech(full_text, wav_path) + + if await convert_wav_to_mp3(wav_path, mp3_path): + logger.info(f"Successfully created audio for: {item.headline}") + else: + logger.error(f"Failed to convert audio for: {item.headline}") + + if wav_path.exists(): + wav_path.unlink() + + except Exception as e: + logger.error(f"Error processing '{item.headline}': {e}") + +async def main(): + OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + + try: + with open(INPUT_FILE, "r", encoding="utf-8") as f: + data = json.load(f) + scripts = [ScriptItem(**item) for item in data] + except Exception as e: + logger.error(f"Failed to load input file {INPUT_FILE}: {e}") + return + + client = KokoroTTS() + + logger.info(f"Loaded {len(scripts)} scripts. Starting local TTS generation via Kokoro...") + semaphore = asyncio.Semaphore(CONCURRENCY_LIMIT) + + tasks = [process_article(semaphore, client, item) for item in scripts] + + total = len(scripts) + for i, coro in enumerate(asyncio.as_completed(tasks), 1): + await coro + logger.info(f"🚀 Progress: 🎙️ {i}/{total} articles processed! ✨") + + logger.info("TTS generation complete.") + +if __name__ == "__main__": + logger.add("tts_generator.log", rotation="1 MB") + asyncio.run(main())