import asyncio
import os
from datetime import datetime
from typing import Optional, Union
from loguru import logger
from scraper.client import GuardianClient
from scraper.parser import GuardianParser
from scraper.models import Article, ScrapeResult
from scraper.utils import save_results_to_json

async def main():
    # Construct dynamic URL for today's headlines
    now = datetime.now()
    year = now.year
    month = now.strftime('%B').lower()[0:3]
    day = str(now.day).zfill(2)
    url = f"https://www.theguardian.com/us-news/{year}/{month}/{day}/all"
    data_dir = os.environ.get("DATA_DIR", "data")
    output_file = os.path.join(data_dir, "output.json")

    client = GuardianClient(concurrency_limit=5)
    parser = GuardianParser()

    try:
        logger.info(f"Starting scraper. Target: {url}")

        # 1. Fetch home page
        home_html = await client.fetch(url)
        if not home_html:
            logger.error("Failed to retrieve home page. Exiting.")
            return

        # 2. Extract article links
        article_links = parser.extract_article_links(home_html)
        logger.info(f"Found {len(article_links)} articles to scrape.")

        # 3. Fetch and parse articles concurrently
        tasks = []
        for headline, link in article_links:
            tasks.append(scrape_article(client, parser, headline, link))

        articles = await asyncio.gather(*tasks)

        # Filter out None results (failed fetches)
        valid_articles = [a for a in articles if a is not None]
        logger.info(f"Successfully scraped {len(valid_articles)}/{len(article_links)} articles.")

        # 4. Save results
        result = ScrapeResult(
            articles=valid_articles,
            total_count=len(valid_articles)
        )
        save_results_to_json(result, output_file)
        logger.info(f"Results saved to {output_file}")

    except Exception as e:
        logger.exception(f"An unexpected error occurred: {e}")
    finally:
        await client.close()

async def scrape_article(client: GuardianClient, parser: GuardianParser, headline: str, url: str) -> Optional[Article]:
    html = await client.fetch(url)
    if not html:
        return None

    body = parser.extract_article_body(html)
    if not body:
        logger.warning(f"Could not extract body for {url}")
        return None

    return Article(
        headline=headline,
        url=url,
        body=body
    )

if __name__ == "__main__":
    asyncio.run(main())