initial commit

main.py

@@ -0,0 +1,76 @@
+import asyncio
+from datetime import datetime
+from typing import Optional, Union
+from loguru import logger
+from scraper.client import GuardianClient
+from scraper.parser import GuardianParser
+from scraper.models import Article, ScrapeResult
+from scraper.utils import save_results_to_json
+
+async def main():
+    # Construct dynamic URL for today's headlines
+    now = datetime.now()
+    year = now.year
+    month = now.strftime('%B').lower()[0:3]
+    day = str(now.day).zfill(2)
+    url = f"https://www.theguardian.com/us-news/{year}/{month}/{day}/all"
+    output_file = "data/output.json"
+
+    client = GuardianClient(concurrency_limit=5)
+    parser = GuardianParser()
+
+    try:
+        logger.info(f"Starting scraper. Target: {url}")
+
+        # 1. Fetch home page
+        home_html = await client.fetch(url)
+        if not home_html:
+            logger.error("Failed to retrieve home page. Exiting.")
+            return
+
+        # 2. Extract article links
+        article_links = parser.extract_article_links(home_html)
+        logger.info(f"Found {len(article_links)} articles to scrape.")
+
+        # 3. Fetch and parse articles concurrently
+        tasks = []
+        for headline, link in article_links:
+            tasks.append(scrape_article(client, parser, headline, link))
+
+        articles = await asyncio.gather(*tasks)
+
+        # Filter out None results (failed fetches)
+        valid_articles = [a for a in articles if a is not None]
+        logger.info(f"Successfully scraped {len(valid_articles)}/{len(article_links)} articles.")
+
+        # 4. Save results
+        result = ScrapeResult(
+            articles=valid_articles,
+            total_count=len(valid_articles)
+        )
+        save_results_to_json(result, output_file)
+        logger.info(f"Results saved to {output_file}")
+
+    except Exception as e:
+        logger.exception(f"An unexpected error occurred: {e}")
+    finally:
+        await client.close()
+
+async def scrape_article(client: GuardianClient, parser: GuardianParser, headline: str, url: str) -> Optional[Article]:
+    html = await client.fetch(url)
+    if not html:
+        return None
+
+    body = parser.extract_article_body(html)
+    if not body:
+        logger.warning(f"Could not extract body for {url}")
+        return None
+
+    return Article(
+        headline=headline,
+        url=url,
+        body=body
+    )
+
+if __name__ == "__main__":
+    asyncio.run(main())