77 lines
2.3 KiB
Python
77 lines
2.3 KiB
Python
import asyncio
|
|
from datetime import datetime
|
|
from typing import Optional, Union
|
|
from loguru import logger
|
|
from scraper.client import GuardianClient
|
|
from scraper.parser import GuardianParser
|
|
from scraper.models import Article, ScrapeResult
|
|
from scraper.utils import save_results_to_json
|
|
|
|
async def main():
|
|
# Construct dynamic URL for today's headlines
|
|
now = datetime.now()
|
|
year = now.year
|
|
month = now.strftime('%B').lower()[0:3]
|
|
day = str(now.day).zfill(2)
|
|
url = f"https://www.theguardian.com/us-news/{year}/{month}/{day}/all"
|
|
output_file = "data/output.json"
|
|
|
|
client = GuardianClient(concurrency_limit=5)
|
|
parser = GuardianParser()
|
|
|
|
try:
|
|
logger.info(f"Starting scraper. Target: {url}")
|
|
|
|
# 1. Fetch home page
|
|
home_html = await client.fetch(url)
|
|
if not home_html:
|
|
logger.error("Failed to retrieve home page. Exiting.")
|
|
return
|
|
|
|
# 2. Extract article links
|
|
article_links = parser.extract_article_links(home_html)
|
|
logger.info(f"Found {len(article_links)} articles to scrape.")
|
|
|
|
# 3. Fetch and parse articles concurrently
|
|
tasks = []
|
|
for headline, link in article_links:
|
|
tasks.append(scrape_article(client, parser, headline, link))
|
|
|
|
articles = await asyncio.gather(*tasks)
|
|
|
|
# Filter out None results (failed fetches)
|
|
valid_articles = [a for a in articles if a is not None]
|
|
logger.info(f"Successfully scraped {len(valid_articles)}/{len(article_links)} articles.")
|
|
|
|
# 4. Save results
|
|
result = ScrapeResult(
|
|
articles=valid_articles,
|
|
total_count=len(valid_articles)
|
|
)
|
|
save_results_to_json(result, output_file)
|
|
logger.info(f"Results saved to {output_file}")
|
|
|
|
except Exception as e:
|
|
logger.exception(f"An unexpected error occurred: {e}")
|
|
finally:
|
|
await client.close()
|
|
|
|
async def scrape_article(client: GuardianClient, parser: GuardianParser, headline: str, url: str) -> Optional[Article]:
|
|
html = await client.fetch(url)
|
|
if not html:
|
|
return None
|
|
|
|
body = parser.extract_article_body(html)
|
|
if not body:
|
|
logger.warning(f"Could not extract body for {url}")
|
|
return None
|
|
|
|
return Article(
|
|
headline=headline,
|
|
url=url,
|
|
body=body
|
|
)
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|