318 lines
9.8 KiB
Python
318 lines
9.8 KiB
Python
"""Nginx access log parser with IP geolocation enrichment."""
|
|
|
|
import asyncio
|
|
import json
|
|
import os
|
|
from datetime import date, datetime
|
|
from typing import Callable, Optional
|
|
|
|
import maxminddb
|
|
|
|
from app.config import settings
|
|
from app.database import async_session
|
|
from app.models import LogParseState, VisitorStatDaily, VisitorStatGeo
|
|
|
|
# ── Geo reader ──────────────────────────────────────────────
|
|
|
|
_geo_reader: Optional[maxminddb.Reader] = None
|
|
|
|
|
|
def get_geo_reader() -> maxminddb.Reader:
|
|
"""Return a singleton MaxMind GeoLite2 City reader."""
|
|
global _geo_reader
|
|
if _geo_reader is None:
|
|
db_path = settings.GEOLITE2_DB_PATH
|
|
if os.path.exists(db_path):
|
|
_geo_reader = maxminddb.open_database(db_path)
|
|
else:
|
|
raise FileNotFoundError(
|
|
f"GeoLite2 database not found at {db_path}. "
|
|
"Download from https://dev.maxmind.com/geoip/geolite2-free-geolocation-data "
|
|
"and place it at backend/geo/GeoLite2-City.mmdb"
|
|
)
|
|
return _geo_reader
|
|
|
|
|
|
# Country centroid fallback for free GeoLite2 (no city/coordinates).
|
|
# Approximate center coordinates per ISO country code — enough for map markers.
|
|
_COUNTRY_CENTROIDS: dict[str, tuple[float, float]] = {
|
|
"US": (39.8, -98.5),
|
|
"GB": (53.5, -1.5),
|
|
"DE": (51.2, 10.4),
|
|
"FR": (46.6, 2.3),
|
|
"JP": (36.2, 138.3),
|
|
"AU": (-25.3, 133.8),
|
|
"BR": (-14.2, -51.9),
|
|
"CA": (56.1, -106.3),
|
|
"IN": (20.6, 79.0),
|
|
"KR": (35.9, 127.8),
|
|
"NL": (52.1, 5.3),
|
|
"MX": (23.6, -102.5),
|
|
"IT": (41.9, 12.6),
|
|
"ES": (40.5, -3.7),
|
|
"SE": (62.0, 15.0),
|
|
"PL": (51.9, 19.1),
|
|
"AR": (-38.4, -63.6),
|
|
"ZA": (-30.6, 22.9),
|
|
"EG": (26.8, 30.8),
|
|
"TR": (39.0, 35.2),
|
|
"RU": (61.5, 105.3),
|
|
"SG": (1.35, 103.8),
|
|
"NZ": (-40.9, 174.9),
|
|
"CO": (4.6, -74.3),
|
|
"CL": (-35.7, -71.5),
|
|
"PE": (-9.2, -75.0),
|
|
"PH": (12.9, 121.8),
|
|
"ID": (-0.8, 113.9),
|
|
"TH": (15.9, 100.9),
|
|
"VN": (14.1, 108.3),
|
|
"MY": (4.2, 101.9),
|
|
"AE": (23.4, 53.8),
|
|
"SA": (23.9, 45.1),
|
|
"IL": (31.0, 34.9),
|
|
"NO": (60.5, 8.5),
|
|
"DK": (56.3, 9.5),
|
|
"FI": (64.1, 26.5),
|
|
"BE": (50.5, 4.5),
|
|
"AT": (47.5, 14.6),
|
|
"CH": (46.8, 8.2),
|
|
"PT": (39.4, -8.2),
|
|
"GR": (39.1, 21.8),
|
|
"CZ": (49.8, 15.5),
|
|
"RO": (45.9, 24.9),
|
|
"HU": (47.2, 19.5),
|
|
"UA": (48.4, 31.2),
|
|
"NG": (9.1, 8.7),
|
|
"KE": (-0.02, 37.9),
|
|
"GH": (7.9, -1.0),
|
|
"CN": (35.9, 104.2),
|
|
"PK": (30.4, 69.3),
|
|
"BD": (23.7, 90.4),
|
|
}
|
|
|
|
|
|
def lookup_geo(ip: str) -> dict:
|
|
"""Look up geolocation for an IP address.
|
|
|
|
Returns dict with country, country_code, city, latitude, longitude
|
|
or empty dict if lookup fails. Falls back to country centroids
|
|
when the free GeoLite2 DB lacks city/coordinate data.
|
|
"""
|
|
try:
|
|
reader = get_geo_reader()
|
|
result = reader.get(ip)
|
|
if result is None:
|
|
return {}
|
|
country = result.get("country", {})
|
|
city = result.get("city", {})
|
|
location = result.get("location", {})
|
|
country_code = country.get("iso_code")
|
|
lat = location.get("latitude")
|
|
lon = location.get("longitude")
|
|
|
|
# Free GeoLite2 City often lacks city/coordinates — use centroid fallback
|
|
if lat is None or lon is None:
|
|
centroid = _COUNTRY_CENTROIDS.get(country_code or "")
|
|
if centroid:
|
|
lat = lat if lat is not None else centroid[0]
|
|
lon = lon if lon is not None else centroid[1]
|
|
|
|
return {
|
|
"country": country.get("names", {}).get("en", country_code or ""),
|
|
"country_code": country_code,
|
|
"city": city.get("names", {}).get("en", ""),
|
|
"latitude": lat,
|
|
"longitude": lon,
|
|
}
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
# ── IP resolution ───────────────────────────────────────────
|
|
|
|
|
|
def resolve_client_ip(entry: dict) -> str:
|
|
"""Extract the real client IP from a log entry.
|
|
|
|
Prefers the leftmost X-Forwarded-For entry (production behind proxy),
|
|
falls back to remote_addr (dev).
|
|
"""
|
|
xff = entry.get("x_forwarded_for", "")
|
|
if xff and xff != "-":
|
|
# Leftmost entry is the original client
|
|
return xff.split(",")[0].strip()
|
|
return entry.get("remote_addr", "")
|
|
|
|
|
|
# ── Log parsing ─────────────────────────────────────────────
|
|
|
|
|
|
def parse_log_line(line: str) -> Optional[dict]:
|
|
"""Parse a single JSON log line. Returns dict or None if unparseable."""
|
|
line = line.strip()
|
|
if not line:
|
|
return None
|
|
try:
|
|
return json.loads(line)
|
|
except (json.JSONDecodeError, ValueError):
|
|
return None
|
|
|
|
|
|
def parse_log_file(filepath: str) -> dict:
|
|
"""Parse a single daily log file and return aggregated counts.
|
|
|
|
Returns a dict with:
|
|
- unique_ips: set of unique client IP addresses
|
|
- geo_counts: dict of (country_code, country, city, lat, lon) -> count
|
|
"""
|
|
unique_ips: set[str] = set()
|
|
geo_counts: dict[tuple, int] = {}
|
|
|
|
with open(filepath, "r") as f:
|
|
for line in f:
|
|
entry = parse_log_line(line)
|
|
if entry is None:
|
|
continue
|
|
|
|
ip = resolve_client_ip(entry)
|
|
if not ip:
|
|
continue
|
|
|
|
unique_ips.add(ip)
|
|
|
|
# Geo enrichment
|
|
geo = lookup_geo(ip)
|
|
if geo.get("country_code"):
|
|
geo_key = (
|
|
geo.get("country_code", ""),
|
|
geo.get("country", ""),
|
|
geo.get("city", ""),
|
|
geo.get("latitude"),
|
|
geo.get("longitude"),
|
|
)
|
|
geo_counts[geo_key] = geo_counts.get(geo_key, 0) + 1
|
|
|
|
return {
|
|
"unique_ips": unique_ips,
|
|
"geo_counts": geo_counts,
|
|
}
|
|
|
|
|
|
async def process_unparsed_logs(
|
|
progress_callback: Optional[Callable] = None,
|
|
) -> dict:
|
|
"""Scan /logs/ for unprocessed daily files, parse them, store aggregates.
|
|
|
|
Only processes files strictly older than today (file must be complete).
|
|
Advances the LogParseState marker after each successful file.
|
|
|
|
Returns a summary dict with files_processed and rows_inserted.
|
|
"""
|
|
logs_dir = settings.LOGS_DIR
|
|
if not os.path.isdir(logs_dir):
|
|
return {"files_processed": 0, "error": f"Logs directory not found: {logs_dir}"}
|
|
|
|
# Get current parse state
|
|
async with async_session() as session:
|
|
from sqlalchemy import select
|
|
|
|
result = await session.execute(
|
|
select(LogParseState).where(LogParseState.key == "default")
|
|
)
|
|
state = result.scalar_one_or_none()
|
|
|
|
if state is None:
|
|
# Initialize with a far-past date so all files get processed
|
|
state = LogParseState(key="default", last_parsed_date=date(2020, 1, 1))
|
|
session.add(state)
|
|
await session.commit()
|
|
|
|
last_parsed = state.last_parsed_date
|
|
|
|
# Discover daily log files
|
|
import glob
|
|
|
|
pattern = os.path.join(logs_dir, "access-*.log")
|
|
log_files = sorted(glob.glob(pattern))
|
|
|
|
files_processed = 0
|
|
rows_inserted = 0
|
|
today = date.today()
|
|
|
|
for filepath in log_files:
|
|
# Extract date from filename
|
|
basename = os.path.basename(filepath)
|
|
try:
|
|
date_str = basename.replace("access-", "").replace(".log", "")
|
|
file_date = date.fromisoformat(date_str)
|
|
except ValueError:
|
|
continue
|
|
|
|
# Only process files strictly before today (file must be complete)
|
|
if file_date >= today:
|
|
continue
|
|
|
|
# Only process files after the last parsed date
|
|
if file_date <= last_parsed:
|
|
continue
|
|
|
|
# Parse this file
|
|
try:
|
|
stats = parse_log_file(filepath)
|
|
except Exception as e:
|
|
print(f" ERROR parsing {filepath}: {e}")
|
|
continue
|
|
|
|
# Store aggregates in database
|
|
async with async_session() as session:
|
|
from sqlalchemy import select
|
|
|
|
# Upsert daily stats
|
|
result = await session.execute(
|
|
select(VisitorStatDaily).where(VisitorStatDaily.date == file_date)
|
|
)
|
|
daily = result.scalar_one_or_none()
|
|
if daily is None:
|
|
daily = VisitorStatDaily(date=file_date)
|
|
session.add(daily)
|
|
|
|
daily.unique_visitors = len(stats["unique_ips"])
|
|
|
|
# Geo stats
|
|
for geo_key, count in stats["geo_counts"].items():
|
|
geo_row = VisitorStatGeo(
|
|
date=file_date,
|
|
country_code=geo_key[0],
|
|
country=geo_key[1],
|
|
city=geo_key[2],
|
|
latitude=geo_key[3],
|
|
longitude=geo_key[4],
|
|
visitor_count=count,
|
|
)
|
|
session.add(geo_row)
|
|
rows_inserted += 1
|
|
|
|
# Advance parse state
|
|
state = (
|
|
await session.execute(
|
|
select(LogParseState).where(LogParseState.key == "default")
|
|
)
|
|
).scalar_one()
|
|
state.last_parsed_date = file_date
|
|
state.updated_at = datetime.utcnow()
|
|
|
|
await session.commit()
|
|
|
|
files_processed += 1
|
|
|
|
# Report progress via callback (for SSE streaming)
|
|
if progress_callback:
|
|
cb_result = progress_callback(
|
|
file_date.isoformat(), files_processed, len(log_files)
|
|
)
|
|
# Support both sync and async callbacks
|
|
if cb_result is not None and asyncio.iscoroutine(cb_result):
|
|
await cb_result
|
|
|
|
return {"files_processed": files_processed, "rows_inserted": rows_inserted}
|