Adds visitor dashboard and related features
This commit is contained in:
@@ -0,0 +1,317 @@
|
||||
"""Nginx access log parser with IP geolocation enrichment."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
from datetime import date, datetime
|
||||
from typing import Callable, Optional
|
||||
|
||||
import maxminddb
|
||||
|
||||
from app.config import settings
|
||||
from app.database import async_session
|
||||
from app.models import LogParseState, VisitorStatDaily, VisitorStatGeo
|
||||
|
||||
# ── Geo reader ──────────────────────────────────────────────
|
||||
|
||||
_geo_reader: Optional[maxminddb.Reader] = None
|
||||
|
||||
|
||||
def get_geo_reader() -> maxminddb.Reader:
|
||||
"""Return a singleton MaxMind GeoLite2 City reader."""
|
||||
global _geo_reader
|
||||
if _geo_reader is None:
|
||||
db_path = settings.GEOLITE2_DB_PATH
|
||||
if os.path.exists(db_path):
|
||||
_geo_reader = maxminddb.open_database(db_path)
|
||||
else:
|
||||
raise FileNotFoundError(
|
||||
f"GeoLite2 database not found at {db_path}. "
|
||||
"Download from https://dev.maxmind.com/geoip/geolite2-free-geolocation-data "
|
||||
"and place it at backend/geo/GeoLite2-City.mmdb"
|
||||
)
|
||||
return _geo_reader
|
||||
|
||||
|
||||
# Country centroid fallback for free GeoLite2 (no city/coordinates).
|
||||
# Approximate center coordinates per ISO country code — enough for map markers.
|
||||
_COUNTRY_CENTROIDS: dict[str, tuple[float, float]] = {
|
||||
"US": (39.8, -98.5),
|
||||
"GB": (53.5, -1.5),
|
||||
"DE": (51.2, 10.4),
|
||||
"FR": (46.6, 2.3),
|
||||
"JP": (36.2, 138.3),
|
||||
"AU": (-25.3, 133.8),
|
||||
"BR": (-14.2, -51.9),
|
||||
"CA": (56.1, -106.3),
|
||||
"IN": (20.6, 79.0),
|
||||
"KR": (35.9, 127.8),
|
||||
"NL": (52.1, 5.3),
|
||||
"MX": (23.6, -102.5),
|
||||
"IT": (41.9, 12.6),
|
||||
"ES": (40.5, -3.7),
|
||||
"SE": (62.0, 15.0),
|
||||
"PL": (51.9, 19.1),
|
||||
"AR": (-38.4, -63.6),
|
||||
"ZA": (-30.6, 22.9),
|
||||
"EG": (26.8, 30.8),
|
||||
"TR": (39.0, 35.2),
|
||||
"RU": (61.5, 105.3),
|
||||
"SG": (1.35, 103.8),
|
||||
"NZ": (-40.9, 174.9),
|
||||
"CO": (4.6, -74.3),
|
||||
"CL": (-35.7, -71.5),
|
||||
"PE": (-9.2, -75.0),
|
||||
"PH": (12.9, 121.8),
|
||||
"ID": (-0.8, 113.9),
|
||||
"TH": (15.9, 100.9),
|
||||
"VN": (14.1, 108.3),
|
||||
"MY": (4.2, 101.9),
|
||||
"AE": (23.4, 53.8),
|
||||
"SA": (23.9, 45.1),
|
||||
"IL": (31.0, 34.9),
|
||||
"NO": (60.5, 8.5),
|
||||
"DK": (56.3, 9.5),
|
||||
"FI": (64.1, 26.5),
|
||||
"BE": (50.5, 4.5),
|
||||
"AT": (47.5, 14.6),
|
||||
"CH": (46.8, 8.2),
|
||||
"PT": (39.4, -8.2),
|
||||
"GR": (39.1, 21.8),
|
||||
"CZ": (49.8, 15.5),
|
||||
"RO": (45.9, 24.9),
|
||||
"HU": (47.2, 19.5),
|
||||
"UA": (48.4, 31.2),
|
||||
"NG": (9.1, 8.7),
|
||||
"KE": (-0.02, 37.9),
|
||||
"GH": (7.9, -1.0),
|
||||
"CN": (35.9, 104.2),
|
||||
"PK": (30.4, 69.3),
|
||||
"BD": (23.7, 90.4),
|
||||
}
|
||||
|
||||
|
||||
def lookup_geo(ip: str) -> dict:
|
||||
"""Look up geolocation for an IP address.
|
||||
|
||||
Returns dict with country, country_code, city, latitude, longitude
|
||||
or empty dict if lookup fails. Falls back to country centroids
|
||||
when the free GeoLite2 DB lacks city/coordinate data.
|
||||
"""
|
||||
try:
|
||||
reader = get_geo_reader()
|
||||
result = reader.get(ip)
|
||||
if result is None:
|
||||
return {}
|
||||
country = result.get("country", {})
|
||||
city = result.get("city", {})
|
||||
location = result.get("location", {})
|
||||
country_code = country.get("iso_code")
|
||||
lat = location.get("latitude")
|
||||
lon = location.get("longitude")
|
||||
|
||||
# Free GeoLite2 City often lacks city/coordinates — use centroid fallback
|
||||
if lat is None or lon is None:
|
||||
centroid = _COUNTRY_CENTROIDS.get(country_code or "")
|
||||
if centroid:
|
||||
lat = lat if lat is not None else centroid[0]
|
||||
lon = lon if lon is not None else centroid[1]
|
||||
|
||||
return {
|
||||
"country": country.get("names", {}).get("en", country_code or ""),
|
||||
"country_code": country_code,
|
||||
"city": city.get("names", {}).get("en", ""),
|
||||
"latitude": lat,
|
||||
"longitude": lon,
|
||||
}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
# ── IP resolution ───────────────────────────────────────────
|
||||
|
||||
|
||||
def resolve_client_ip(entry: dict) -> str:
|
||||
"""Extract the real client IP from a log entry.
|
||||
|
||||
Prefers the leftmost X-Forwarded-For entry (production behind proxy),
|
||||
falls back to remote_addr (dev).
|
||||
"""
|
||||
xff = entry.get("x_forwarded_for", "")
|
||||
if xff and xff != "-":
|
||||
# Leftmost entry is the original client
|
||||
return xff.split(",")[0].strip()
|
||||
return entry.get("remote_addr", "")
|
||||
|
||||
|
||||
# ── Log parsing ─────────────────────────────────────────────
|
||||
|
||||
|
||||
def parse_log_line(line: str) -> Optional[dict]:
|
||||
"""Parse a single JSON log line. Returns dict or None if unparseable."""
|
||||
line = line.strip()
|
||||
if not line:
|
||||
return None
|
||||
try:
|
||||
return json.loads(line)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def parse_log_file(filepath: str) -> dict:
|
||||
"""Parse a single daily log file and return aggregated counts.
|
||||
|
||||
Returns a dict with:
|
||||
- unique_ips: set of unique client IP addresses
|
||||
- geo_counts: dict of (country_code, country, city, lat, lon) -> count
|
||||
"""
|
||||
unique_ips: set[str] = set()
|
||||
geo_counts: dict[tuple, int] = {}
|
||||
|
||||
with open(filepath, "r") as f:
|
||||
for line in f:
|
||||
entry = parse_log_line(line)
|
||||
if entry is None:
|
||||
continue
|
||||
|
||||
ip = resolve_client_ip(entry)
|
||||
if not ip:
|
||||
continue
|
||||
|
||||
unique_ips.add(ip)
|
||||
|
||||
# Geo enrichment
|
||||
geo = lookup_geo(ip)
|
||||
if geo.get("country_code"):
|
||||
geo_key = (
|
||||
geo.get("country_code", ""),
|
||||
geo.get("country", ""),
|
||||
geo.get("city", ""),
|
||||
geo.get("latitude"),
|
||||
geo.get("longitude"),
|
||||
)
|
||||
geo_counts[geo_key] = geo_counts.get(geo_key, 0) + 1
|
||||
|
||||
return {
|
||||
"unique_ips": unique_ips,
|
||||
"geo_counts": geo_counts,
|
||||
}
|
||||
|
||||
|
||||
async def process_unparsed_logs(
|
||||
progress_callback: Optional[Callable] = None,
|
||||
) -> dict:
|
||||
"""Scan /logs/ for unprocessed daily files, parse them, store aggregates.
|
||||
|
||||
Only processes files strictly older than today (file must be complete).
|
||||
Advances the LogParseState marker after each successful file.
|
||||
|
||||
Returns a summary dict with files_processed and rows_inserted.
|
||||
"""
|
||||
logs_dir = settings.LOGS_DIR
|
||||
if not os.path.isdir(logs_dir):
|
||||
return {"files_processed": 0, "error": f"Logs directory not found: {logs_dir}"}
|
||||
|
||||
# Get current parse state
|
||||
async with async_session() as session:
|
||||
from sqlalchemy import select
|
||||
|
||||
result = await session.execute(
|
||||
select(LogParseState).where(LogParseState.key == "default")
|
||||
)
|
||||
state = result.scalar_one_or_none()
|
||||
|
||||
if state is None:
|
||||
# Initialize with a far-past date so all files get processed
|
||||
state = LogParseState(key="default", last_parsed_date=date(2020, 1, 1))
|
||||
session.add(state)
|
||||
await session.commit()
|
||||
|
||||
last_parsed = state.last_parsed_date
|
||||
|
||||
# Discover daily log files
|
||||
import glob
|
||||
|
||||
pattern = os.path.join(logs_dir, "access-*.log")
|
||||
log_files = sorted(glob.glob(pattern))
|
||||
|
||||
files_processed = 0
|
||||
rows_inserted = 0
|
||||
today = date.today()
|
||||
|
||||
for filepath in log_files:
|
||||
# Extract date from filename
|
||||
basename = os.path.basename(filepath)
|
||||
try:
|
||||
date_str = basename.replace("access-", "").replace(".log", "")
|
||||
file_date = date.fromisoformat(date_str)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
# Only process files strictly before today (file must be complete)
|
||||
if file_date >= today:
|
||||
continue
|
||||
|
||||
# Only process files after the last parsed date
|
||||
if file_date <= last_parsed:
|
||||
continue
|
||||
|
||||
# Parse this file
|
||||
try:
|
||||
stats = parse_log_file(filepath)
|
||||
except Exception as e:
|
||||
print(f" ERROR parsing {filepath}: {e}")
|
||||
continue
|
||||
|
||||
# Store aggregates in database
|
||||
async with async_session() as session:
|
||||
from sqlalchemy import select
|
||||
|
||||
# Upsert daily stats
|
||||
result = await session.execute(
|
||||
select(VisitorStatDaily).where(VisitorStatDaily.date == file_date)
|
||||
)
|
||||
daily = result.scalar_one_or_none()
|
||||
if daily is None:
|
||||
daily = VisitorStatDaily(date=file_date)
|
||||
session.add(daily)
|
||||
|
||||
daily.unique_visitors = len(stats["unique_ips"])
|
||||
|
||||
# Geo stats
|
||||
for geo_key, count in stats["geo_counts"].items():
|
||||
geo_row = VisitorStatGeo(
|
||||
date=file_date,
|
||||
country_code=geo_key[0],
|
||||
country=geo_key[1],
|
||||
city=geo_key[2],
|
||||
latitude=geo_key[3],
|
||||
longitude=geo_key[4],
|
||||
visitor_count=count,
|
||||
)
|
||||
session.add(geo_row)
|
||||
rows_inserted += 1
|
||||
|
||||
# Advance parse state
|
||||
state = (
|
||||
await session.execute(
|
||||
select(LogParseState).where(LogParseState.key == "default")
|
||||
)
|
||||
).scalar_one()
|
||||
state.last_parsed_date = file_date
|
||||
state.updated_at = datetime.utcnow()
|
||||
|
||||
await session.commit()
|
||||
|
||||
files_processed += 1
|
||||
|
||||
# Report progress via callback (for SSE streaming)
|
||||
if progress_callback:
|
||||
cb_result = progress_callback(
|
||||
file_date.isoformat(), files_processed, len(log_files)
|
||||
)
|
||||
# Support both sync and async callbacks
|
||||
if cb_result is not None and asyncio.iscoroutine(cb_result):
|
||||
await cb_result
|
||||
|
||||
return {"files_processed": files_processed, "rows_inserted": rows_inserted}
|
||||
Reference in New Issue
Block a user