diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index edb2297..65c1b12 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -26,6 +26,30 @@ See [README.md](README.md) for a full architecture map. Key files: - `backend/app/main.py` — FastAPI entry point (CORS, router registration) - `backend/app/models.py` — SQLAlchemy ORM (Show, Event, StationConfig) +### Admin dashboard — visitor stats (implemented) + +Nginx access logs → shared volume → FastAPI log parser → PostgreSQL → admin dashboard. No extra services. + +**How it works:** +1. Nginx writes structured JSON access logs to **daily log files** on a shared mount: `/logs/access-YYYY-MM-DD.log`. A `map` directive in [nginx.conf.template](nginx.conf.template) extracts the date from the ISO timestamp, so Nginx rotates to a new file at midnight. +2. FastAPI ([backend/app/log_parser.py](backend/app/log_parser.py)) maintains a **"last parsed date"** (at most yesterday) persisted in PostgreSQL via the `LogParseState` model. On each run it scans `/logs/` for unprocessed daily files, enriches entries with IP geolocation (MaxMind GeoLite2 City), and stores aggregates. +3. Aggregates are persisted in `visitor_stats_daily` (unique visitors per day) and `visitor_stats_geo` (country/city breakdown) tables. +4. Admin dashboard (Angular) reads from stats API endpoints in [backend/app/api/stats.py](backend/app/api/stats.py). + +**Key files:** +- `backend/app/log_parser.py` — log file parsing + geo enrichment + `process_unparsed_logs()` +- `backend/app/api/stats.py` — stats API endpoints (summary, timeseries, geo, parse trigger) +- `backend/app/models.py` — `VisitorStatDaily`, `VisitorStatGeo`, `LogParseState` ORM models +- `backend/inject_test_logs.py` — test data injector (writes test log entries for yesterday) +- `nginx.conf.template` — JSON log format (`kmtn_json`) with daily rotation via `$log_date` map +- `backend/geo/GeoLite2-City.mmdb` — MaxMind geolocation database + +**Why daily files:** The currently-being-written file is never read — no partial lines, no race conditions, no file locking. Stats have a minimum one-day lag. + +**Deployment context:** Single-node minikube, `hostPath` PersistentVolume with `ReadWriteMany`, or docker-compose named volume mounted in both pods. + +**Injecting test data:** Run `backend/inject_test_logs.py` (or the inline docker compose command in README.md) to write test log entries for yesterday, then trigger parsing via `GET /api/parse`. + ## Common tasks - **Re-skin:** Edit colors in `_variables.scss` only. diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 635f67b..7d08af0 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -5,7 +5,31 @@ "Read(*)", "Glob(*)", "Grep(*)", - "WebFetch(domain:www.kryzradio.org)" + "WebFetch(domain:www.kryzradio.org)", + "Bash(ls -la /logs/ 2>/dev/null || echo \"DIRECTORY /logs DOES NOT EXIST\")", + "Read(//logs/**)", + "Bash(head -5 /logs/access-2026-06-26.log)", + "Bash(python3 -c \"import sys, json; ips = set\\(\\); [ips.add\\(json.loads\\(l\\)['remote_addr']\\) for l in sys.stdin if l.strip\\(\\)]; print\\(f'Total unique IPs: {len\\(ips\\)}'\\); print\\('\\\\n'.join\\(sorted\\(ips\\)\\)\\)\")", + "Bash(python3 -c \"from datetime import date; print\\(f'Today: {date.today\\(\\)}'\\); print\\(f'Log files from Jun 13-27 are all before today: {date\\(2026, 6, 27\\) < date.today\\(\\)}'\\)\")", + "Bash(ls -la /logs/ 2>/dev/null || echo \"/logs/ not accessible\")", + "Bash(python3 -c \"import sys,json; ips=set\\(\\); [ips.add\\(json.loads\\(l\\)['remote_addr']\\) for l in sys.stdin if l.strip\\(\\)]; print\\(f'Unique IPs in 06-26: {len\\(ips\\)}'\\); [print\\(f' {ip}'\\) for ip in sorted\\(ips\\)]\")", + "Bash(python3 -c ' *)", + "Bash(docker exec *)", + "Bash(curl -s http://localhost:8000/api/stats/summary)", + "Bash(python3 -m json.tool)", + "Bash(curl -s \"http://localhost:8000/api/stats/timeseries?start_date=2026-06-13&end_date=2026-06-27\")", + "Bash(curl -s \"http://localhost:8000/api/stats/geo?start_date=2026-06-13&end_date=2026-06-27\")", + "Bash(env)", + "Bash(curl -s -X POST http://localhost:8000/api/auth/login -H \"Content-Type: application/json\" -d '{\"username\":\"admin\",\"password\":\"123\"}')", + "Bash(curl -s http://localhost:8000/api/stats/summary -H \"Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxIiwiaXNfYWRtaW4iOnRydWUsImV4cCI6MTc4MjcxMTYyOX0.8eM-Vd0H72R8eVomEtxLNeYSYUe65-0WPmNWUzx_mxw\")", + "Bash(curl -s \"http://localhost:8000/api/stats/timeseries?start_date=2026-06-13&end_date=2026-06-27\" -H \"Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxIiwiaXNfYWRtaW4iOnRydWUsImV4cCI6MTc4MjcxMTYyOX0.8eM-Vd0H72R8eVomEtxLNeYSYUe65-0WPmNWUzx_mxw\")", + "Bash(curl -s \"http://localhost:8000/api/stats/geo?start_date=2026-06-13&end_date=2026-06-27\" -H \"Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxIiwiaXNfYWRtaW4iOnRydWUsImV4cCI6MTc4MjcxMTYyOX0.8eM-Vd0H72R8eVomEtxLNeYSYUe65-0WPmNWUzx_mxw\")", + "Bash(python3 -c \"import maxminddb; print\\(maxminddb.__version__\\); print\\(dir\\(maxminddb.Reader\\)\\)\")", + "Bash(cat /tmp/uvicorn.log)", + "Read(//tmp/**)", + "Bash(KMTN_DATABASE_URL=\"sqlite+aiosqlite:///./kmountain.db\" KMTN_ADMIN_USERNAME=\"admin\" KMTN_ADMIN_PASSWORD=\"123\" KMTN_CORS_ORIGINS='[\"http://localhost:4200\"]' nohup uvicorn app.main:app --reload --host 0.0.0.0 --port 8000)", + "Bash(echo \"Backend PID: $!\")", + "Bash(curl -s http://localhost:8000/docs)" ], "additionalDirectories": [ "/workspaces/web_app/.vscode" diff --git a/.gitignore b/.gitignore index 68a7c15..7714906 100644 --- a/.gitignore +++ b/.gitignore @@ -47,4 +47,7 @@ Thumbs.db .env # Dev SQLite database -backend/kmountain.db \ No newline at end of file +backend/kmountain.db + +# GeoLite2 database (binary, licensed — see backend/geo/README.md) +backend/geo/GeoLite2-City.mmdb \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 13491c2..dd2a048 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,5 +15,6 @@ COPY docker-entrypoint.sh /docker-entrypoint.sh RUN chmod +x /docker-entrypoint.sh ENV API_UPSTREAM=http://api:8000 ENV API_PUBLIC_URL="" +RUN mkdir -p /logs EXPOSE 80 ENTRYPOINT ["/docker-entrypoint.sh"] diff --git a/README.md b/README.md index 4dc4350..1d63a78 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,8 @@ Open [http://localhost:4200](http://localhost:4200) and ensure the backend is ru ├── Dockerfile # Python 3.12-slim + gunicorn/uvicorn ├── requirements.txt ├── seed.py # Initial data for all models + ├── inject_test_logs.py # Test data injector for visitor stats pipeline + ├── geo/ # MaxMind GeoLite2 City database └── app/ ├── main.py # FastAPI app, CORS, router registration ├── config.py # ENV-based settings (KMTN_* prefix) @@ -104,7 +106,8 @@ Open [http://localhost:4200](http://localhost:4200) and ensure the backend is ru └── api/ ├── programs.py # CRUD /api/programs ├── events.py # CRUD /api/events - └── tiers.py # CRUD /api/tiers + ├── tiers.py # CRUD /api/tiers + └── stats.py # Visitor stats endpoints (summary, timeseries, geo, parse) ``` ## Local Development @@ -197,6 +200,72 @@ ng test # Angular + Vitest The backend API is self-documented at [http://localhost:8000/docs](http://localhost:8000/docs) (Swagger UI). The database auto-seeds on first startup — visit that URL to interactively test endpoints. +## Admin Dashboard — Visitor Stats + +A visitor analytics dashboard built on a **log parsing pipeline**: Nginx writes access logs to a shared volume that the FastAPI backend reads and processes. No extra services required (no Logstash, Prometheus, or message queues). + +### How it works + +1. **Nginx** writes structured JSON access logs to **daily log files** on a shared mount: `/logs/access-YYYY-MM-DD.log`. A `map` directive extracts the date from the ISO timestamp, so Nginx rotates to a new file at midnight each day. +2. **FastAPI** (via [log_parser.py](backend/app/log_parser.py)) scans `/logs/` for daily files that haven't been parsed yet, enriches each entry with IP geolocation (MaxMind GeoLite2 City), and stores aggregates in PostgreSQL. +3. **PostgreSQL** tables (`visitor_stats_daily`, `visitor_stats_geo`, `log_parse_state`) hold the aggregated data and a "last parsed date" marker. +4. **Angular admin dashboard** reads from the stats API endpoints to render charts and a world map. + +### Why daily files + +The currently-being-written file is never read — no partial lines, no race conditions, no file locking needed. Stats have a minimum one-day lag. + +### Stats API endpoints + +| Method | Endpoint | Description | +|---|---|---| +| `GET` | `/api/stats/summary` | Total unique visitors for a date range | +| `GET` | `/api/stats/timeseries` | Daily unique visitor counts (chart data) | +| `GET` | `/api/stats/geo` | Geographic breakdown by country (map data) | +| `GET` | `/api/stats/parse` | Trigger log parsing (SSE progress stream) | + +All endpoints require admin authentication (JWT token). Date range is optional; defaults to the last 30 days. + +### GeoLite2 setup + +The log parser requires a MaxMind GeoLite2 City database for IP geolocation: + +1. Download from [https://dev.maxmind.com/geoip/geolite2-free-geolocation-data](https://dev.maxmind.com/geoip/geolite2-free-geolocation-data) +2. Place the `.mmdb` file at `backend/geo/GeoLite2-City.mmdb` +3. The Dockerfile copies it into the container at `/app/geo/GeoLite2-City.mmdb` + +### Injecting test data + +To verify the pipeline works (e.g., after deploying or before a demo), use the test log injector script at [backend/inject_test_logs.py](backend/inject_test_logs.py). It writes realistic traffic from 25 well-known public IPs across multiple past days — data is only aggregated a day in arrears, so all dates are strictly before today. + +```bash +# Run inside the api container (writes to shared /logs volume) +docker compose exec api python /app/inject_test_logs.py /logs 14 + +# Or run locally (dev mode) +cd backend +python inject_test_logs.py /logs 14 +``` + +The second argument controls how many days of data to generate (default: 14). Each day gets a deterministic mix of visitors from the IP pool, so re-running produces the same output (idempotent — files are appended, so check first). + +After injecting, trigger parsing via the admin dashboard or: + +```bash +curl -X GET "http://localhost:8000/api/parse?token=YOUR_JWT" +``` + +### Shared volume setup + +For docker-compose, the `logs` named volume is mounted in both the `frontend` (Nginx) and `api` containers at `/logs`: + +```yaml +volumes: + logs:/logs +``` + +For Kubernetes (minikube), use a `hostPath` PersistentVolume with `ReadWriteMany` access mode, claimed by a PersistentVolumeClaim and mounted in both pods. + ## API Reference The backend exposes CRUD endpoints for three models: diff --git a/backend/Dockerfile b/backend/Dockerfile index 1a14197..8b100de 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -9,5 +9,11 @@ RUN pip install --no-cache-dir -r requirements.txt COPY . . +# GeoLite2 City database for IP geolocation +# Download from https://dev.maxmind.com/geoip/geolite2-free-geolocation-data +# and place at backend/geo/GeoLite2-City.mmdb before building +RUN mkdir -p /app/geo +COPY geo/GeoLite2-City.mmdb /app/geo/GeoLite2-City.mmdb 2>/dev/null || true + EXPOSE 8000 CMD ["gunicorn", "app.main:app", "-k", "uvicorn.workers.UvicornWorker", "-b", "0.0.0.0:8000", "--workers", "2", "--timeout", "120"] diff --git a/backend/app/__pycache__/config.cpython-312.pyc b/backend/app/__pycache__/config.cpython-312.pyc index 7a00907..c33ab09 100644 Binary files a/backend/app/__pycache__/config.cpython-312.pyc and b/backend/app/__pycache__/config.cpython-312.pyc differ diff --git a/backend/app/__pycache__/log_parser.cpython-312.pyc b/backend/app/__pycache__/log_parser.cpython-312.pyc new file mode 100644 index 0000000..91114cc Binary files /dev/null and b/backend/app/__pycache__/log_parser.cpython-312.pyc differ diff --git a/backend/app/__pycache__/main.cpython-312.pyc b/backend/app/__pycache__/main.cpython-312.pyc index 0bcc23a..f639758 100644 Binary files a/backend/app/__pycache__/main.cpython-312.pyc and b/backend/app/__pycache__/main.cpython-312.pyc differ diff --git a/backend/app/__pycache__/models.cpython-312.pyc b/backend/app/__pycache__/models.cpython-312.pyc index 2b33ce1..2706e0b 100644 Binary files a/backend/app/__pycache__/models.cpython-312.pyc and b/backend/app/__pycache__/models.cpython-312.pyc differ diff --git a/backend/app/__pycache__/schemas.cpython-312.pyc b/backend/app/__pycache__/schemas.cpython-312.pyc index 4394e61..4fa7891 100644 Binary files a/backend/app/__pycache__/schemas.cpython-312.pyc and b/backend/app/__pycache__/schemas.cpython-312.pyc differ diff --git a/backend/app/api/__pycache__/stats.cpython-312.pyc b/backend/app/api/__pycache__/stats.cpython-312.pyc new file mode 100644 index 0000000..482bca4 Binary files /dev/null and b/backend/app/api/__pycache__/stats.cpython-312.pyc differ diff --git a/backend/app/api/stats.py b/backend/app/api/stats.py new file mode 100644 index 0000000..3d29133 --- /dev/null +++ b/backend/app/api/stats.py @@ -0,0 +1,187 @@ +"""Stats API: trigger log parsing and query aggregated visitor data.""" + +import asyncio +from datetime import date, timedelta +from typing import AsyncGenerator, Optional + +from fastapi import APIRouter, Depends, Query, HTTPException, status +from fastapi.responses import StreamingResponse +from sqlalchemy import desc, func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.auth import decode_token, get_current_admin_user +from app.database import get_session +from app.log_parser import process_unparsed_logs +from app.models import LogParseState, VisitorStatDaily, VisitorStatGeo +from app.user_models import User + +router = APIRouter() + + +# ── Helpers ──────────────────────────────────────────────────── + +def _default_date_range() -> tuple[date, date]: + """Return (start_date, end_date) for last 30 days.""" + end = date.today() - timedelta(days=1) # today isn't parsed yet + start = end - timedelta(days=29) + return start, end + + +async def _verify_admin_from_token(token: str) -> User: + """Verify admin from a raw JWT token string (for SSE query-param auth).""" + from app.database import async_session + from sqlalchemy import select as sa_select + + payload = decode_token(token) + user_id = int(payload["sub"]) + if not payload.get("is_admin"): + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required") + + async with async_session() as session: + result = await session.execute(sa_select(User).where(User.id == user_id)) + user = result.scalar_one_or_none() + if user is None: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="User not found") + return user + + +# ── SSE parse endpoint ─────────────────────────────────────── + +@router.get("/parse") +async def trigger_parse(token: Optional[str] = Query(None)): + """Trigger log file parsing. Streams progress via Server-Sent Events. + + Accepts JWT via `token` query parameter (EventSource can't send headers). + """ + # Authenticate — SSE can't send Authorization headers, so use query param + if token: + await _verify_admin_from_token(token) + else: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Token required") + + async def event_generator() -> AsyncGenerator[str, None]: + # Collect progress events, then yield them + events: list[tuple[str, int, int]] = [] + result = await process_unparsed_logs(lambda fd, c, t: events.append((fd, c, t))) + + for file_date, current, total in events: + yield f'event: progress\ndata: {{"file": "{file_date}", "current": {current}, "total": {total}}}\n\n' + + yield f'event: done\ndata: {{"files_processed": {result["files_processed"]}, "rows_inserted": {result["rows_inserted"]}}}\n\n' + + return StreamingResponse( + event_generator(), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "X-Accel-Buffering": "no", # Disable nginx buffering for SSE + }, + ) + + +# ── Summary ──────────────────────────────────────────────────── + +@router.get("/summary") +async def get_summary( + start_date: Optional[date] = Query(None), + end_date: Optional[date] = Query(None), + session: AsyncSession = Depends(get_session), + _: User = Depends(get_current_admin_user), +): + """Get total unique visitors for a date range.""" + if start_date is None or end_date is None: + start_date, end_date = _default_date_range() + + result = await session.execute( + select(func.coalesce(func.sum(VisitorStatDaily.unique_visitors), 0)).where( + VisitorStatDaily.date >= start_date, + VisitorStatDaily.date <= end_date, + ) + ) + total_unique = result.scalar() + + return {"total_unique_visitors": total_unique} + + +# ── Time series ──────────────────────────────────────────────── + +@router.get("/timeseries") +async def get_timeseries( + start_date: Optional[date] = Query(None), + end_date: Optional[date] = Query(None), + session: AsyncSession = Depends(get_session), + _: User = Depends(get_current_admin_user), +): + """Get daily unique visitor counts for a date range.""" + if start_date is None or end_date is None: + start_date, end_date = _default_date_range() + + stmt = ( + select( + VisitorStatDaily.date, + VisitorStatDaily.unique_visitors, + ) + .where( + VisitorStatDaily.date >= start_date, + VisitorStatDaily.date <= end_date, + ) + .order_by(VisitorStatDaily.date) + ) + result = await session.execute(stmt) + rows = result.all() + return [ + {"date": row.date.isoformat(), "unique_visitors": row.unique_visitors} + for row in rows + ] + + +# ── Geo ──────────────────────────────────────────────────────── + +@router.get("/geo") +async def get_geo_stats( + start_date: Optional[date] = Query(None), + end_date: Optional[date] = Query(None), + session: AsyncSession = Depends(get_session), + _: User = Depends(get_current_admin_user), +): + """Get geographic distribution of visitors for a date range. + + Aggregates by country (country, country_code, lat, lon), summing visitor_count. + Returns one row per country/region for the map to render. + """ + if start_date is None or end_date is None: + start_date, end_date = _default_date_range() + + stmt = ( + select( + VisitorStatGeo.country, + VisitorStatGeo.country_code, + VisitorStatGeo.latitude, + VisitorStatGeo.longitude, + func.sum(VisitorStatGeo.visitor_count).label("total_visitors"), + ) + .where( + VisitorStatGeo.date >= start_date, + VisitorStatGeo.date <= end_date, + ) + .group_by( + VisitorStatGeo.country, + VisitorStatGeo.country_code, + VisitorStatGeo.latitude, + VisitorStatGeo.longitude, + ) + .order_by(desc(func.sum(VisitorStatGeo.visitor_count))) + ) + result = await session.execute(stmt) + rows = result.all() + return [ + { + "country": row.country or "Unknown", + "country_code": row.country_code or "", + "latitude": row.latitude, + "longitude": row.longitude, + "visitors": row.total_visitors, + } + for row in rows + ] diff --git a/backend/app/config.py b/backend/app/config.py index 8b04a71..fe79eea 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -14,6 +14,10 @@ class Settings(BaseSettings): JWT_SECRET_KEY: str = "change-me-in-production" JWT_EXPIRE_MINUTES: int = 1440 # 24 hours + # Visitor stats + LOGS_DIR: str = "/logs" + GEOLITE2_DB_PATH: str = "/app/geo/GeoLite2-City.mmdb" + model_config = {"env_prefix": "KMTN_"} diff --git a/backend/app/log_parser.py b/backend/app/log_parser.py new file mode 100644 index 0000000..03c55c0 --- /dev/null +++ b/backend/app/log_parser.py @@ -0,0 +1,317 @@ +"""Nginx access log parser with IP geolocation enrichment.""" + +import asyncio +import json +import os +from datetime import date, datetime +from typing import Callable, Optional + +import maxminddb + +from app.config import settings +from app.database import async_session +from app.models import LogParseState, VisitorStatDaily, VisitorStatGeo + +# ── Geo reader ────────────────────────────────────────────── + +_geo_reader: Optional[maxminddb.Reader] = None + + +def get_geo_reader() -> maxminddb.Reader: + """Return a singleton MaxMind GeoLite2 City reader.""" + global _geo_reader + if _geo_reader is None: + db_path = settings.GEOLITE2_DB_PATH + if os.path.exists(db_path): + _geo_reader = maxminddb.open_database(db_path) + else: + raise FileNotFoundError( + f"GeoLite2 database not found at {db_path}. " + "Download from https://dev.maxmind.com/geoip/geolite2-free-geolocation-data " + "and place it at backend/geo/GeoLite2-City.mmdb" + ) + return _geo_reader + + +# Country centroid fallback for free GeoLite2 (no city/coordinates). +# Approximate center coordinates per ISO country code — enough for map markers. +_COUNTRY_CENTROIDS: dict[str, tuple[float, float]] = { + "US": (39.8, -98.5), + "GB": (53.5, -1.5), + "DE": (51.2, 10.4), + "FR": (46.6, 2.3), + "JP": (36.2, 138.3), + "AU": (-25.3, 133.8), + "BR": (-14.2, -51.9), + "CA": (56.1, -106.3), + "IN": (20.6, 79.0), + "KR": (35.9, 127.8), + "NL": (52.1, 5.3), + "MX": (23.6, -102.5), + "IT": (41.9, 12.6), + "ES": (40.5, -3.7), + "SE": (62.0, 15.0), + "PL": (51.9, 19.1), + "AR": (-38.4, -63.6), + "ZA": (-30.6, 22.9), + "EG": (26.8, 30.8), + "TR": (39.0, 35.2), + "RU": (61.5, 105.3), + "SG": (1.35, 103.8), + "NZ": (-40.9, 174.9), + "CO": (4.6, -74.3), + "CL": (-35.7, -71.5), + "PE": (-9.2, -75.0), + "PH": (12.9, 121.8), + "ID": (-0.8, 113.9), + "TH": (15.9, 100.9), + "VN": (14.1, 108.3), + "MY": (4.2, 101.9), + "AE": (23.4, 53.8), + "SA": (23.9, 45.1), + "IL": (31.0, 34.9), + "NO": (60.5, 8.5), + "DK": (56.3, 9.5), + "FI": (64.1, 26.5), + "BE": (50.5, 4.5), + "AT": (47.5, 14.6), + "CH": (46.8, 8.2), + "PT": (39.4, -8.2), + "GR": (39.1, 21.8), + "CZ": (49.8, 15.5), + "RO": (45.9, 24.9), + "HU": (47.2, 19.5), + "UA": (48.4, 31.2), + "NG": (9.1, 8.7), + "KE": (-0.02, 37.9), + "GH": (7.9, -1.0), + "CN": (35.9, 104.2), + "PK": (30.4, 69.3), + "BD": (23.7, 90.4), +} + + +def lookup_geo(ip: str) -> dict: + """Look up geolocation for an IP address. + + Returns dict with country, country_code, city, latitude, longitude + or empty dict if lookup fails. Falls back to country centroids + when the free GeoLite2 DB lacks city/coordinate data. + """ + try: + reader = get_geo_reader() + result = reader.get(ip) + if result is None: + return {} + country = result.get("country", {}) + city = result.get("city", {}) + location = result.get("location", {}) + country_code = country.get("iso_code") + lat = location.get("latitude") + lon = location.get("longitude") + + # Free GeoLite2 City often lacks city/coordinates — use centroid fallback + if lat is None or lon is None: + centroid = _COUNTRY_CENTROIDS.get(country_code or "") + if centroid: + lat = lat if lat is not None else centroid[0] + lon = lon if lon is not None else centroid[1] + + return { + "country": country.get("names", {}).get("en", country_code or ""), + "country_code": country_code, + "city": city.get("names", {}).get("en", ""), + "latitude": lat, + "longitude": lon, + } + except Exception: + return {} + + +# ── IP resolution ─────────────────────────────────────────── + + +def resolve_client_ip(entry: dict) -> str: + """Extract the real client IP from a log entry. + + Prefers the leftmost X-Forwarded-For entry (production behind proxy), + falls back to remote_addr (dev). + """ + xff = entry.get("x_forwarded_for", "") + if xff and xff != "-": + # Leftmost entry is the original client + return xff.split(",")[0].strip() + return entry.get("remote_addr", "") + + +# ── Log parsing ───────────────────────────────────────────── + + +def parse_log_line(line: str) -> Optional[dict]: + """Parse a single JSON log line. Returns dict or None if unparseable.""" + line = line.strip() + if not line: + return None + try: + return json.loads(line) + except (json.JSONDecodeError, ValueError): + return None + + +def parse_log_file(filepath: str) -> dict: + """Parse a single daily log file and return aggregated counts. + + Returns a dict with: + - unique_ips: set of unique client IP addresses + - geo_counts: dict of (country_code, country, city, lat, lon) -> count + """ + unique_ips: set[str] = set() + geo_counts: dict[tuple, int] = {} + + with open(filepath, "r") as f: + for line in f: + entry = parse_log_line(line) + if entry is None: + continue + + ip = resolve_client_ip(entry) + if not ip: + continue + + unique_ips.add(ip) + + # Geo enrichment + geo = lookup_geo(ip) + if geo.get("country_code"): + geo_key = ( + geo.get("country_code", ""), + geo.get("country", ""), + geo.get("city", ""), + geo.get("latitude"), + geo.get("longitude"), + ) + geo_counts[geo_key] = geo_counts.get(geo_key, 0) + 1 + + return { + "unique_ips": unique_ips, + "geo_counts": geo_counts, + } + + +async def process_unparsed_logs( + progress_callback: Optional[Callable] = None, +) -> dict: + """Scan /logs/ for unprocessed daily files, parse them, store aggregates. + + Only processes files strictly older than today (file must be complete). + Advances the LogParseState marker after each successful file. + + Returns a summary dict with files_processed and rows_inserted. + """ + logs_dir = settings.LOGS_DIR + if not os.path.isdir(logs_dir): + return {"files_processed": 0, "error": f"Logs directory not found: {logs_dir}"} + + # Get current parse state + async with async_session() as session: + from sqlalchemy import select + + result = await session.execute( + select(LogParseState).where(LogParseState.key == "default") + ) + state = result.scalar_one_or_none() + + if state is None: + # Initialize with a far-past date so all files get processed + state = LogParseState(key="default", last_parsed_date=date(2020, 1, 1)) + session.add(state) + await session.commit() + + last_parsed = state.last_parsed_date + + # Discover daily log files + import glob + + pattern = os.path.join(logs_dir, "access-*.log") + log_files = sorted(glob.glob(pattern)) + + files_processed = 0 + rows_inserted = 0 + today = date.today() + + for filepath in log_files: + # Extract date from filename + basename = os.path.basename(filepath) + try: + date_str = basename.replace("access-", "").replace(".log", "") + file_date = date.fromisoformat(date_str) + except ValueError: + continue + + # Only process files strictly before today (file must be complete) + if file_date >= today: + continue + + # Only process files after the last parsed date + if file_date <= last_parsed: + continue + + # Parse this file + try: + stats = parse_log_file(filepath) + except Exception as e: + print(f" ERROR parsing {filepath}: {e}") + continue + + # Store aggregates in database + async with async_session() as session: + from sqlalchemy import select + + # Upsert daily stats + result = await session.execute( + select(VisitorStatDaily).where(VisitorStatDaily.date == file_date) + ) + daily = result.scalar_one_or_none() + if daily is None: + daily = VisitorStatDaily(date=file_date) + session.add(daily) + + daily.unique_visitors = len(stats["unique_ips"]) + + # Geo stats + for geo_key, count in stats["geo_counts"].items(): + geo_row = VisitorStatGeo( + date=file_date, + country_code=geo_key[0], + country=geo_key[1], + city=geo_key[2], + latitude=geo_key[3], + longitude=geo_key[4], + visitor_count=count, + ) + session.add(geo_row) + rows_inserted += 1 + + # Advance parse state + state = ( + await session.execute( + select(LogParseState).where(LogParseState.key == "default") + ) + ).scalar_one() + state.last_parsed_date = file_date + state.updated_at = datetime.utcnow() + + await session.commit() + + files_processed += 1 + + # Report progress via callback (for SSE streaming) + if progress_callback: + cb_result = progress_callback( + file_date.isoformat(), files_processed, len(log_files) + ) + # Support both sync and async callbacks + if cb_result is not None and asyncio.iscoroutine(cb_result): + await cb_result + + return {"files_processed": files_processed, "rows_inserted": rows_inserted} diff --git a/backend/app/main.py b/backend/app/main.py index 1ac648a..e6564b6 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1,4 +1,6 @@ from contextlib import asynccontextmanager +import asyncio +import os from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware @@ -8,7 +10,7 @@ from app.config import settings from app.database import async_session, engine from app.models import Base, Show, StationConfig, HistoryEntry, TeamMember, CommunityHighlight, Underwriter from app.user_models import User -from app.api import events, auth, station_config, history, team, community, shows, storage, underwriters +from app.api import events, auth, station_config, history, team, community, shows, storage, underwriters, stats def _migrate_add_missing_columns(sync_conn): @@ -117,6 +119,27 @@ async def lifespan(app: FastAPI): await session.commit() print(f" → Bootstrap admin created: {settings.ADMIN_USERNAME}") + # Check GeoLite2 database + geo_db = settings.GEOLITE2_DB_PATH + if os.path.exists(geo_db): + print(f" ✓ GeoLite2 database loaded from {geo_db}") + else: + print(f" WARNING: GeoLite2 database not found at {geo_db} — geo lookups will be disabled") + + # Auto-parse any unprocessed logs on startup (background task) + async def _startup_parse(): + try: + from app.log_parser import process_unparsed_logs + result = await process_unparsed_logs() + if result.get("files_processed"): + print(f" ✓ Parsed {result['files_processed']} log files ({result['rows_inserted']} geo rows)") + else: + print(" No unprocessed log files to parse") + except Exception as e: + print(f" ERROR during startup log parse: {e}") + + asyncio.create_task(_startup_parse()) + yield @@ -145,6 +168,7 @@ app.include_router(community.router, prefix="/api/community", tags=["community"] app.include_router(shows.router, prefix="/api/shows", tags=["shows"]) app.include_router(storage.router, prefix="/api/storage", tags=["storage"]) app.include_router(underwriters.router, prefix="/api/underwriters", tags=["underwriters"]) +app.include_router(stats.router, prefix="/api/stats", tags=["stats"]) # Dev-only admin router (disabled in production) if settings.ENV != "production": diff --git a/backend/app/models.py b/backend/app/models.py index 12cb0f5..50d24f8 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -3,6 +3,7 @@ from datetime import datetime from sqlalchemy import ( Column, DateTime, + Float, Integer, LargeBinary, String, @@ -11,6 +12,7 @@ from sqlalchemy import ( Date, ForeignKey, UniqueConstraint, + Index, ) from sqlalchemy.orm import DeclarativeBase, relationship @@ -159,3 +161,47 @@ class StorageBlob(Base): size = Column(Integer, nullable=False) data = Column(LargeBinary, nullable=False) created_at = Column(DateTime, nullable=False, default=datetime.utcnow) + + +# ── Visitor Stats ────────────────────────────────────────────── + +class VisitorStatDaily(Base): + """Daily aggregated visitor statistics.""" + __tablename__ = "visitor_stats_daily" + + id = Column(Integer, primary_key=True, autoincrement=True) + date = Column(Date, nullable=False, unique=True) + unique_visitors = Column(Integer, nullable=False, default=0) + + __table_args__ = ( + Index("idx_visitor_stats_date", "date"), + ) + + +class VisitorStatGeo(Base): + """Geographic breakdown of visitors per day.""" + __tablename__ = "visitor_stats_geo" + + id = Column(Integer, primary_key=True, autoincrement=True) + date = Column(Date, nullable=False) + country = Column(String(100), nullable=True) + country_code = Column(String(10), nullable=True) + city = Column(String(200), nullable=True) + latitude = Column(Float, nullable=True) + longitude = Column(Float, nullable=True) + visitor_count = Column(Integer, nullable=False, default=0) + + __table_args__ = ( + Index("idx_visitor_stats_geo_date", "date"), + Index("idx_visitor_stats_geo_country", "country_code"), + ) + + +class LogParseState(Base): + """Tracks the last successfully parsed log date.""" + __tablename__ = "log_parse_state" + + id = Column(Integer, primary_key=True, autoincrement=True) + key = Column(String(20), unique=True, nullable=False, default="default") + last_parsed_date = Column(Date, nullable=False) + updated_at = Column(DateTime, nullable=False, default=datetime.utcnow) diff --git a/backend/app/schemas.py b/backend/app/schemas.py index f8515ae..6c80ce7 100644 --- a/backend/app/schemas.py +++ b/backend/app/schemas.py @@ -1,7 +1,7 @@ from datetime import date as _date from typing import Optional -from pydantic import BaseModel, computed_field, field_validator +from pydantic import BaseModel, computed_field # ── Shared ─────────────────────────────────────────────── @@ -284,3 +284,27 @@ class ShowUpdate(BaseModel): display_order: Optional[int] = None active: Optional[bool] = None schedules: Optional[list[ShowScheduleCreate]] = None + + +# ── Stats ──────────────────────────────────────────────────────── + +class StatsSummaryResponse(BaseModel): + total_unique_visitors: int + + +class TimeSeriesPoint(BaseModel): + date: str + unique_visitors: int + + +class GeoStatResponse(BaseModel): + country: str + country_code: str + latitude: Optional[float] = None + longitude: Optional[float] = None + visitors: int + + +class ParseResultResponse(BaseModel): + files_processed: int + rows_inserted: int diff --git a/backend/geo/README.md b/backend/geo/README.md new file mode 100644 index 0000000..f7e9281 --- /dev/null +++ b/backend/geo/README.md @@ -0,0 +1,15 @@ +# GeoLite2 City Database + +The visitor stats dashboard uses MaxMind GeoLite2 for IP geolocation. + +## Setup + +1. Create a free account at [MaxMind GeoLite2](https://dev.maxmind.com/geoip/geolite2-free-geolocation-data) +2. Download the **GeoLite2 City** database (`.mmdb` format, not CSV) +3. Place the downloaded file here as `GeoLite2-City.mmdb` + +The file is git-ignored (~60 MB, licensed). The Docker build copies it into the container at `/app/geo/GeoLite2-City.mmdb`. + +## Without the database + +The backend starts but geo lookups are disabled. The dashboard shows visitor counts without country/region breakdown. diff --git a/backend/inject_test_logs.py b/backend/inject_test_logs.py new file mode 100644 index 0000000..795ebd1 --- /dev/null +++ b/backend/inject_test_logs.py @@ -0,0 +1,169 @@ +"""Inject test visitor log entries into the shared logs directory. + +Writes JSON-formatted access log lines (matching the nginx kmtn_json format) +for multiple past days so the log parser will pick them up — data is only +aggregated a day in arrears, so all dates are strictly before today. + +Usage: + python inject_test_logs.py # writes to /logs (default) + python inject_test_logs.py /tmp/logs # writes to custom path + python inject_test_logs.py /logs 14 # writes 14 days of data +""" + +import json +import os +import sys +import hashlib +import struct +from datetime import date, timedelta, timezone + +# Real, well-known public IPs that resolve in MaxMind GeoLite2 City. +# These are actual infrastructure IPs (DNS, CDN, search, cloud) — +# not random addresses, so they're far more likely to be in the DB. +# Each tuple: (ip, expected_country, expected_city) +TEST_IPS = [ + # United States + ("8.8.8.8", "US", "Mountain View"), # Google DNS + ("8.8.4.4", "US", "Mountain View"), # Google DNS + ("216.58.214.206", "US", "Mountain View"), # Google + ("142.250.80.46", "US", "Mountain View"), # Google + ("13.107.42.14", "US", "Redmond"), # Microsoft + ("20.190.128.1", "US", "Boydton"), # Microsoft Azure + ("208.67.222.222", "US", "San Francisco"), # OpenDNS + ("208.67.220.220", "US", "San Francisco"), # OpenDNS + ("151.101.1.69", "US", "San Francisco"), # Fastly + ("52.84.74.1", "US", "Seattle"), # AWS CloudFront + ("99.86.0.1", "US", "Seattle"), # AWS + ("54.239.28.85", "US", "Seattle"), # AWS + ("185.199.108.1", "US", "San Francisco"), # GitHub Pages + # United Kingdom + ("212.58.244.3", "GB", "London"), # Virgin Media + ("178.173.57.1", "GB", "London"), # DigitalOcean + ("51.15.48.1", "GB", "London"), # OVH UK + # France + ("51.140.137.15", "FR", "Paris"), # OVH Paris + ("193.70.150.1", "FR", "Paris"), # Free + # Germany + ("217.160.0.1", "DE", "Frankfurt"), # Telekom + ("212.227.156.1", "DE", "Munich"), # Deutsche Telekom + ("217.69.70.1", "DE", "Frankfurt"), # Freenet + # Japan + ("210.170.232.46", "JP", "Tokyo"), # KDDI + ("202.214.69.1", "JP", "Tokyo"), # NTT + ("210.188.206.1", "JP", "Tokyo"), # Softbank + # Australia + ("1.35.128.1", "AU", "Sydney"), # Telstra + ("203.50.224.1", "AU", "Melbourne"), # iiNet + # Brazil + ("177.72.120.1", "BR", "Sao Paulo"), # Vivo + ("200.149.130.1", "BR", "Rio de Janeiro"), # Telefonica Brasil + ("189.75.1.1", "BR", "Sao Paulo"), # Claro + # Canada + ("76.69.248.1", "CA", "Toronto"), # Bell + ("24.199.12.1", "CA", "Vancouver"), # Shaw + ("99.248.1.1", "CA", "Montreal"), # Bell + # India + ("117.239.1.1", "IN", "Mumbai"), # Reliance Jio + ("180.151.1.1", "IN", "New Delhi"), # Airtel + ("106.192.1.1", "IN", "Bangalore"), # BSNL + # South Korea + ("1.235.198.1", "KR", "Seoul"), # KT + ("211.36.1.1", "KR", "Seoul"), # SK Telecom + # Netherlands + ("185.30.1.1", "NL", "Amsterdam"), # KPN + ("94.23.1.1", "NL", "Amsterdam"), # XS4ALL + # Mexico + ("189.203.1.1", "MX", "Mexico City"), # Telmex + ("200.44.1.1", "MX", "Guadalajara"), # Telcel + # Italy + ("217.20.1.1", "IT", "Rome"), # Telecom Italia + ("93.45.1.1", "IT", "Milan"), # TIM + # Spain + ("213.99.1.1", "ES", "Madrid"), # Telefonica Espana + ("88.0.1.1", "ES", "Madrid"), # Movistar + # Sweden + ("213.248.1.1", "SE", "Stockholm"), # Telia + # Poland + ("79.133.1.1", "PL", "Warsaw"), # Orange Polska + # Argentina + ("190.15.1.1", "AR", "Buenos Aires"), # Telecom Argentina + # South Africa + ("196.201.1.1", "ZA", "Johannesburg"), # Telkom SA + # Egypt + ("213.53.1.1", "EG", "Cairo"), # Telecom Egypt + # Turkey + ("37.146.1.1", "TR", "Istanbul"), # Turk Telekom + # Russia + ("77.88.55.1", "RU", "Moscow"), # Yandex DNS + ("5.255.255.1", "RU", "Moscow"), # Rostelecom + # Singapore + ("182.16.1.1", "SG", "Singapore"), # Singtel + # New Zealand + ("210.55.1.1", "NZ", "Auckland"), # Telecom NZ +] + + +def _deterministic_count(seed: str) -> int: + """Return a pseudo-random visit count (1-8) for a given seed.""" + h = hashlib.md5(seed.encode()).digest() + return struct.unpack("I", h[:4])[0] % 8 + 1 + + +def _deterministic_hour(seed: str) -> int: + """Return a pseudo-random hour (0-23) for a given seed.""" + h = hashlib.md5(seed.encode()).digest() + return struct.unpack("I", h[:4])[0] % 24 + + +def write_test_logs(logs_dir: str, days: int = 14): + """Write test log entries for the given number of past days. + + Skips today (parser only processes files strictly before today). + Writes a realistic mix of visitors per day with varying counts. + """ + os.makedirs(logs_dir, exist_ok=True) + + today = date.today() + total_entries = 0 + files_written = 0 + + # Write for the last `days` days, skipping today + for offset in range(1, days + 1): + file_date = today - timedelta(days=offset) + + filename = f"access-{file_date.isoformat()}.log" + filepath = os.path.join(logs_dir, filename) + + entries: list[str] = [] + for ip, country, city in TEST_IPS: + seed = f"{file_date}-{ip}" + count = _deterministic_count(seed) + hour = _deterministic_hour(seed) + + for j in range(count): + # Deterministic minute spread (hash() is non-deterministic across runs) + ip_hash = struct.unpack("I", hashlib.md5(ip.encode()).digest()[4:8])[0] + minute = (j * 7 + ip_hash % 60) % 60 + entry = { + "time": f"{file_date}T{hour}:{minute:02d}:00+00:00", + "remote_addr": ip, + "x_forwarded_for": "-", + } + entries.append(json.dumps(entry)) + + with open(filepath, "a") as f: + for line in entries: + f.write(line + "\n") + + total_entries += len(entries) + files_written += 1 + + print(f"Wrote {total_entries} log entries across {files_written} daily files ({len(TEST_IPS)} unique IPs)") + print(f"Date range: {(today - timedelta(days=days)).isoformat()} to {(today - timedelta(days=1)).isoformat()}") + print(f"Directory: {logs_dir}") + + +if __name__ == "__main__": + target = sys.argv[1] if len(sys.argv) > 1 else "/logs" + num_days = int(sys.argv[2]) if len(sys.argv) > 2 else 14 + write_test_logs(target, days=num_days) diff --git a/backend/probe_geo.py b/backend/probe_geo.py new file mode 100644 index 0000000..74c41b2 --- /dev/null +++ b/backend/probe_geo.py @@ -0,0 +1,200 @@ +"""Probe the GeoLite2 database to find IPs that resolve to real countries. + +Run inside the api container: + docker compose exec api python /app/probe_geo.py + +Or locally if the DB is present: + python probe_geo.py /path/to/GeoLite2-City.mmdb + +Tests a curated list of well-known public IPs and prints the ones that resolve. +""" + +import sys +import json +import os +import maxminddb + +# Well-known public IPs from various countries +CANDIDATE_IPS = [ + # United States + ("23.21.30.100", "US"), + ("72.14.198.234", "US"), + ("68.49.71.1", "US"), + ("208.67.222.222", "US"), # OpenDNS + ("74.125.200.100", "US"), # Google + ("151.101.1.69", "US"), # Fastly + ("142.250.80.46", "US"), # Google + ("216.58.214.206", "US"), # Google + ("172.217.164.110", "US"), # Google + ("13.107.42.14", "US"), # Microsoft + ("20.190.128.1", "US"), # Microsoft + ("52.84.74.1", "US"), # AWS CloudFront + ("99.86.0.1", "US"), # AWS + # United Kingdom + ("81.2.69.142", "GB"), + ("212.58.224.10", "GB"), + ("51.140.137.15", "GB"), + ("178.173.57.1", "GB"), + # France + ("51.140.137.15", "FR"), + ("193.70.150.1", "FR"), + # Japan + ("210.170.232.46", "JP"), + ("133.244.200.1", "JP"), + ("202.214.69.1", "JP"), # NTT + ("210.188.206.1", "JP"), # Softbank + # Germany + ("217.160.0.1", "DE"), + ("212.227.156.1", "DE"), + ("2a02:2e0::1", "DE"), # IPv6 - may not resolve + ("217.69.70.1", "DE"), + # Australia + ("101.168.100.1", "AU"), + ("203.50.224.1", "AU"), + ("1.35.128.1", "AU"), # Telstra + # Brazil + ("177.72.120.1", "BR"), + ("200.149.130.1", "BR"), + ("189.75.1.1", "BR"), + # Canada + ("76.69.248.1", "CA"), + ("24.199.12.1", "CA"), + ("99.248.1.1", "CA"), # Bell + # India + ("117.239.1.1", "IN"), + ("180.151.1.1", "IN"), + ("106.192.1.1", "IN"), + # South Korea + ("1.235.198.1", "KR"), + ("211.36.1.1", "KR"), + # Netherlands + ("185.30.1.1", "NL"), + ("94.23.1.1", "NL"), + # Mexico + ("189.203.1.1", "MX"), + ("200.44.1.1", "MX"), + # Italy + ("217.20.1.1", "IT"), + ("93.45.1.1", "IT"), + # Spain + ("213.99.1.1", "ES"), + ("88.0.1.1", "ES"), + # Sweden + ("213.248.1.1", "SE"), + # Poland + ("79.133.1.1", "PL"), + # Argentina + ("190.15.1.1", "AR"), + # South Africa + ("196.201.1.1", "ZA"), + # Egypt + ("213.53.1.1", "EG"), + # Turkey + ("37.146.1.1", "TR"), + # Russia + ("77.88.55.1", "RU"), # Yandex DNS + ("5.255.255.1", "RU"), +] + + +def probe(db_path: str): + """Test each candidate IP against the GeoLite2 database.""" + reader = maxminddb.open_database(db_path) + results = [] + + # Deduplicate IPs but keep all expected countries + seen_ips = set() + for ip, expected_country in CANDIDATE_IPS: + if ip in seen_ips: + continue + seen_ips.add(ip) + + try: + result = reader.lookup(ip) + if result is None: + continue + + country = result.get("country", {}) + city = result.get("city", {}) + location = result.get("location", {}) + + country_code = country.get("iso_code", "") + country_name = country.get("names", {}).get("en", country_code) + city_name = city.get("names", {}).get("en", "") + lat = location.get("latitude") + lon = location.get("longitude") + + results.append({ + "ip": ip, + "country": country_name, + "country_code": country_code, + "city": city_name, + "latitude": lat, + "longitude": lon, + }) + except Exception as e: + print(f" ERROR looking up {ip}: {e}") + + reader.close() + return results + + +def main(): + # Determine DB path + db_path = None + + if len(sys.argv) > 1: + # Explicit path provided + db_path = sys.argv[1] + else: + # Try common locations + candidates = [ + "/app/geo/GeoLite2-City.mmdb", # container path + os.path.join(os.path.dirname(__file__), "geo", "GeoLite2-City.mmdb"), # local + ] + for p in candidates: + if os.path.exists(p): + db_path = p + break + + if db_path is None: + print("GeoLite2-City.mmdb not found.") + print("Usage:") + print(" docker compose exec api python /app/probe_geo.py") + print(" python probe_geo.py /path/to/GeoLite2-City.mmdb") + sys.exit(1) + + print(f"Probing GeoLite2 at: {db_path}") + results = probe(db_path) + + if not results: + print("No IPs resolved! The database might be empty or outdated.") + sys.exit(1) + + print(f"\nResolved {len(results)} IPs:\n") + + # Group by country + by_country = {} + for r in results: + by_country.setdefault(r["country_code"], []).append(r) + + for cc in sorted(by_country.keys()): + entries = by_country[cc] + print(f" {cc} ({entries[0]['country']}):") + for e in entries: + print(f" {e['ip']:20s} -> {e['city'] or 'N/A':20s} ({e['latitude']}, {e['longitude']})") + + # Output as Python tuples for inject_test_logs.py + print(f"\n\nTuples for inject_test_logs.py TEST_IPS:") + print("TEST_IPS = [") + for r in results: + print(f' ("{r["ip"]}", "{r["country"]}", "{r["city"]}"),') + print("]") + + # Also output JSON for convenience + print(f"\nJSON output:") + print(json.dumps(results, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/backend/requirements.txt b/backend/requirements.txt index f37a45f..2bf94a6 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -11,3 +11,4 @@ pydantic==2.10.4 pydantic-settings==2.7.1 python-jose[cryptography]==3.3.0 httpx==0.27.2 +maxminddb==2.6.2 diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 1ec448a..79da340 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -10,6 +10,8 @@ services: KMTN_ENV: production # Override with your actual production domain(s) KMTN_CORS_ORIGINS: '["https://yourdomain.com"]' + volumes: + - logs:/logs # No port mapping — traffic arrives via reverse proxy / load balancer frontend: @@ -18,4 +20,6 @@ services: API_UPSTREAM: https://api.yourdomain.com # Browser-facing API URL. Empty = proxy mode. Set to direct URL if ingress breaks proxy chain. API_PUBLIC_URL: "" + volumes: + - logs:/logs # No port mapping — traffic arrives via reverse proxy / load balancer diff --git a/docker-compose.yml b/docker-compose.yml index a76cfb9..42f4104 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,6 +28,8 @@ services: KMTN_CORS_ORIGINS: '["http://localhost:4200", "http://localhost"]' ports: - "8000:8000" + volumes: + - logs:/logs depends_on: db: condition: service_healthy @@ -42,8 +44,11 @@ services: API_PUBLIC_URL: "" ports: - "4200:80" + volumes: + - logs:/logs depends_on: - api volumes: pgdata: + logs: diff --git a/nginx.conf.template b/nginx.conf.template index 59cf4c0..deec1ff 100644 --- a/nginx.conf.template +++ b/nginx.conf.template @@ -1,3 +1,18 @@ +# Map ISO timestamp to date-only string for daily log rotation +map $time_iso8601 $log_date { + ~^(?\d{4}-\d{2}-\d{2}) $date; + default "unknown"; +} + +log_format kmtn_json escape=json + '{' + '"time":"$time_iso8601",' + '"remote_addr":"$remote_addr",' + '"x_forwarded_for":"$http_x_forwarded_for"' + '}'; + +access_log /logs/access-$log_date.log kmtn_json; + server { listen 80; server_name _; diff --git a/package-lock.json b/package-lock.json index 118a06a..382cb78 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,12 +8,18 @@ "name": "radio-station", "version": "0.0.0", "dependencies": { + "@angular/cdk": "^21.2.0", "@angular/common": "^21.2.0", "@angular/compiler": "^21.2.0", "@angular/core": "^21.2.0", "@angular/forms": "^21.2.0", "@angular/platform-browser": "^21.2.0", "@angular/router": "^21.2.0", + "@types/leaflet": "^1.9.17", + "chart.js": "^4.4.0", + "leaflet": "^1.9.4", + "ng2-charts": "^10.0.0", + "ngx-leaflet": "^0.0.16", "rxjs": "~7.8.0", "tslib": "^2.3.0", "zone.js": "^0.16.2" @@ -415,6 +421,22 @@ } } }, + "node_modules/@angular/cdk": { + "version": "21.2.14", + "resolved": "https://registry.npmjs.org/@angular/cdk/-/cdk-21.2.14.tgz", + "integrity": "sha512-806REq/CLf37nEhmmd8Q+ILN8z/RVG2vk2n8YZ/4TdHpcBCi5ux4AxLbpMmduLwGPOzPagJ6ggRzE5fnX0rmcQ==", + "license": "MIT", + "dependencies": { + "parse5": "^8.0.0", + "tslib": "^2.3.0" + }, + "peerDependencies": { + "@angular/common": "^21.0.0 || ^22.0.0", + "@angular/core": "^21.0.0 || ^22.0.0", + "@angular/platform-browser": "^21.0.0 || ^22.0.0", + "rxjs": "^6.5.3 || ^7.4.0" + } + }, "node_modules/@angular/cli": { "version": "21.2.14", "resolved": "https://registry.npmjs.org/@angular/cli/-/cli-21.2.14.tgz", @@ -1821,6 +1843,12 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@kurkle/color": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/@kurkle/color/-/color-0.3.4.tgz", + "integrity": "sha512-M5UknZPHRu3DEDWoipU6sE8PdkZ6Z/S+v4dD+Ke8IaNlpdSQah50lz1KtcFBa2vsdOnwbbnxJwVM4wty6udA5w==", + "license": "MIT" + }, "node_modules/@listr2/prompt-adapter-inquirer": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/@listr2/prompt-adapter-inquirer/-/prompt-adapter-inquirer-3.0.5.tgz", @@ -3662,6 +3690,21 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/geojson": { + "version": "7946.0.16", + "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.16.tgz", + "integrity": "sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg==", + "license": "MIT" + }, + "node_modules/@types/leaflet": { + "version": "1.9.21", + "resolved": "https://registry.npmjs.org/@types/leaflet/-/leaflet-1.9.21.tgz", + "integrity": "sha512-TbAd9DaPGSnzp6QvtYngntMZgcRk+igFELwR2N99XZn7RXUdKgsXMR+28bUO0rPsWp8MIu/f47luLIQuSLYv/w==", + "license": "MIT", + "dependencies": { + "@types/geojson": "*" + } + }, "node_modules/@vitejs/plugin-basic-ssl": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/@vitejs/plugin-basic-ssl/-/plugin-basic-ssl-2.1.4.tgz", @@ -4063,6 +4106,18 @@ "dev": true, "license": "MIT" }, + "node_modules/chart.js": { + "version": "4.5.1", + "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.5.1.tgz", + "integrity": "sha512-GIjfiT9dbmHRiYi6Nl2yFCq7kkwdkp1W/lp2J99rX0yo9tgJGn3lKQATztIjb5tVtevcBtIdICNWqlq5+E8/Pw==", + "license": "MIT", + "dependencies": { + "@kurkle/color": "^0.3.0" + }, + "engines": { + "pnpm": ">=8" + } + }, "node_modules/chokidar": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-5.0.0.tgz", @@ -4557,6 +4612,16 @@ "node": ">= 0.4" } }, + "node_modules/es-toolkit": { + "version": "1.49.0", + "resolved": "https://registry.npmjs.org/es-toolkit/-/es-toolkit-1.49.0.tgz", + "integrity": "sha512-G5iZ6Pc/FNRY/soKZHC+TxGDD83rHUDXxzaWhGCX44vAv/tMs56WMusnm/KMNK+luUPsgA9U28cGr4RDlSzL2g==", + "license": "MIT", + "workspaces": [ + "docs", + "benchmarks" + ] + }, "node_modules/esbuild": { "version": "0.27.3", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.3.tgz", @@ -5379,6 +5444,12 @@ ], "license": "MIT" }, + "node_modules/leaflet": { + "version": "1.9.4", + "resolved": "https://registry.npmjs.org/leaflet/-/leaflet-1.9.4.tgz", + "integrity": "sha512-nxS1ynzJOmOlHp+iL3FyWqK89GtNL8U8rvlMOsQdTTssxZwCXh8N2NB3GDQOL+YR3XnWyZAxwQixURb+FA74PA==", + "license": "BSD-2-Clause" + }, "node_modules/listr2": { "version": "9.0.5", "resolved": "https://registry.npmjs.org/listr2/-/listr2-9.0.5.tgz", @@ -5908,6 +5979,38 @@ "node": ">= 0.6" } }, + "node_modules/ng2-charts": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/ng2-charts/-/ng2-charts-10.0.0.tgz", + "integrity": "sha512-mdL75XJrk/0s0YO2ySPQpAHPja85ECDEGNWFlcElJiy/bYliTNGEpeCtctAqZuozTff/E2CwGjyfPFM1ScP2og==", + "license": "MIT", + "dependencies": { + "es-toolkit": "^1.39.7", + "tslib": "^2.3.0" + }, + "peerDependencies": { + "@angular/cdk": ">=21.0.0", + "@angular/common": ">=21.0.0", + "@angular/core": ">=21.0.0", + "@angular/platform-browser": ">=21.0.0", + "chart.js": "^3.4.0 || ^4.0.0", + "rxjs": "^6.5.3 || ^7.4.0" + } + }, + "node_modules/ngx-leaflet": { + "version": "0.0.16", + "resolved": "https://registry.npmjs.org/ngx-leaflet/-/ngx-leaflet-0.0.16.tgz", + "integrity": "sha512-4cm06T9ZrD3i0VI+/fghrfx/Oe3UkOcYsDYKLFgh+XkoBvCt3oXIgdrABjoXIgMLcKtzS/oDHcVgRkDy/x5/MA==", + "dependencies": { + "tslib": "^2.3.0" + }, + "peerDependencies": { + "@angular/common": "*", + "@angular/core": "*", + "@types/leaflet": "^1.9.0", + "leaflet": "^1.9.3" + } + }, "node_modules/node-addon-api": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz", @@ -6276,7 +6379,6 @@ "version": "8.0.1", "resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.1.tgz", "integrity": "sha512-z1e/HMG90obSGeidlli3hj7cbocou0/wa5HacvI3ASx34PecNjNQeaHNo5WIZpWofN9kgkqV1q5YvXe3F0FoPw==", - "dev": true, "license": "MIT", "dependencies": { "entities": "^8.0.0" @@ -6330,7 +6432,6 @@ "version": "8.0.0", "resolved": "https://registry.npmjs.org/entities/-/entities-8.0.0.tgz", "integrity": "sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA==", - "dev": true, "license": "BSD-2-Clause", "engines": { "node": ">=20.19.0" diff --git a/package.json b/package.json index 78d8389..2233974 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,12 @@ "@angular/forms": "^21.2.0", "@angular/platform-browser": "^21.2.0", "@angular/router": "^21.2.0", + "@types/leaflet": "^1.9.17", + "@angular/cdk": "^21.2.0", + "chart.js": "^4.4.0", + "leaflet": "^1.9.4", + "ng2-charts": "^10.0.0", + "ngx-leaflet": "^0.0.16", "rxjs": "~7.8.0", "tslib": "^2.3.0", "zone.js": "^0.16.2" diff --git a/src/app/admin/admin-stats-dashboard.component.html b/src/app/admin/admin-stats-dashboard.component.html new file mode 100644 index 0000000..f330183 --- /dev/null +++ b/src/app/admin/admin-stats-dashboard.component.html @@ -0,0 +1,103 @@ +
+ +
+
+ @for (preset of presets; track preset.value) { + + } +
+ + @if (selectedPreset === 'custom') { +
+ + +
+ } + + +
+ + @if (parseProgress) { +
{{ parseProgress }}
+ } + + @if (error) { +
{{ error }}
+ } + + @if (loading) { +
Loading stats...
+ } @else { + +
+
+
{{ summary?.total_unique_visitors ?? 0 }}
+
Unique Visitors
+
+
+
{{ geoStats.length }}
+
Countries
+
+
+ + +
+

Visitors by Location

+
+
+
+
+ + +
+

Visitors Over Time

+
+ +
+
+ + +
+

Visitors by Country

+ + + + + + + + + + @for (geo of geoStats; track geo.country_code) { + + + + + + } @empty { + + } + +
CountryCodeVisitors
{{ geo.country }}{{ geo.country_code }}{{ geo.visitors }}
No geo data available yet
+
+ } +
\ No newline at end of file diff --git a/src/app/admin/admin-stats-dashboard.component.scss b/src/app/admin/admin-stats-dashboard.component.scss new file mode 100644 index 0000000..04ed9b4 --- /dev/null +++ b/src/app/admin/admin-stats-dashboard.component.scss @@ -0,0 +1,197 @@ +@use '../../styles/variables' as *; +@use '../../styles/mixins' as *; + +.stats-dashboard { + padding: $spacing-md; +} + +// ── Controls ────────────────────────────────────────────── + +.stats-controls { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: $spacing-md; + margin-bottom: $spacing-lg; +} + +.preset-buttons { + display: flex; + flex-wrap: wrap; + gap: $spacing-xs; +} + +.preset-btn { + background: $neutral-white; + border: 1px solid $neutral-light; + border-radius: $radius-md; + padding: $spacing-xs $spacing-md; + font-size: 0.85rem; + font-weight: 600; + color: $neutral-dark; + cursor: pointer; + transition: all $transition-fast; + + &:hover { + border-color: $primary-blue-muted; + color: $primary-blue; + } + + &.active { + background: $primary-blue; + border-color: $primary-blue; + color: $neutral-white; + } +} + +.custom-dates { + display: flex; + gap: $spacing-sm; + align-items: center; + + label { + font-size: 0.85rem; + color: $neutral-medium; + display: flex; + align-items: center; + gap: $spacing-xs; + } + + input[type="date"] { + padding: $spacing-xs $spacing-sm; + border: 1px solid $neutral-light; + border-radius: $radius-sm; + font-size: 0.85rem; + } +} + +.btn-parse { + @include button-style($success-green, $neutral-white, darken($success-green, 8%)); + margin-left: auto; +} + +.parse-progress { + background: rgba($info-blue, 0.1); + color: $info-blue; + padding: $spacing-sm $spacing-md; + border-radius: $radius-md; + font-size: 0.85rem; + margin-bottom: $spacing-md; +} + +.error-message { + background: rgba(#d32f2f, 0.1); + color: #d32f2f; + padding: $spacing-sm $spacing-md; + border-radius: $radius-md; + font-size: 0.85rem; + margin-bottom: $spacing-md; +} + +// ── Summary cards ───────────────────────────────────────── + +.stats-summary { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); + gap: $spacing-md; + margin-bottom: $spacing-lg; +} + +.stat-card { + @include card-style; + padding: $spacing-lg; + text-align: center; + + .stat-value { + font-family: $font-heading; + font-size: 2rem; + color: $primary-blue; + margin-bottom: $spacing-xs; + } + + .stat-label { + font-size: 0.85rem; + color: $neutral-medium; + text-transform: uppercase; + letter-spacing: 0.5px; + } +} + +// ── Map ─────────────────────────────────────────────────── + +.map-section { + @include card-style; + padding: $spacing-lg; + margin-bottom: $spacing-lg; + + h3 { + font-family: $font-heading; + color: $primary-blue; + margin: 0 0 $spacing-md; + } +} + +.map-container { + height: 400px; + border-radius: $radius-md; + overflow: hidden; + position: relative; + + .map-leaflet { + width: 100%; + height: 100%; + } +} + +// ── Chart ───────────────────────────────────────────────── + +.chart-section { + @include card-style; + padding: $spacing-lg; + margin-bottom: $spacing-lg; + + h3 { + font-family: $font-heading; + color: $primary-blue; + margin: 0 0 $spacing-md; + } +} + +.chart-container { + height: 300px; + position: relative; +} + +// ── Geo table ───────────────────────────────────────────── + +.geo-table-section { + @include card-style; + padding: $spacing-lg; + + h3 { + font-family: $font-heading; + color: $primary-blue; + margin: 0 0 $spacing-md; + } +} + +// ── Responsive ──────────────────────────────────────────── + +@include responsive(md) { + .stats-controls { + flex-direction: column; + align-items: flex-start; + } + + .btn-parse { + margin-left: 0; + } + + .map-container { + height: 280px; + } + + .chart-container { + height: 220px; + } +} \ No newline at end of file diff --git a/src/app/admin/admin-stats-dashboard.component.ts b/src/app/admin/admin-stats-dashboard.component.ts new file mode 100644 index 0000000..6c42012 --- /dev/null +++ b/src/app/admin/admin-stats-dashboard.component.ts @@ -0,0 +1,303 @@ +import { Component, OnInit, inject, AfterViewInit, OnDestroy, ViewChild, ElementRef, NgZone } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import { FormsModule } from '@angular/forms'; +import { Chart, registerables } from 'chart.js'; +import { BaseChartDirective } from 'ng2-charts'; +import L from 'leaflet'; + +import { StatsService } from '../services/stats.service'; +import { + StatsSummary, + TimeSeriesPoint, + GeoStat, + TimePreset, +} from '../interfaces/visitor-stat'; + +import { getAppConfig } from '../services/app-config.service'; + +// Register Chart.js components once +Chart.register(...registerables); + +@Component({ + selector: 'app-admin-stats-dashboard', + standalone: true, + imports: [CommonModule, FormsModule, BaseChartDirective], + templateUrl: './admin-stats-dashboard.component.html', + styleUrl: './admin-stats-dashboard.component.scss', +}) +export class AdminStatsDashboardComponent implements OnInit, AfterViewInit, OnDestroy { + private readonly statsService: StatsService = inject(StatsService); + private readonly ngZone = inject(NgZone); + + @ViewChild('mapContainer', { static: false }) mapContainerRef!: ElementRef; + + // ── Time range ────────────────────────────────────────── + readonly presets = [ + { label: 'Yesterday', value: 'yesterday' as TimePreset }, + { label: '7 Days', value: '7d' as TimePreset }, + { label: '30 Days', value: '30d' as TimePreset }, + { label: '90 Days', value: '90d' as TimePreset }, + { label: 'Year', value: 'year' as TimePreset }, + { label: 'Custom', value: 'custom' as TimePreset }, + ]; + selectedPreset: TimePreset = '30d'; + customStart = ''; + customEnd = ''; + + // ── Data ──────────────────────────────────────────────── + summary: StatsSummary | null = null; + timeSeries: TimeSeriesPoint[] = []; + geoStats: GeoStat[] = []; + loading = true; + error: string | null = null; + parsing = false; + parseProgress = ''; + + // ── Chart ─────────────────────────────────────────────── + chartData: { labels: string[]; datasets: any[] } = { labels: [], datasets: [] }; + chartOptions: any = { + responsive: true, + maintainAspectRatio: false, + plugins: { + legend: { display: true, position: 'top' as const }, + }, + scales: { + x: { title: { display: true, text: 'Date' } }, + y: { title: { display: true, text: 'Visitors' }, beginAtZero: true }, + }, + }; + + // ── Leaflet ───────────────────────────────────────────── + private map: L.Map | null = null; + private markersLayer: L.LayerGroup | null = null; + + ngOnInit(): void { + this.loadAll(); + } + + ngAfterViewInit(): void { + // Initialize Leaflet map + this.map = L.map(this.mapContainerRef.nativeElement, { + center: [20, 0], + zoom: 2, + zoomControl: true, + worldCopyJump: true, + }); + + L.tileLayer('https://{s}.basemaps.cartocdn.com/light_all/{z}/{x}/{y}{r}.png', { + attribution: '© OpenStreetMap contributors © CARTO', + maxZoom: 19, + }).addTo(this.map); + } + + ngOnDestroy(): void { + if (this.map) { + this.map.remove(); + this.map = null; + } + } + + // ── Date helpers ──────────────────────────────────────── + + private _localDateStr(d: Date): string { + // Format as YYYY-MM-DD using local time (no UTC shift) + const y = d.getFullYear(); + const m = String(d.getMonth() + 1).padStart(2, '0'); + const day = String(d.getDate()).padStart(2, '0'); + return `${y}-${m}-${day}`; + } + + getDates(): { start: string; end: string } { + const today = new Date(); + today.setHours(0, 0, 0, 0); + const yesterday = new Date(today); + yesterday.setDate(today.getDate() - 1); + + switch (this.selectedPreset) { + case 'yesterday': + return { start: this._localDateStr(yesterday), end: this._localDateStr(yesterday) }; + case '7d': { + const start = new Date(yesterday); + start.setDate(yesterday.getDate() - 6); + return { start: this._localDateStr(start), end: this._localDateStr(yesterday) }; + } + case '30d': { + const start = new Date(yesterday); + start.setDate(yesterday.getDate() - 29); + return { start: this._localDateStr(start), end: this._localDateStr(yesterday) }; + } + case '90d': { + const start = new Date(yesterday); + start.setDate(yesterday.getDate() - 89); + return { start: this._localDateStr(start), end: this._localDateStr(yesterday) }; + } + case 'year': { + const start = new Date(yesterday); + start.setDate(yesterday.getDate() - 364); + return { start: this._localDateStr(start), end: this._localDateStr(yesterday) }; + } + case 'custom': + return { + start: this.customStart || this._localDateStr(this._daysAgo(30)), + end: this.customEnd || this._localDateStr(yesterday), + }; + default: + return { start: this._localDateStr(yesterday), end: this._localDateStr(yesterday) }; + } + } + + private _daysAgo(n: number): Date { + const d = new Date(); + d.setHours(0, 0, 0, 0); + d.setDate(d.getDate() - n); + return d; + } + + onPresetChange(): void { + this.loadAll(); + } + + // ── Data loading ──────────────────────────────────────── + + async loadAll(): Promise { + this.loading = true; + this.error = null; + const { start, end } = this.getDates(); + try { + const [summary, timeSeries, geoStats] = await Promise.all([ + this._firstValue(this.statsService.getSummary(start, end)), + this._firstValue(this.statsService.getTimeSeries(start, end)), + this._firstValue(this.statsService.getGeoStats(start, end)), + ]); + + this.summary = summary; + this.timeSeries = timeSeries; + this.geoStats = geoStats; + + this._updateChart(); + this._updateMapMarkers(); + } catch (err) { + console.error('Failed to load stats:', err); + this.error = 'Failed to load stats data. Please check your connection and try again.'; + } finally { + this.loading = false; + } + } + + async triggerParse(): Promise { + this.parsing = true; + this.parseProgress = ''; + + const apiUrl = `${getAppConfig().apiBaseUrl}/api/stats/parse`; + // Append auth token as query param for SSE (EventSource can't send headers) + let token: string | null = null; + try { + token = localStorage.getItem('kmtn_access_token'); + } catch { + // localStorage may be blocked + } + const url = token ? `${apiUrl}?token=${encodeURIComponent(token)}` : apiUrl; + + return new Promise((resolve) => { + const es = new EventSource(url); + + es.addEventListener('progress', (e: MessageEvent) => { + const data = JSON.parse(e.data); + this.ngZone.run(() => { + this.parseProgress = `Parsing ${data.file}... ${data.current}/${data.total}`; + }); + }); + + es.addEventListener('done', (e: MessageEvent) => { + const data = JSON.parse(e.data); + es.close(); + this.ngZone.run(() => { + this.parseProgress = `Done — ${data.files_processed} files, ${data.rows_inserted} geo rows`; + this.parsing = false; + this.loadAll().then(() => resolve()); + }); + }); + + es.onerror = (e: Event) => { + console.error('Parse SSE error:', e); + es.close(); + this.ngZone.run(() => { + this.parseProgress = 'Parse failed'; + this.parsing = false; + resolve(); + }); + }; + }); + } + + private _firstValue(obs: import('rxjs').Observable): Promise { + return new Promise((resolve, reject) => { + obs.subscribe({ next: (v) => resolve(v), error: reject }); + }); + } + + // ── Chart ─────────────────────────────────────────────── + + private _updateChart(): void { + this.chartData = { + labels: this.timeSeries.map((p) => this._formatDate(p.date)), + datasets: [ + { + label: 'Unique Visitors', + data: this.timeSeries.map((p) => p.unique_visitors), + borderColor: '#1a3a5c', + backgroundColor: 'rgba(26, 58, 92, 0.1)', + fill: true, + tension: 0.3, + pointRadius: 2, + }, + ], + }; + } + + private _formatDate(dateStr: string): string { + const d = new Date(dateStr); + return `${d.getMonth() + 1}/${d.getDate()}`; + } + + // ── Map ───────────────────────────────────────────────── + + private _updateMapMarkers(): void { + if (!this.map) return; + + // Remove old markers + if (this.markersLayer) { + this.map.removeLayer(this.markersLayer); + } + + this.markersLayer = L.layerGroup(); + + const geoWithCoords = this.geoStats.filter((g) => g.latitude && g.longitude); + + if (geoWithCoords.length === 0) { + this.markersLayer.addTo(this.map); + return; + } + + const maxVisitors = Math.max(...geoWithCoords.map((g) => g.visitors), 1); + + geoWithCoords.forEach((geo) => { + const radius = Math.max(5, Math.min(30, 5 + (geo.visitors / maxVisitors) * 25)); + const marker = L.circleMarker([geo.latitude!, geo.longitude!], { + radius, + fillColor: '#e87a2e', + color: '#c85a1e', + weight: 1, + opacity: 0.8, + fillOpacity: 0.6, + }); + marker.bindTooltip(`${geo.country}: ${geo.visitors} visitors`, { + sticky: true, + direction: 'top', + }); + this.markersLayer?.addLayer(marker); + }); + + this.markersLayer.addTo(this.map); + } +} \ No newline at end of file diff --git a/src/app/admin/admin.component.html b/src/app/admin/admin.component.html index 35c6f42..ba51624 100644 --- a/src/app/admin/admin.component.html +++ b/src/app/admin/admin.component.html @@ -42,6 +42,11 @@ [class.active]="activeTab() === 'underwriters'" (click)="activeTab.set('underwriters')" >Underwriters + @@ -360,6 +365,13 @@ } } + + + @if (activeTab() === 'stats') { +
+ +
+ } diff --git a/src/app/admin/admin.component.ts b/src/app/admin/admin.component.ts index 330d518..3319372 100644 --- a/src/app/admin/admin.component.ts +++ b/src/app/admin/admin.component.ts @@ -24,8 +24,9 @@ import { AdminHistoryFormComponent } from './admin-history-form.component'; import { AdminTeamFormComponent } from './admin-team-form.component'; import { AdminCommunityFormComponent } from './admin-community-form.component'; import { AdminUnderwriterFormComponent } from './admin-underwriter-form.component'; +import { AdminStatsDashboardComponent } from './admin-stats-dashboard.component'; -type TabKey = 'shows' | 'events' | 'station' | 'history' | 'team' | 'community' | 'underwriters'; +type TabKey = 'shows' | 'events' | 'station' | 'history' | 'team' | 'community' | 'underwriters' | 'stats'; @Component({ selector: 'app-admin', @@ -41,6 +42,7 @@ type TabKey = 'shows' | 'events' | 'station' | 'history' | 'team' | 'community' AdminTeamFormComponent, AdminCommunityFormComponent, AdminUnderwriterFormComponent, + AdminStatsDashboardComponent, ], templateUrl: './admin.component.html', styleUrl: './admin.component.scss', diff --git a/src/app/interfaces/visitor-stat.ts b/src/app/interfaces/visitor-stat.ts new file mode 100644 index 0000000..f590527 --- /dev/null +++ b/src/app/interfaces/visitor-stat.ts @@ -0,0 +1,29 @@ +export interface StatsSummary { + total_unique_visitors: number; +} + +export interface TimeSeriesPoint { + date: string; + unique_visitors: number; +} + +export interface GeoStat { + country: string; + country_code: string; + latitude: number | null; + longitude: number | null; + visitors: number; +} + +export interface ParseProgress { + file: string; + current: number; + total: number; +} + +export interface ParseResult { + files_processed: number; + rows_inserted: number; +} + +export type TimePreset = 'yesterday' | '7d' | '30d' | '90d' | 'year' | 'custom'; diff --git a/src/app/services/stats.service.ts b/src/app/services/stats.service.ts new file mode 100644 index 0000000..33014b9 --- /dev/null +++ b/src/app/services/stats.service.ts @@ -0,0 +1,36 @@ +import { Injectable, inject } from '@angular/core'; +import { HttpClient } from '@angular/common/http'; +import { Observable } from 'rxjs'; + +import { getAppConfig } from './app-config.service'; +import { + StatsSummary, + TimeSeriesPoint, + GeoStat, +} from '../interfaces/visitor-stat'; + +@Injectable({ + providedIn: 'root', +}) +export class StatsService { + private http = inject(HttpClient); + private readonly baseUrl = `${getAppConfig().apiBaseUrl}/api/stats`; + + getSummary(startDate: string, endDate: string): Observable { + return this.http.get(`${this.baseUrl}/summary`, { + params: { start_date: startDate, end_date: endDate }, + }); + } + + getTimeSeries(startDate: string, endDate: string): Observable { + return this.http.get(`${this.baseUrl}/timeseries`, { + params: { start_date: startDate, end_date: endDate }, + }); + } + + getGeoStats(startDate: string, endDate: string): Observable { + return this.http.get(`${this.baseUrl}/geo`, { + params: { start_date: startDate, end_date: endDate }, + }); + } +} diff --git a/src/styles.scss b/src/styles.scss index 6c2e226..fcaa721 100644 --- a/src/styles.scss +++ b/src/styles.scss @@ -6,6 +6,9 @@ @use './styles/mixins' as *; @use './styles/themes'; +// Leaflet CSS for the visitor stats map +@import 'leaflet/dist/leaflet.css'; + // ===== Base reset and global styles ===== *,