From 77d1c04d86f1798481fd151ea240d1234e41dd83 Mon Sep 17 00:00:00 2001 From: kfj001 Date: Sun, 28 Jun 2026 06:41:23 +0000 Subject: [PATCH] Adds visitor dashboard and related features --- .claude/CLAUDE.md | 24 ++ .claude/settings.local.json | 26 +- .gitignore | 5 +- Dockerfile | 1 + README.md | 71 +++- backend/Dockerfile | 6 + .../app/__pycache__/config.cpython-312.pyc | Bin 1070 -> 1195 bytes .../__pycache__/log_parser.cpython-312.pyc | Bin 0 -> 12292 bytes backend/app/__pycache__/main.cpython-312.pyc | Bin 8801 -> 10234 bytes .../app/__pycache__/models.cpython-312.pyc | Bin 7577 -> 9506 bytes .../app/__pycache__/schemas.cpython-312.pyc | Bin 10494 -> 11469 bytes .../app/api/__pycache__/stats.cpython-312.pyc | Bin 0 -> 8862 bytes backend/app/api/stats.py | 187 +++++++++++ backend/app/config.py | 4 + backend/app/log_parser.py | 317 ++++++++++++++++++ backend/app/main.py | 26 +- backend/app/models.py | 46 +++ backend/app/schemas.py | 26 +- backend/geo/README.md | 15 + backend/inject_test_logs.py | 169 ++++++++++ backend/probe_geo.py | 200 +++++++++++ backend/requirements.txt | 1 + docker-compose.prod.yml | 4 + docker-compose.yml | 5 + nginx.conf.template | 15 + package-lock.json | 105 +++++- package.json | 6 + .../admin-stats-dashboard.component.html | 103 ++++++ .../admin-stats-dashboard.component.scss | 197 +++++++++++ .../admin/admin-stats-dashboard.component.ts | 303 +++++++++++++++++ src/app/admin/admin.component.html | 12 + src/app/admin/admin.component.ts | 4 +- src/app/interfaces/visitor-stat.ts | 29 ++ src/app/services/stats.service.ts | 36 ++ src/styles.scss | 3 + 35 files changed, 1938 insertions(+), 8 deletions(-) create mode 100644 backend/app/__pycache__/log_parser.cpython-312.pyc create mode 100644 backend/app/api/__pycache__/stats.cpython-312.pyc create mode 100644 backend/app/api/stats.py create mode 100644 backend/app/log_parser.py create mode 100644 backend/geo/README.md create mode 100644 backend/inject_test_logs.py create mode 100644 backend/probe_geo.py create mode 100644 src/app/admin/admin-stats-dashboard.component.html create mode 100644 src/app/admin/admin-stats-dashboard.component.scss create mode 100644 src/app/admin/admin-stats-dashboard.component.ts create mode 100644 src/app/interfaces/visitor-stat.ts create mode 100644 src/app/services/stats.service.ts diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index edb2297..65c1b12 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -26,6 +26,30 @@ See [README.md](README.md) for a full architecture map. Key files: - `backend/app/main.py` — FastAPI entry point (CORS, router registration) - `backend/app/models.py` — SQLAlchemy ORM (Show, Event, StationConfig) +### Admin dashboard — visitor stats (implemented) + +Nginx access logs → shared volume → FastAPI log parser → PostgreSQL → admin dashboard. No extra services. + +**How it works:** +1. Nginx writes structured JSON access logs to **daily log files** on a shared mount: `/logs/access-YYYY-MM-DD.log`. A `map` directive in [nginx.conf.template](nginx.conf.template) extracts the date from the ISO timestamp, so Nginx rotates to a new file at midnight. +2. FastAPI ([backend/app/log_parser.py](backend/app/log_parser.py)) maintains a **"last parsed date"** (at most yesterday) persisted in PostgreSQL via the `LogParseState` model. On each run it scans `/logs/` for unprocessed daily files, enriches entries with IP geolocation (MaxMind GeoLite2 City), and stores aggregates. +3. Aggregates are persisted in `visitor_stats_daily` (unique visitors per day) and `visitor_stats_geo` (country/city breakdown) tables. +4. Admin dashboard (Angular) reads from stats API endpoints in [backend/app/api/stats.py](backend/app/api/stats.py). + +**Key files:** +- `backend/app/log_parser.py` — log file parsing + geo enrichment + `process_unparsed_logs()` +- `backend/app/api/stats.py` — stats API endpoints (summary, timeseries, geo, parse trigger) +- `backend/app/models.py` — `VisitorStatDaily`, `VisitorStatGeo`, `LogParseState` ORM models +- `backend/inject_test_logs.py` — test data injector (writes test log entries for yesterday) +- `nginx.conf.template` — JSON log format (`kmtn_json`) with daily rotation via `$log_date` map +- `backend/geo/GeoLite2-City.mmdb` — MaxMind geolocation database + +**Why daily files:** The currently-being-written file is never read — no partial lines, no race conditions, no file locking. Stats have a minimum one-day lag. + +**Deployment context:** Single-node minikube, `hostPath` PersistentVolume with `ReadWriteMany`, or docker-compose named volume mounted in both pods. + +**Injecting test data:** Run `backend/inject_test_logs.py` (or the inline docker compose command in README.md) to write test log entries for yesterday, then trigger parsing via `GET /api/parse`. + ## Common tasks - **Re-skin:** Edit colors in `_variables.scss` only. diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 635f67b..7d08af0 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -5,7 +5,31 @@ "Read(*)", "Glob(*)", "Grep(*)", - "WebFetch(domain:www.kryzradio.org)" + "WebFetch(domain:www.kryzradio.org)", + "Bash(ls -la /logs/ 2>/dev/null || echo \"DIRECTORY /logs DOES NOT EXIST\")", + "Read(//logs/**)", + "Bash(head -5 /logs/access-2026-06-26.log)", + "Bash(python3 -c \"import sys, json; ips = set\\(\\); [ips.add\\(json.loads\\(l\\)['remote_addr']\\) for l in sys.stdin if l.strip\\(\\)]; print\\(f'Total unique IPs: {len\\(ips\\)}'\\); print\\('\\\\n'.join\\(sorted\\(ips\\)\\)\\)\")", + "Bash(python3 -c \"from datetime import date; print\\(f'Today: {date.today\\(\\)}'\\); print\\(f'Log files from Jun 13-27 are all before today: {date\\(2026, 6, 27\\) < date.today\\(\\)}'\\)\")", + "Bash(ls -la /logs/ 2>/dev/null || echo \"/logs/ not accessible\")", + "Bash(python3 -c \"import sys,json; ips=set\\(\\); [ips.add\\(json.loads\\(l\\)['remote_addr']\\) for l in sys.stdin if l.strip\\(\\)]; print\\(f'Unique IPs in 06-26: {len\\(ips\\)}'\\); [print\\(f' {ip}'\\) for ip in sorted\\(ips\\)]\")", + "Bash(python3 -c ' *)", + "Bash(docker exec *)", + "Bash(curl -s http://localhost:8000/api/stats/summary)", + "Bash(python3 -m json.tool)", + "Bash(curl -s \"http://localhost:8000/api/stats/timeseries?start_date=2026-06-13&end_date=2026-06-27\")", + "Bash(curl -s \"http://localhost:8000/api/stats/geo?start_date=2026-06-13&end_date=2026-06-27\")", + "Bash(env)", + "Bash(curl -s -X POST http://localhost:8000/api/auth/login -H \"Content-Type: application/json\" -d '{\"username\":\"admin\",\"password\":\"123\"}')", + "Bash(curl -s http://localhost:8000/api/stats/summary -H \"Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxIiwiaXNfYWRtaW4iOnRydWUsImV4cCI6MTc4MjcxMTYyOX0.8eM-Vd0H72R8eVomEtxLNeYSYUe65-0WPmNWUzx_mxw\")", + "Bash(curl -s \"http://localhost:8000/api/stats/timeseries?start_date=2026-06-13&end_date=2026-06-27\" -H \"Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxIiwiaXNfYWRtaW4iOnRydWUsImV4cCI6MTc4MjcxMTYyOX0.8eM-Vd0H72R8eVomEtxLNeYSYUe65-0WPmNWUzx_mxw\")", + "Bash(curl -s \"http://localhost:8000/api/stats/geo?start_date=2026-06-13&end_date=2026-06-27\" -H \"Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxIiwiaXNfYWRtaW4iOnRydWUsImV4cCI6MTc4MjcxMTYyOX0.8eM-Vd0H72R8eVomEtxLNeYSYUe65-0WPmNWUzx_mxw\")", + "Bash(python3 -c \"import maxminddb; print\\(maxminddb.__version__\\); print\\(dir\\(maxminddb.Reader\\)\\)\")", + "Bash(cat /tmp/uvicorn.log)", + "Read(//tmp/**)", + "Bash(KMTN_DATABASE_URL=\"sqlite+aiosqlite:///./kmountain.db\" KMTN_ADMIN_USERNAME=\"admin\" KMTN_ADMIN_PASSWORD=\"123\" KMTN_CORS_ORIGINS='[\"http://localhost:4200\"]' nohup uvicorn app.main:app --reload --host 0.0.0.0 --port 8000)", + "Bash(echo \"Backend PID: $!\")", + "Bash(curl -s http://localhost:8000/docs)" ], "additionalDirectories": [ "/workspaces/web_app/.vscode" diff --git a/.gitignore b/.gitignore index 68a7c15..7714906 100644 --- a/.gitignore +++ b/.gitignore @@ -47,4 +47,7 @@ Thumbs.db .env # Dev SQLite database -backend/kmountain.db \ No newline at end of file +backend/kmountain.db + +# GeoLite2 database (binary, licensed — see backend/geo/README.md) +backend/geo/GeoLite2-City.mmdb \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 13491c2..dd2a048 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,5 +15,6 @@ COPY docker-entrypoint.sh /docker-entrypoint.sh RUN chmod +x /docker-entrypoint.sh ENV API_UPSTREAM=http://api:8000 ENV API_PUBLIC_URL="" +RUN mkdir -p /logs EXPOSE 80 ENTRYPOINT ["/docker-entrypoint.sh"] diff --git a/README.md b/README.md index 4dc4350..1d63a78 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,8 @@ Open [http://localhost:4200](http://localhost:4200) and ensure the backend is ru ├── Dockerfile # Python 3.12-slim + gunicorn/uvicorn ├── requirements.txt ├── seed.py # Initial data for all models + ├── inject_test_logs.py # Test data injector for visitor stats pipeline + ├── geo/ # MaxMind GeoLite2 City database └── app/ ├── main.py # FastAPI app, CORS, router registration ├── config.py # ENV-based settings (KMTN_* prefix) @@ -104,7 +106,8 @@ Open [http://localhost:4200](http://localhost:4200) and ensure the backend is ru └── api/ ├── programs.py # CRUD /api/programs ├── events.py # CRUD /api/events - └── tiers.py # CRUD /api/tiers + ├── tiers.py # CRUD /api/tiers + └── stats.py # Visitor stats endpoints (summary, timeseries, geo, parse) ``` ## Local Development @@ -197,6 +200,72 @@ ng test # Angular + Vitest The backend API is self-documented at [http://localhost:8000/docs](http://localhost:8000/docs) (Swagger UI). The database auto-seeds on first startup — visit that URL to interactively test endpoints. +## Admin Dashboard — Visitor Stats + +A visitor analytics dashboard built on a **log parsing pipeline**: Nginx writes access logs to a shared volume that the FastAPI backend reads and processes. No extra services required (no Logstash, Prometheus, or message queues). + +### How it works + +1. **Nginx** writes structured JSON access logs to **daily log files** on a shared mount: `/logs/access-YYYY-MM-DD.log`. A `map` directive extracts the date from the ISO timestamp, so Nginx rotates to a new file at midnight each day. +2. **FastAPI** (via [log_parser.py](backend/app/log_parser.py)) scans `/logs/` for daily files that haven't been parsed yet, enriches each entry with IP geolocation (MaxMind GeoLite2 City), and stores aggregates in PostgreSQL. +3. **PostgreSQL** tables (`visitor_stats_daily`, `visitor_stats_geo`, `log_parse_state`) hold the aggregated data and a "last parsed date" marker. +4. **Angular admin dashboard** reads from the stats API endpoints to render charts and a world map. + +### Why daily files + +The currently-being-written file is never read — no partial lines, no race conditions, no file locking needed. Stats have a minimum one-day lag. + +### Stats API endpoints + +| Method | Endpoint | Description | +|---|---|---| +| `GET` | `/api/stats/summary` | Total unique visitors for a date range | +| `GET` | `/api/stats/timeseries` | Daily unique visitor counts (chart data) | +| `GET` | `/api/stats/geo` | Geographic breakdown by country (map data) | +| `GET` | `/api/stats/parse` | Trigger log parsing (SSE progress stream) | + +All endpoints require admin authentication (JWT token). Date range is optional; defaults to the last 30 days. + +### GeoLite2 setup + +The log parser requires a MaxMind GeoLite2 City database for IP geolocation: + +1. Download from [https://dev.maxmind.com/geoip/geolite2-free-geolocation-data](https://dev.maxmind.com/geoip/geolite2-free-geolocation-data) +2. Place the `.mmdb` file at `backend/geo/GeoLite2-City.mmdb` +3. The Dockerfile copies it into the container at `/app/geo/GeoLite2-City.mmdb` + +### Injecting test data + +To verify the pipeline works (e.g., after deploying or before a demo), use the test log injector script at [backend/inject_test_logs.py](backend/inject_test_logs.py). It writes realistic traffic from 25 well-known public IPs across multiple past days — data is only aggregated a day in arrears, so all dates are strictly before today. + +```bash +# Run inside the api container (writes to shared /logs volume) +docker compose exec api python /app/inject_test_logs.py /logs 14 + +# Or run locally (dev mode) +cd backend +python inject_test_logs.py /logs 14 +``` + +The second argument controls how many days of data to generate (default: 14). Each day gets a deterministic mix of visitors from the IP pool, so re-running produces the same output (idempotent — files are appended, so check first). + +After injecting, trigger parsing via the admin dashboard or: + +```bash +curl -X GET "http://localhost:8000/api/parse?token=YOUR_JWT" +``` + +### Shared volume setup + +For docker-compose, the `logs` named volume is mounted in both the `frontend` (Nginx) and `api` containers at `/logs`: + +```yaml +volumes: + logs:/logs +``` + +For Kubernetes (minikube), use a `hostPath` PersistentVolume with `ReadWriteMany` access mode, claimed by a PersistentVolumeClaim and mounted in both pods. + ## API Reference The backend exposes CRUD endpoints for three models: diff --git a/backend/Dockerfile b/backend/Dockerfile index 1a14197..8b100de 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -9,5 +9,11 @@ RUN pip install --no-cache-dir -r requirements.txt COPY . . +# GeoLite2 City database for IP geolocation +# Download from https://dev.maxmind.com/geoip/geolite2-free-geolocation-data +# and place at backend/geo/GeoLite2-City.mmdb before building +RUN mkdir -p /app/geo +COPY geo/GeoLite2-City.mmdb /app/geo/GeoLite2-City.mmdb 2>/dev/null || true + EXPOSE 8000 CMD ["gunicorn", "app.main:app", "-k", "uvicorn.workers.UvicornWorker", "-b", "0.0.0.0:8000", "--workers", "2", "--timeout", "120"] diff --git a/backend/app/__pycache__/config.cpython-312.pyc b/backend/app/__pycache__/config.cpython-312.pyc index 7a00907c0eb2562f0ab894246e1a3c6cb0ec72fc..c33ab09e4df24800fccb17237f1ec27048cc6eca 100644 GIT binary patch delta 261 zcmZ3-v6_?jG%qg~0}vSfv&%X-k++U<*TkNgYEc5IEUBz1LN-8|6yYdAFi!-`6HO7z zWQ-E36xWp4tj}n~=upL~pOc?ne2c@!-#s|q#WSc%T0gO%KtDY-U*A17-zT#q)kxPl zv!qflH#a5emVmpfzmI2#t5Lj*Q+$A9h{xnaCQ(M=$>mH%Tt$LF`-+5s#N?Mu{)|17 zEt&I_Kd>=~$~1U%BwuEcz9D0Ifkk8jkZJiqL}7vY1!1j5pCY-*o0*NJWI=l6K!iMy pxW!?Uo1apelWJF_G>Jt_LXc6rqvi_(kou^>z$4Oa)yM%>3;^tKM}7bR delta 169 zcmZ3@xsHSPG%qg~0}z<`nPi1c{WJw9UuG(r?9c4aSUPz=a~{tF5rqZn7lgGMeTrlz8?YFANCV9+ zk^vF2K;jmMO>TZlX-=wLkphs*2*kx8@ej<5jEr{~w7#(MGHQ3!d|?1m9~BsQM7pgS HIl$Thql+vu diff --git a/backend/app/__pycache__/log_parser.cpython-312.pyc b/backend/app/__pycache__/log_parser.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..91114cc901c7b927f60243a7d49e820fbf090afa GIT binary patch literal 12292 zcmd5?Yj7Lab>7AMNq`_okd#QSNJ=CmJ|s)BBr>8wS+dxM4hN<{SOY+@gslw z-MhO0B~#<+On-DqoO{l>=bn4+``okpGq>Bp;Q8*~9i13zV3=PMLVwH}vi6#RVHTKY z8J-zsc$PQB*in|IhEW4ejiW}S#+Yf+JZhdajha{@XNp-St)tdS+o+AwhM0ZS4!$|& zm~@UhDbEsfO}a^R`1sdHcpP zV+P*wb<=2Rl;NGc>vfiQ^F@Fj-V0dF`v6OLKVT_e1{mPW0W0`Qz)k#Sz$(5Pu!iRV zxA3)qb$mVGR(=~`1K$W3|=1@Faf<@HF55I{ST6(*QpR`Wb!* zaF{;>_$+@G@Hzf@z!82F@Jsv)fG_ed0fzXO0mJ+`zzEL+3j7%0IDZ~6%1;1Z;A4Q3 zd>k;rPm!AVFY{s)>1de{5XyxLp;D+EGmzd_A;v(WV;~FV7>B@^iJwNFZ{lAO0-(!5 zUj)6GzeMwGLV%Vv@iX97@w53-uY#_&GyGThB*xrIq%?hVUpJ4|%ou`~XZQ7wN8^{c za3mr~5*JI1b5mha5=8D|R6fu34RYf`B9@4R$WtTZEjYySeU zz{onEnIDm{_S9#>q+^A2j*%@oHDS}SLOS6r;5wFM-egr=+$6ho?nIG}71Dxac%4nd zj9jeC@NAO#F4pNE8Fihnu?6dgnV@lY=MZf#7v>~%XiShXG^fLtPDkTBht4?_m4&Cc zuBbf2sU6fZTa#yFoWkeAlEB3iGB=i(Mv<`0&A!^gbtf*yV~H@&jfsg#?z}8dNo}pI zyzolPWcbn~bhktjldb6X=oEo5QcLrgCd%EE7lb%Z zinaD?1~wA|TP7#@b3wc4LhD5HKVDdD94!GMGqaSefYR_2H^7wA@to1D`hOQ1>-gWAz%71+n{66d#nmv0oJC}!{ z=Opy}r&_#2lV1yg-+wBv2koK)yGy}8{d``}KD{4Ci2gI+t9rV1`5@$bNWankISRho zA#TVIlm4N6WH!5GT8~hH^Ue%1QWvxF4pl6h{S5Y#r zzS`|=djB5*{|wPX>z&6qss%g1A0qv%j$^-O|L`dI!$glN_TTNAVI$!8kAhF_k-9Z| z?g`*`4S=tXi%-+@`p`1?q+L|c-Hv6oqWjN)KS=sPm9N$K%S3+gIQXjlK~0bJ67=_V z>w0c}_(N6U%nRV3Daf~K?K(j0JWIx1?YAaPPsx|SKRuGy@73gIh^CRC zA13{*lHgOrt{wxwkF-l2$9COL(x3gLA1MDQ`coCy0)F>NUC%bmPnBN>{;@utZyD71 z^KIaF=I6;#%m-E82Yv^cZx%<_Fh4*48sn}CaLZV~WIm|P z0blL#y+jWgU$wo*^?F;uA0q9d)vX=V`)5D+$I1Lr=LzbihRu_C`z+}<)ja4?k#_L= z$-Gj@THPMfKPO3hU-kcG@4wG}>xWuJo#6M9cB%SnbbE5xXSy)HVm<0pZFiA5JDBg| zu_IcY!_R= zCv~3fjGWJXlL|1*9Ghe(h|FV8;nnCGEV8xQcVuq>bpK96ROMp3n|Qp{*O$$UE27sr7Brk~D27`-Qq-x2^*g z@~-chhnc!{PFp=Anpq~wL<}Ps<%`TE;|Oz+4Z3G@rxJ+^-1O8VF6~%Cgu6l<-7wFK za7bGm4i1m%Vo4kyjmWAm9Dz?Ei!)7}hJ+#sUH}C*riqKeL6)Zp9ZST=DN5BqyD&Kg zPmJn`jv+g70Va&WtCd=~WAKzE>K<`&LRZX1;9!f1C@)bJ7tafEPChTdFB61!4l6B@EkzUL>(l>2(L*TAky}pOA&!y1A-25kZ|Cx z>u$R=9h0e2Rm~Zq<^>{SgjoQIxMC~x1~SOdsE-&Rat^gWiGwEcV7!E%)BwZhnOxB( zv+rJUO{RED%91S(EL7g8T#0ZT8CS z;Yq~D6gTF?Yq^P3AC;TcBccoZ+E}l^g-LHcWX0&)7j09S%3M$@f*h%pIbVY z@dsD^&1rx0E&g_A#=kFHzA0O=`6H*r>&P({yWj*r}!K8oDtY(66wX) zs=KYs8(mug-?2KoOO5aNSwO{1=HbjEdobBq=pOt%APY>AnaJ`!Kk*8}HA2ekbJ3WT%J$4-=&SzR{fwTj0vJdOT>gRc`_l%-1E)H65_?Mh}%8Ea0^DSaE()9 zf}f7id+|BpJh^*;b7>~nMAzQfBP&}JCKIv{B3lgCh$~f4TcDmxA#6aZZi0!=qoE4h ztguaV2`bKYC4)xsMUrinrf~T}B8$-}oV)^I8ZV8FNo3V>oVqfKv6&@euLvQv(V^&+ z7)SQ+;U}F0zyfhGPVcqgmEekh>ynspY@hGCYjvcg>ydBF-uL*f&0LvTEcxbF7Q<-| zx9ZuJ_H4^|8dq$MS(G|(<-qk9mpU`fh80tTNQO)?#iDUxYUtm-qKniX5kC-+?5RTr0baeuOX#o0_&y8kXjvCjRUNggQrL;rT>&nr4C z#vhqkK*ctO+cLQsKQd}|NYW+#081N@ll%hDj}%;_IYVKTl88*|%Y82cr^(pOy!Zn+ z5GFaljFb5VK4(sv=h>^pvK95vCs}ZEEJ=&RUai3!Li(+HE?G~#C9G<=?;y6)++>`A%KjK z!s5Ik1lg)pCkPHwLa`Ff7lav!%y!aX`h0FnAY&kY6(VooC*=STcPe5w*L-*H{E2Mc z*7*|;?C#Y0MdKa&<|XzQ{?b(UJ%8=eo{YaS)syx3Q*&8w`GexxrM;QrV5;+ONh#?2 zuCitv z=B`$&Qj9Yf6H<~w5gfQAiucG~M&gQJ!*A_noCGH^=hsJqC-P?pB(!IaU3Yof;6&MQ zUt;xDE7^$alVrck{E>k-k~LS@r}W)(5A!v=$9WYy?6(aI#@{oWv6s>}IC!%&hZ72` zJ|7F#tzsAhlQhaco!0A{DAhTIv>-K6R^aLwZ@JJ2S4!Ju^z>_tW|2{|$b`!oIn*C} z^fu5vyhA2~LLc5DZ_49V-p1SBBqs=-xAJ=x$`~f9RPK6M*!^kSYu4wfxL)hyRzlu9 z!F^tS4*M|q)Azuy86NF#`ew`&E1t7{!B}toylyzM4qfk@O>WR7u%FlIcp**Z_jA3c z=kT^}&Viq^&|?!Ak;fms&2z4#y_>mic<}_zQ}-9xyW?>!I2+vgnv3jAvS_0D^D^X3 zp5CSn^WBB~InO$~TRyKV>B-mUO?s2A3%JdR?A376d)acKlH%j=@OJAZ*V$M6f4Wxo z>ow^1Y`9kPQWLQ4@h9p1KUpg!$&&dh`GBrH>3z$q+ca15gf&s3OTg1Pm@iq(7n4P) z=Os%*)$8(nByXZ!=M++XF2O5{p*YNsiZ^eKL_qchUIW45fcTws4>sYUTEe?^&0>dCn&z zte`%8QPPb!r=Fw_=WqVLK{W8DIp0@&(DQv2yO_TNQVW{^*{i>y(6^7d`g8o2zu4-l zE2_H4fI{2;zz4C8E@d7px+=EQ$&8b{#IW>Q@tgPHN)losWbXCYv}3 zZ@b}V>&~_ARS(4BhU>*8juwP0-onesNW}2=8wyaLoZ!QVNlNGN2HSoadxz`a!_hbc|N|SWp!A<>^Tj(!!eb zQvqsQNVgSll@SmT6BngWG>$i^xG$(759(#lR-%3qTNQ-}8a`8a8QsRg`Sv`f3xO!| zDz_i+axIb&Lm9l&6~@8{9!4KvJJOwYBV1|48w*Ra`u-LlXyBqU>ry{%Xx`p}L7BB` z6uk$`nkdi9aXmvr14DF3@JgMVHNX&7Bu-4hi~o@Liyso+hnUHTFsb=C;h2r?6N>v4 z^(zMT0)s%!*CDC6AL*1}qx#z0c50w^IMm%YB$Dk%Bp#q*j!Nhuk;E_*hxFxGI2OTN znGs0{N3jZ*gvd0;i(DNPGlImTAf^cJli&)6#Smf@p@bNUBUKHZBe@pLipi)fULk~O zJeD|zz(pb&N6#k6C@E$-l@uHKNPvWX6{{%FclnB&1kDgQ#-yGM%Od$Gf!<0)5_DC} zQ(_c>G4kyb0=Gh3@n}~ltrsM;M;0a}(Mcp61tUe$nTlmvj>Hoeaq*fWgT{vt|F!2U zQ>^rhktja=h)R)!n3%?=Bto!Ue2tVvTzgy!p|2FH_DKotBk^UDi7*-TDb97%LL{LT z#fr!W`Y%ob-{T=V<>&~BzmHt;CIR#o_Z=ckzi$a)@>0L~}|6)GD z{m;*M%U3-dVsrDy^)C(%Upc&3jxP?fb@eN|PQH8m-L{q1vuV$BE4Jrw#q(BOyL|QX zQo}E8Tkj)|efh@ayDs0i4rg3jR$UEgSHoNG)y4zq#shDUq#KWAT)k^XL%HK)#^7_L zEQm%I2Ufkov^SXX?ns&MJH4sH5Byt}qVLo%%kMt3GWblUbLftL7#B`g$+eSLPA-~p zvCR4^7Fuq!WPG)&uG$q>?ftS%t7T2;vZj>t<0{5mbM5k4Eo1gRDDqw#zcNmP>HC*n z$`tLH@4e?OyDl%x+?ZJkWCEyk`+VR1vcUX6)?Ye*8lkRhC$60M=BfFf2mY#*?LkTD z!oC~(R!i#AC3Q>3X{##|$lI?hijsMaWgyiPBKW@L-e%D)eC(w}bHm-VG(%zOkt?jF=-RaiuckSucXENR)bm`_I zbZJ>p>KF=@Zc;<`xLs=(^?T)wx1x8-_g^<>y@3Vi4d-Id9dGS}`i5KPTd{QG!A$+3 zRL>pP78I^)yyZ&Q?p>`tn65qeZ?%VS6kWG2mfovu%vRK9w=~`=exowGxo)ZB=80?t zw^V$yGFw%@^vuoCZ1vWq@Xd*AZNo#4jo!|EOmR7_tM`t#KI^SoDo%TAZ@6;-rtV0t zn!#L2+(_K@SAOp>eaX<8_P73xdv#|=dS}P-NP1^~#y^0D1$<~&vF~FGYZ!s5 zyXC>Gw_?>>m-g0Wyjzzp|AX=EzP~R*jTJ>0F&pEp_`p;DR$w)FC>=cX_Q<~ld$VOV z3!XplGWc+b4+rQiLV_%04#82wk{--TJ+3F?uUMHm9Vc7lyrgzwmb`bx_ z?i=tK{&Bwx_)pEwfm*{)y+s7CW(OU%pVknyKdtRB4+hNdHn4+k+q;cjWgvcbgdOzS ze%4NN-ZQa-CARm>CS<&4Cw08%X9r6y@0C)#who-1TT2P7HWOIK4mMeSUhgLOZWfmP z{Amggu!M8SGT^bkZ(s+^uJ?^rf;(*BzwfgT)R^DjL`-_Wnk6_j@BLc)pw;~Tj^aUs z`TawrPgV?Oa8|4)g1cE*wc@3)gu+r%%Sym97__fcv4dREO0|{XTS+ZJwm--z)=-E~ zL_#6)``9|gFB3pdS90LcBT=!)GgCP4MKj@$aF0l08j1s-f#GYn_*isYJ)T{Jp?_tj zoydeWiTHj@qGvEY`RL(Bzh+Y`L+V>jPbd_Q<6}Ge1*HUI$RVJ<8a0uKCIX5$Rm}1< zd^N>9hSvwO8c8H?Yt#_qR|!UrfInmCs}}XBupL6}(=p+Q_+w;YDlyV!oDP_>EUPew zf6WyAf^q(WvHyyx`xUeG*Npc+n3`WQ+kV9yOEbr?Y<9HHJ5$eJkFJ(&OXJ_&kYQSK z7B?GMw6B&orpp`G7!bMoy-ypmCR?gGX9RX1SX0gn3_Fn74%|v`8^L$v>;!iZygBD2 zxQpN|IXA(JRNO;wuZkBF+(+=HTnWMb1aHoj611g})(HxYcZidPZ5 zTE%M!&Z+nog5!XJ4IkAJyq@6Axvd1>rph!Byivu21mCX8>>zlPiZ>Iyh2Sl@R)X(T z<#!SMDV4vQ;Cod0y##++#rF|>zlt9qc$ zrMgw>w>^|PMyVaSUP>L;s6I-a(8`^p)G00RG^Mc3(i#RRHK^r1L#ZJxZ`MEeT4VOIhTIfe%lfYG&M`<9_pFg5 zH^@5J#>F!^2FX%njU+kF#`+gaatxBC@>}(9RId>?Om_2jmNjO*Wksmxzx*oS%kTST|*oM$wZaA_8Gftc6OJU zaSRq_Awo(jS*q!zf+(O=?Mo$ygz7%PTTKjUs9miUkkbu$XqsLkRjp%qNN7~`oUx4^ za1}N==b9OJ=x`JW|yILBs=!K z>6L4ym!pbj8#l%&Z*HxwR-*OKs-Jx%xoXX}s*w0x*&@l#JzW-6qnhVRuU@n9T2%L3 z*)>bq-)$e&qrh{e*RI)k18RJ(?7EfgRqdl7YI-VN7?hqK9{|vXB8b*!tu8XkUsBnke_-TN;pTGeEFB+0N=v}1P*2r=*OW`V)+!bs$ws#vh-3wF+ zKkNY zgo%!AVC*kgQ{x}O@$b@YaG%lOAIJ@nkru_zFB{@MITK6ja;u0EdZIU> z$zm##Kr#~h)r=yJG>PJ|Xm{6vu6^M(r0I+<_GgCDNKEMBP&Fy~`PsL{0Xc)KWR4GI zHE}qZN{PL)h>}{OHzlJ!ENXr-hCF5l{y+Q|hJyg&L(7T}EqqkrV`~)Kc!-T3`O2tK zXuRa{<VWDW zrBH)KTgTb!zE842Qw0!M1g&6t0F>p*O2$`eaS(8A@jpswShJ zRAxXcxcZYRS?gH}%4o=e<%=_N*p!JQ0v^Kw(bS2J1&5j$)^a_`v?i;%j7DtcC=+LJ zq*fHWG9qqZu_-)xE17ymG}|4i5yj5#?nB)o8d8(#0TC~LRUgWlQd1EYgKWW>$Yw)0 zyT^2?>T9@L1$sc%RkFNu1#9QYJ~^u=GwC302Gzqv0cDLGh(oQxS)@|KUwTj#v3Z|(iW<^AbM z?k6MpbsO%vHa>8BFFAhEpKsnd8|cW_@A{qa`=j|IeY0_tkI8vCllNusyH9-K_U3)t z@4F-OR^V&<=8+A!%N78%Mji-7`I4>IPh9WId%|(wGNpjB$Uf zootz^l|Z{O;}6(Z_R4PKzy46}APF5MKn`*>j-fy&5K_AcbMNas6gzOFv$bc>?w-TZ zBl`n%L-9DKq*!xsDq02pbLOUL`s^s&iz~ECLtsWw!$#kg$ z1YRN$TLLM`bhu{-zg7hfFWer~tM=lEMtY0qbMac7Rm<`$s;}c9$=A+ITrT@9{NbYj zaQfyPfmui3?A`~w_08dP!*8BBcV@i*9$)#u;>f!FPu@UM(@e%Lrx-kbYoz6zAoetogyV#kG! zZ?!9UCB9MCWr6R)7{y(tF?+8$=AAtbzN$y&3Z<2F9AWDV8PNX79>b;FK4}nwcq|8z)Q!V626i zY9NKC+F{(vO@&FJsU5DkhrTVqxWL_ZFoYM8hHqEExQn@6xf%02MKIpT-SH6p9dEQg z-bQ~IhVc#Dhufn8Yq$ix>+X&cwY@-uiCBEn3V$NZ0w@3I&JmFM8=6JJ)A zK<9{UQzDz(L_Rn)wZeG3zJ5Jf@xe_6+tLS0D%q=&Ekr#*AZ=`}YJn|=T=g4o1#y@Q zhr$wJJoUQBm1@XPmFkVL--kJ`V=@n1yVxyy5c= delta 1960 zcma)*ZA@EL7{|}k`_}gsSPKOTrL?7m0UMjJu@~5!FYE1wBf9A|hIBi34D2m>%iM?> z)GS7mCQ42+j4s(?GX22NXifa$2iXeCC|`|AqKV)1iy~^WEc@U&r*%$~B|XXg_kW(} z_ndor?sM*p{B+1&a=Gk?j{Ns~C)?`h+zqEz1cc@hMi?^401be7OU7adn!vq~5e-R` zxEC|Bp=gR>)vSCjWo(9BvvV(J9EMYK8ZOPn=Ss$Hcr=gU)#m1WO*~}H_>Fq4o_kxS z!3bypqfwh%3O-Fk_6tZ0V+YlC9k5duPFt|+g1Er8ONSP*AYH(2?7?0=a;;`Tti+>w zbn-~q->l;*Dui1VdH^( zpu#ucKvjHGg>S@7Rq@Rgz8MFr;)4~w1-Dkkw^aBLZmWuKt?*&oUKJmz@DUuXif=3P zburxWKi*D;!PgysegGjeMtd|t8>ma`df=DoDzCJo6L2cj8X|Vt{;**XSOsP+T4sz9g2O{K(<5}nnPzl`| z!)Ag!=?qH`)2O@fvh$<>!{kTLJnSMbd3(J5G;BV`PlKu5mKe5RL9WFFb?J0gv=hh5WBvzvB7{*DI84Z-V@1_gKh(#4{JUzRmJM$iIJI z-ryN!0OdcV&w27&L%Zv(a(+=b2MHT#4Gh4ez^yO2&97W0VEsbO9FM5yOR-N$qC6OdCAqxd;*d!@=~YB ze2;@7OS>W2CN2%uQh%M2gYtD3E4}U=QfU<(y@gW4(=2iZc!eI7L~EIUCB%gZj2`5N}(=`1}Ra#6%J9qYGKu@E|^Wp;;NfZnt=YF)fTQ4 zrRj*A98gv}A=&L*?NXTUv$6O-pt~<2t^y-VGFO|dHY+6$sI|_LS7E-M@ue`(4VR*v ziOcF9rPKrJc4w(qQMc1cKZDi)=)TrOu6(&QtDI_7*4&V)ajtn3=KV@4NWKog4tn8W z`xYSMk+APY`kjv^wyxVN7da?ZU< zBGXmwFoTWJ+FaP2_K&muQHUvnFqUnM{VQXI5Qu(6ckD-DFtT4M^C%q@_P*yxilREj z5!~mV^PKm+=e_TF&U=rpef!tv1E2W)9tnQ`{Bm+}F|iRCkS}i>dZ||OWql=o)-Ov$ zB33m}3TA_)P&On>7U`TsY-+WJBob>?f?P8;x z^#SV#w$^@djMD+2gI&gP&W3;ucd`AP?E^N_#SU;b3T&*49pr2r*#0hdh_eH0(jpn` zqK7#>1jfT%>E+DcN)pMW=ePL{mP~F;!M8r6tof4O|8|(>|k? z72Wu;{F>9Wzp_N=RpX9)V}AsE8A22;%?_|8-M6{eK%38Rgk!fpvQ*^oexOVzq4P>b z)o;k`ka5y>{sr`7F@!jd`*S&6nN#VKQlhyWv*Caf2P{Rx+}N<|MO~%LWBlGW`@9z! zhmzNemnjQl$F1cTD5j79Wam_SW^#9a*Fh`#gp3_5Hcln?jMe;2siZ+NxAxJvlRNb7fLxr*PgU zCQijsgpy?ZE%K(woNA0mzvSVt1280(^}#REv};8}dk9Q~bOt>;YiU~Nie=NiyinH5 zxe8OC5Mdwq@Evmry%DY&e~K}YH|+AfU5GbCps2TQ{p~$@BF2|-mQIZ9ICj52byv^? zz#^>~*Zb#0U`X=7_7D84H-jR17IfnZ6Cplj8s9{Bu_d5odauwqt*F!864|T36?8>- z-?%=gi=|$HP4Y#C?~f#m%%rUBRW##e(q6<0%QYZ&Hin%}h4-}Z#=`Kkvdj4S@ZsJ# z-oXhFJ~TcVRz=>hi+QVImHCxKrL0V(tcA!~E3ss47Kz%agII}m+t&G3Iyvhic3>SH z*4?tT6X&+8!+L=A5;yVO#Qei=Ue5ZeUiip_-Y7x>%k2BQqE9QeqFP-UomSwMR0;(~ z3s8(m^0Kf=YG72<^kQC{Fe5E8hoqJx=#yc56Q?Yqu<%rN7Tk;%$<t*(ujX|_0`WDk~?rWFjjZm za~*qVwZt9oro|@?pUsz1lAnGVVG&!4ljVyDOXeZYoA4z3@kMTrdvTHZxNFz(ny+Kv z-qpa5<23NtLxa0;Au**Ew2V zNdXj?vb<2tC+8Sd-Xi5IOUd$lTVh&rnX)8NsuO1Taj6t&xg{^L8L>2r&ukJ~k7PDf zB(+|hfiozyr8P70RUx85zFb+-S=IEl8bA=5PDOk7`!8_5uhnFY7gf{QdJr?f!?fj# zdX+h_=~7`I=@mjvw_08*h^D{&zQXH?Dt;fQNBA9F+BE=F@rnDmfi?GxB5ymI<$GZ{`&1Rjp*d+#g_Ju)MxIwPBdNqyMgigJNI0Z5AD`R+W>>h z8_x{IYM830DQ6%gC}@u)OB}q6;rR}JVqd_naBREVQp@taKz+nP{fi^Qu7fpbZF@Sp z4f2ZTzfdlGPhlFxl1Ep@Gfc_9r6u(Rnp71{PimEXo@&~BMTH``jD{qsmJ1+ColclR zXy!P}3m+AlWEWv3ak^q@Xtur=IDb0zl~3Fg1>{E83<8!nGlcPo$%v3Ip6wa_(_ueF z15afcqary)@2zzopivC*uW)I2(n6V>z8^hkd=wgeuGyd1B=y%D{U`3)qRnvsrmKGH z_Tj%A`@^wD`24%)MLk3JbZVb5H8NG>M`m7I&46}ERa7|Qm#PzGR$?JA6K8Au6C%#+ zDQxj`DNfDLSo{=&M+Wg3=HEbGj=7o{kNEM5ZMt48mGOVXBs;nVepm~&j{zQ9Wm*1* zl=$4{mo1-5ZTJswjTn82x4*F?x#U@SXUHQv*7Y3;ZaYIEXlHgLxb2KNVRU`^zY^a0 G+x{O0FC~ru delta 2010 zcmb7_PiP!f9LMMF&hF0c?9TqlW_QPEnlwq%p?@w>p>4Fa^XCUdvLq|%=DUTJLRVYSh?J0ClPvkJ zWTouFw@D0Ea#8xC=Z4*0q+Bus zdl@vgL*8J=5?hmHm`xECc>EOLuJog<&>Gj7+0eJs$&i$%U>s#E`#m#sgDpbm*gg4* zia~S?QDwi%janLKB^-+?_>u~oz_n?X6;rJj@pz)Nxf(d@HZ9^H>#wfaR_NM39fz@4 zvW{>1f2gs~QV-MIi_XrZck1&PsQGJNzkLwBOeff6X10C=zU|^NBK`$+13nokejP5M^6ey|5RUPVa+!)AjlgaLu$Fk5008 zwHBFSkF=j=N7%B$wu;}7EW1|vR3EZ_85cvmvb|s0A;SA*p7B2_-#w9Wk>}zFd5+`_ z+oVG&j^vJKZO{c=l$YLN*>vzL;-hIhVPmidC-Oq7)tO!aYm=m_n zW8}f{N6C)kMcG?xUf}WbKHMeTILdEb52F$5@c0h z<&n;weLpcxGVHgBcrEQcqis9jkg56SGkV@bKA@PF*ABcyPN<@ngAABGrB#H=pfCSuo*N(S~i-_dA zlQT2-ockE>_^(5s9}RsL3^oYx7x`@gyP~;7Y2t)qM%4hz2a4TieCvR4NBwd zzGMD6VV|HhJtrtZ)%%tQw+X_r0HGm3!w$58&}Kke9B3n& zqn#9Zl;8n?2OW#s`5R)i6r;HfFgN6wixWBw=tc*cAan%K+Z|{Jp_>4`!+|CV-3;g! z2ii&KRzSDeXo&=N5xgDXI~|Ktgzf-zrvvRKbQI8C4m3^Z7@)fyXok=|fZpXmdlv7O z@A(mlWwFhYl)S0RqUArhU~2k&z983ey{>cl^}3^?nB>2E#=PyYl(q96(w*L3dw9*4 z;4e$>i7~#XKDO2J=4VXpaa9g8OgQU8lR|^7$0Rh}Xwqn6Xfm)do*smuB~9ykf%PCt z-urMgK!$%(f6AsD;7EDI`w~&c#XGl*W)GuJO3hImkrY=Ij^M-*{&i%RO?!lBze=9xAGJkF58*5& zlU$LPYAHf)W?D{sRSmUvlMt+C&3YWi2{fsU(}rfM)Yx?hV-Dp8caK$5 zNP8*VxytgIEq(%L(R>OYV+$ClBtMh9!_M-Fa>rk>yM0{|t9r$R$^Wa{YwiB;Zhs7X zGc;fF8=W~w&P3Onqtzy_F7!V4}f{{qt&RAw%+Cy>E?LO4qw2bwR^v&{$H3if8o-{?sNh?|WxxlU@F z!_JW^9aWiL7wCi4X5Tw-+3w;OFxO(7FLfkJi*>TcYMV6krZHJOc{0xyA5@J6ecn*5 zsHvOzLasQkohqui$2B-{Iy16Y>)i^gNp}Q`I?h#0uL!5f5C6ewN~W$$Df_;4PT=qQ z;{2zyJRLr)om3}Prm4n+j+ZG9thT~#a+XJzE}HX*p&3M)APj(}A$WFtH5$Jz#ci(h z)L`~Ivbp3-z$#Tzq%n9Zo@0L1t5&dj#MA)HdU3Sqk$qNppQ>Y$)|2C#(TkvR(P}CW z;C82|S<7F@o0?fvR4Y)>=TF!}e{SFPxvqeh za01OR20Q`-2sqe$_Wsq@=!z6wP4}JI|Ki+I@tnRQ^{r<5&y2tL#Krn^r4^}vHPrIl z1I+G&B}&OpX}mTBN)V@HMJDphP$9}mn=Yzus#NM{Q>A_PKET6ZM@5>;%(=gF z?|tw2J-qnzW$V`#i;1Iue}6x=@*uEd_3*Ksu6TLXkTUR`s1r|!da*$?h(^&Qnr}Cx zjCI@vPHeo+iI%MXlX`IFxReP|D@B`Bs2NckMVnP^$*Mz8tqFxnhL)1r6zY1+eG(gdy3bi8|qG+oMbs!q1XoR5wEX;{;8-?3d#xAnX zyUH!7>!i9amClW5jG}@H^&r|!(YOlrBHBaIlPc7QXfH+kRHz@(=P26GP)0C-@BoDe zRmMRQu5&GhP&Y(%!zx|to2P}5U!gjIS9D^gl%@X-)9aOt6Ue0F=qSA!_(wmt=o;phM-cNNYo)3@_-I(4@Bg$@aRX41Uve$bNFL|W< zf_!(vSq8R>KC@KHZ)SxiIS3XZ5Un6!9mzTnVGt1z9uRFHydZoa{2(Bd9HT!68mQY`DNu#v$dN#SrEBp1lF%06%_`TFfPd25ldD!4s>; ze0}h#DX$-}e)56o4h!Co!87KcdGeAa*CkI-y%5v}9t1C>kAWj3VF~iR)h?zWTit!%ewFO*98z-|G$hg&E#(7LatVxoyMOmmx z3^nD6lYewE^3*n3o&pOTcM{&FLBM+G|NZHSD{_Lhb*+-=R1f!{7NheX3XVy8&!LO+zfkcecG7!!>`kr z&5>o>j8?TBiBi?4kuS~2nXP=X+18iA830wqSX|0w#f?I?ctF9dourBaWH{W8ims3^ zmgF@6kI~Y%=V)%J{j(P<8}2QzEK`w`--g$BK-7UiKJU^up;4QiZAX%cI$F^R*<1ss zt02_vCDF@l4Q=7D>>WSq_iG;S^qe}ZS;j;TZ6Y+)>`E`8p1qa4I9)u7AckC>#&+lX zL7eF}wMhzL#Q>MIQhqhd?nz8>%of$UJ&pHd=j+yi-$irc>Lp` zi)UVj{Lwu$tg4?{D~2(8v7ysEE8n1>JbjA8R6eQadHyli_L%ek!kv4>o!gV+5#IDC KXXHol>h&KUkTSCX diff --git a/backend/app/api/__pycache__/stats.cpython-312.pyc b/backend/app/api/__pycache__/stats.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..482bca4b2b0ed8f477010b18e0891e58657ec98f GIT binary patch literal 8862 zcmb_iYit`=cD_Rn$sswSMCxfhtg&QEqO6Bs%_g#AN0uxuN@)@GvbV7(tId0 z!^onkT5B7Pv`Jwbn?_DIi>TQBk-%QSKlVqqD6n0-y9m(!AeSyA51b}JQf%{&jsheb z^heLRbIdktl-#O=w{=K5YK_CTpJQmv0M99ZjFcNKn!dl8i$P$r= zk4TJU3Ntlhud$=xA@2ihA_!?;05#AJO^fgl2 z8E%R+`eVZe#zSfA~6KGvUcuSt*`?uSH9V6g{&h@8-ndm?mnfF!=P*0YOuOQ&X}cgkw{}w5X`T z=#(HvCE>-mtjr4H)RZDmiJB}4F9p@07E=UC)Wp6|@gyAgnsgS5vToOc5m}PMng~^{ zL3K77I4nnHMT7x5_tdl&j77z;*P`2j=V&ah$%<|pmZ#;Yr0SLv$V#s`GCuzFp>qM5 z_SUT`$QW0>jP4%O6j_XbV571+9gC{6*9?4RHK4PT@n`_L%3(R6L06u#9+OoS7=kFg zBnM)W?AKyv<)~gaC2RgbTv6nx<`*TH(H~c31;#nCMa>O+htX@Nl~@2u{&*CZM3(%p zIx29jcp^6SG}hn(%6iq0b8CX7VKEq<)mb?2xDpNI6eyRo@G!$~ zEd<#T(em0wNC;48V={C~vluNBbF-j>2|%nlk|ZUq<_)b(l1YNbj}b4M=pUuh2wkwo zisr{H>=9ruDCHF(?1ha&Z^TQtY`m>JmD9or~42?ijMm~)vKT$oCauz_*QyLEJZi;RgR z8RI1EA(t45I}h9Nx@pP$cNPnovm`B{+TvJ)f>9=|&L|l&giz!Ssn{#jxSoc(H44LK5nV%su%&*NejMtGE{;{kC zCjt2oAB0II77;{25od&Br^W?}BZ3N;2`JfxdNMY4CH!lAwx7Q=xva%7hBRNRztFskWhz&Zsz90d=s z?m^h~@7b~2Kl0S*!K1^&hmPwu`J5a;kj5jTTV~G4imaQ0lI~FhVpvrCv8e2iDgG!t z_1gRdJN+k*51t%9^3>>2-=SeIr))zWR2sV~!Wtk(>$YieHXIWr-G}G}@;_vLSA-&fI^$BY~wPx#J5SN+4;J*Qp zgTW+GYKq3F2zj9B%z9OeHfaDqxul^DpebZEsNz#H-8362dl6h_4f)cf8O}D$UT{k- zm)SW+D?n%|>`7M11^{4}9B-O$I&b9pQ>8yfRhUn_K3?L}Oa{ZU;n?&EIafjzz`%oJ z1M2otP!z^w1x{A)7`O$(p_d?6`y38DdV`>n)8Htf>SMB+%b+Df7acVg ziz@+H2#C>6O;EwU31?(cf}`y1qqBkPj-ww^P;sRjRzN|Z4dm7$m*dh@1AwgkBM>jM zxlNfOKV^%XvcQ7Z&0OY6Fqo8X%^X=|hskqb3!a4qnq%fnb7skuWPVO0bJ7GkSSZLX zzp(w9eVtii81ia8OC)QONH)}YXybsk=V{9?EJnXsMAw};tUMkl>^_V^$D|s5Ydw(t z1G7XH&7o3*3$>V%V8W87NwXn3I3&z*fJ!TX7_6C>)lHL<&IEKu)2%Yy4Hb_TY{z^K z2e^Go%Kqbv!pIj4O1zx8Py*Ie^&wj8+%o(t5Iq|O0PxP z3lH6n6Hxz6x8)^5=L}>~4nez$y8}xoJeFI-$_-1D==gq1xgY1c1lgAqWai2)M@CZ*kEO;YQ&W*tbS^#hx2eesSwbE&jhNt3dD!$h zA@;+jtdn@2VZKt)m|m(Gbix51w6oWD0hMrT@;R-azFjZaX^0yVTr}1j5WyBE9A{;D zx>pPbUjlO$jd{(9iXm+19g0OYB^K5lKt#bzLG+MtpYBEd5BDC7Pfmj2iB5rs>wqvP zDEhV8X<4@!>TRP2sAyA_hd?j|R~$^erYJm6UQ^Bu}Vc*fPc!u6)P-cNEiYdGCCoO<%PyRPR`-1AwMvm7@4YX5KAZ)vx@ z>GpkV1S>wTA$-d{u0F$UTCMK*u%&agv14`1wg)!W?fBfr@zyMKvu0a}z3(d(J<*Hq zL4o-hGq{<3ZO{p1g6mh~k%$Npnxhyl=p5)OJ#NYX4AL2Ypn(5SMu!VnVO4$t z05o4w1Tn)Qd={2t4nQht3YFps498RqX*6`%X$lprhfxFlime5UQqqchKvT$NP{pT# zMR2KsTLa0c38CM#A!)_A$~f#va`st}=G*%)X94v$^4e%$=h!4$&Yd^bZO#Hr zU&>xZ-vM#TP>)<>o;!fbbI^zbRkVy$IkHuR^Q3H>uAfe|^xkjXm1*6R=9(8ASC4#H z-MDn%+JXC=>s9T_+~VAu<8PgQ~w$xa^AQqQ(ek9A2n#QF^uRZ#+1M^RO$hR*)n{w_- zvAeQvX#PsYV{!3d-ND_=?FQ!HPWJYuO+Y7{{di3Ra2|vOEL9hLX{s)t{1xi*eO<2- z$E=Ln7>6)~!N7)8GO~Ys!PM za2e$-C=W!b!{iFYlT##Mdd>?F1aWD(cLBQ)Ay}E)8L8VfBka-Zfxt z1$+)&G%3p`+_5m%Ej73Yk6ICIIy2*vEi<#ZN<}>_H<_K zUD;}4v1jW@O%tm9@}5j}*Gfg#{K%@)ePv*A;FW#z!w;;arUCfy){m;|7ly$5@YF6f zTx(cyx2N6h8TYn#+gApjObvf;l+b3Lpzw; z5Ttq8+d>1<5BCjiXYX{dK;PMJfjq(XPsy=$dR{P#mth~juUY(I^t?pR%9+Qa78eW` zykw;`2KYwY}KWpbRXW zTaT#2-;Ai|7!}YEoR_lbnQvvGwaj`#ZJ84awOb8&iciV%rUh_#qy&eh4LEE~q!EWf zQKuAf`b;n&NI|^bn}}l!AjBrW#eK@(5g1=cRAFK^hu!eqq>F;MGTUR+&`Z1?xK`I7 zgKrZ(0(#YWjhJ`1paZ~u?xrAjJ+BH7GztK~!Zh45k_!JD3(r}+X> zXE2+9tc>ZYK+^;J<)lxQoOte>u3?+`7`-lWy+$Q}eD{rqza~xA-^s z)lJQBwY<>+rIm*6bVE0N$E*v6KfsBy#9p+5Q2vX$4c7ePA>nRuq43AO7RbH5y3OyGVgbKjx6$kHU`*#Q3PJxN=d~aT>Q)GO zwn_L11>c7h4DA$zUIi^^G0@iC`5t|6mq))0sx;VE`mh^4RP;!qlcd)buf+S{j-W3u zi-IOi=kN+0@9ybExq^mF>BS74Ki!3`@Ch(?&!i~$jYW50WdyF~!>WQ;NjWbJbwjVj zCVIb}Z>RU@Rygwb;nSkUr^C6=S_*z=&)pvEKxz~-jB=<)MGyBogx*FOS6?t;JS^{5 zE tuple[date, date]: + """Return (start_date, end_date) for last 30 days.""" + end = date.today() - timedelta(days=1) # today isn't parsed yet + start = end - timedelta(days=29) + return start, end + + +async def _verify_admin_from_token(token: str) -> User: + """Verify admin from a raw JWT token string (for SSE query-param auth).""" + from app.database import async_session + from sqlalchemy import select as sa_select + + payload = decode_token(token) + user_id = int(payload["sub"]) + if not payload.get("is_admin"): + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required") + + async with async_session() as session: + result = await session.execute(sa_select(User).where(User.id == user_id)) + user = result.scalar_one_or_none() + if user is None: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="User not found") + return user + + +# ── SSE parse endpoint ─────────────────────────────────────── + +@router.get("/parse") +async def trigger_parse(token: Optional[str] = Query(None)): + """Trigger log file parsing. Streams progress via Server-Sent Events. + + Accepts JWT via `token` query parameter (EventSource can't send headers). + """ + # Authenticate — SSE can't send Authorization headers, so use query param + if token: + await _verify_admin_from_token(token) + else: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Token required") + + async def event_generator() -> AsyncGenerator[str, None]: + # Collect progress events, then yield them + events: list[tuple[str, int, int]] = [] + result = await process_unparsed_logs(lambda fd, c, t: events.append((fd, c, t))) + + for file_date, current, total in events: + yield f'event: progress\ndata: {{"file": "{file_date}", "current": {current}, "total": {total}}}\n\n' + + yield f'event: done\ndata: {{"files_processed": {result["files_processed"]}, "rows_inserted": {result["rows_inserted"]}}}\n\n' + + return StreamingResponse( + event_generator(), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "X-Accel-Buffering": "no", # Disable nginx buffering for SSE + }, + ) + + +# ── Summary ──────────────────────────────────────────────────── + +@router.get("/summary") +async def get_summary( + start_date: Optional[date] = Query(None), + end_date: Optional[date] = Query(None), + session: AsyncSession = Depends(get_session), + _: User = Depends(get_current_admin_user), +): + """Get total unique visitors for a date range.""" + if start_date is None or end_date is None: + start_date, end_date = _default_date_range() + + result = await session.execute( + select(func.coalesce(func.sum(VisitorStatDaily.unique_visitors), 0)).where( + VisitorStatDaily.date >= start_date, + VisitorStatDaily.date <= end_date, + ) + ) + total_unique = result.scalar() + + return {"total_unique_visitors": total_unique} + + +# ── Time series ──────────────────────────────────────────────── + +@router.get("/timeseries") +async def get_timeseries( + start_date: Optional[date] = Query(None), + end_date: Optional[date] = Query(None), + session: AsyncSession = Depends(get_session), + _: User = Depends(get_current_admin_user), +): + """Get daily unique visitor counts for a date range.""" + if start_date is None or end_date is None: + start_date, end_date = _default_date_range() + + stmt = ( + select( + VisitorStatDaily.date, + VisitorStatDaily.unique_visitors, + ) + .where( + VisitorStatDaily.date >= start_date, + VisitorStatDaily.date <= end_date, + ) + .order_by(VisitorStatDaily.date) + ) + result = await session.execute(stmt) + rows = result.all() + return [ + {"date": row.date.isoformat(), "unique_visitors": row.unique_visitors} + for row in rows + ] + + +# ── Geo ──────────────────────────────────────────────────────── + +@router.get("/geo") +async def get_geo_stats( + start_date: Optional[date] = Query(None), + end_date: Optional[date] = Query(None), + session: AsyncSession = Depends(get_session), + _: User = Depends(get_current_admin_user), +): + """Get geographic distribution of visitors for a date range. + + Aggregates by country (country, country_code, lat, lon), summing visitor_count. + Returns one row per country/region for the map to render. + """ + if start_date is None or end_date is None: + start_date, end_date = _default_date_range() + + stmt = ( + select( + VisitorStatGeo.country, + VisitorStatGeo.country_code, + VisitorStatGeo.latitude, + VisitorStatGeo.longitude, + func.sum(VisitorStatGeo.visitor_count).label("total_visitors"), + ) + .where( + VisitorStatGeo.date >= start_date, + VisitorStatGeo.date <= end_date, + ) + .group_by( + VisitorStatGeo.country, + VisitorStatGeo.country_code, + VisitorStatGeo.latitude, + VisitorStatGeo.longitude, + ) + .order_by(desc(func.sum(VisitorStatGeo.visitor_count))) + ) + result = await session.execute(stmt) + rows = result.all() + return [ + { + "country": row.country or "Unknown", + "country_code": row.country_code or "", + "latitude": row.latitude, + "longitude": row.longitude, + "visitors": row.total_visitors, + } + for row in rows + ] diff --git a/backend/app/config.py b/backend/app/config.py index 8b04a71..fe79eea 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -14,6 +14,10 @@ class Settings(BaseSettings): JWT_SECRET_KEY: str = "change-me-in-production" JWT_EXPIRE_MINUTES: int = 1440 # 24 hours + # Visitor stats + LOGS_DIR: str = "/logs" + GEOLITE2_DB_PATH: str = "/app/geo/GeoLite2-City.mmdb" + model_config = {"env_prefix": "KMTN_"} diff --git a/backend/app/log_parser.py b/backend/app/log_parser.py new file mode 100644 index 0000000..03c55c0 --- /dev/null +++ b/backend/app/log_parser.py @@ -0,0 +1,317 @@ +"""Nginx access log parser with IP geolocation enrichment.""" + +import asyncio +import json +import os +from datetime import date, datetime +from typing import Callable, Optional + +import maxminddb + +from app.config import settings +from app.database import async_session +from app.models import LogParseState, VisitorStatDaily, VisitorStatGeo + +# ── Geo reader ────────────────────────────────────────────── + +_geo_reader: Optional[maxminddb.Reader] = None + + +def get_geo_reader() -> maxminddb.Reader: + """Return a singleton MaxMind GeoLite2 City reader.""" + global _geo_reader + if _geo_reader is None: + db_path = settings.GEOLITE2_DB_PATH + if os.path.exists(db_path): + _geo_reader = maxminddb.open_database(db_path) + else: + raise FileNotFoundError( + f"GeoLite2 database not found at {db_path}. " + "Download from https://dev.maxmind.com/geoip/geolite2-free-geolocation-data " + "and place it at backend/geo/GeoLite2-City.mmdb" + ) + return _geo_reader + + +# Country centroid fallback for free GeoLite2 (no city/coordinates). +# Approximate center coordinates per ISO country code — enough for map markers. +_COUNTRY_CENTROIDS: dict[str, tuple[float, float]] = { + "US": (39.8, -98.5), + "GB": (53.5, -1.5), + "DE": (51.2, 10.4), + "FR": (46.6, 2.3), + "JP": (36.2, 138.3), + "AU": (-25.3, 133.8), + "BR": (-14.2, -51.9), + "CA": (56.1, -106.3), + "IN": (20.6, 79.0), + "KR": (35.9, 127.8), + "NL": (52.1, 5.3), + "MX": (23.6, -102.5), + "IT": (41.9, 12.6), + "ES": (40.5, -3.7), + "SE": (62.0, 15.0), + "PL": (51.9, 19.1), + "AR": (-38.4, -63.6), + "ZA": (-30.6, 22.9), + "EG": (26.8, 30.8), + "TR": (39.0, 35.2), + "RU": (61.5, 105.3), + "SG": (1.35, 103.8), + "NZ": (-40.9, 174.9), + "CO": (4.6, -74.3), + "CL": (-35.7, -71.5), + "PE": (-9.2, -75.0), + "PH": (12.9, 121.8), + "ID": (-0.8, 113.9), + "TH": (15.9, 100.9), + "VN": (14.1, 108.3), + "MY": (4.2, 101.9), + "AE": (23.4, 53.8), + "SA": (23.9, 45.1), + "IL": (31.0, 34.9), + "NO": (60.5, 8.5), + "DK": (56.3, 9.5), + "FI": (64.1, 26.5), + "BE": (50.5, 4.5), + "AT": (47.5, 14.6), + "CH": (46.8, 8.2), + "PT": (39.4, -8.2), + "GR": (39.1, 21.8), + "CZ": (49.8, 15.5), + "RO": (45.9, 24.9), + "HU": (47.2, 19.5), + "UA": (48.4, 31.2), + "NG": (9.1, 8.7), + "KE": (-0.02, 37.9), + "GH": (7.9, -1.0), + "CN": (35.9, 104.2), + "PK": (30.4, 69.3), + "BD": (23.7, 90.4), +} + + +def lookup_geo(ip: str) -> dict: + """Look up geolocation for an IP address. + + Returns dict with country, country_code, city, latitude, longitude + or empty dict if lookup fails. Falls back to country centroids + when the free GeoLite2 DB lacks city/coordinate data. + """ + try: + reader = get_geo_reader() + result = reader.get(ip) + if result is None: + return {} + country = result.get("country", {}) + city = result.get("city", {}) + location = result.get("location", {}) + country_code = country.get("iso_code") + lat = location.get("latitude") + lon = location.get("longitude") + + # Free GeoLite2 City often lacks city/coordinates — use centroid fallback + if lat is None or lon is None: + centroid = _COUNTRY_CENTROIDS.get(country_code or "") + if centroid: + lat = lat if lat is not None else centroid[0] + lon = lon if lon is not None else centroid[1] + + return { + "country": country.get("names", {}).get("en", country_code or ""), + "country_code": country_code, + "city": city.get("names", {}).get("en", ""), + "latitude": lat, + "longitude": lon, + } + except Exception: + return {} + + +# ── IP resolution ─────────────────────────────────────────── + + +def resolve_client_ip(entry: dict) -> str: + """Extract the real client IP from a log entry. + + Prefers the leftmost X-Forwarded-For entry (production behind proxy), + falls back to remote_addr (dev). + """ + xff = entry.get("x_forwarded_for", "") + if xff and xff != "-": + # Leftmost entry is the original client + return xff.split(",")[0].strip() + return entry.get("remote_addr", "") + + +# ── Log parsing ───────────────────────────────────────────── + + +def parse_log_line(line: str) -> Optional[dict]: + """Parse a single JSON log line. Returns dict or None if unparseable.""" + line = line.strip() + if not line: + return None + try: + return json.loads(line) + except (json.JSONDecodeError, ValueError): + return None + + +def parse_log_file(filepath: str) -> dict: + """Parse a single daily log file and return aggregated counts. + + Returns a dict with: + - unique_ips: set of unique client IP addresses + - geo_counts: dict of (country_code, country, city, lat, lon) -> count + """ + unique_ips: set[str] = set() + geo_counts: dict[tuple, int] = {} + + with open(filepath, "r") as f: + for line in f: + entry = parse_log_line(line) + if entry is None: + continue + + ip = resolve_client_ip(entry) + if not ip: + continue + + unique_ips.add(ip) + + # Geo enrichment + geo = lookup_geo(ip) + if geo.get("country_code"): + geo_key = ( + geo.get("country_code", ""), + geo.get("country", ""), + geo.get("city", ""), + geo.get("latitude"), + geo.get("longitude"), + ) + geo_counts[geo_key] = geo_counts.get(geo_key, 0) + 1 + + return { + "unique_ips": unique_ips, + "geo_counts": geo_counts, + } + + +async def process_unparsed_logs( + progress_callback: Optional[Callable] = None, +) -> dict: + """Scan /logs/ for unprocessed daily files, parse them, store aggregates. + + Only processes files strictly older than today (file must be complete). + Advances the LogParseState marker after each successful file. + + Returns a summary dict with files_processed and rows_inserted. + """ + logs_dir = settings.LOGS_DIR + if not os.path.isdir(logs_dir): + return {"files_processed": 0, "error": f"Logs directory not found: {logs_dir}"} + + # Get current parse state + async with async_session() as session: + from sqlalchemy import select + + result = await session.execute( + select(LogParseState).where(LogParseState.key == "default") + ) + state = result.scalar_one_or_none() + + if state is None: + # Initialize with a far-past date so all files get processed + state = LogParseState(key="default", last_parsed_date=date(2020, 1, 1)) + session.add(state) + await session.commit() + + last_parsed = state.last_parsed_date + + # Discover daily log files + import glob + + pattern = os.path.join(logs_dir, "access-*.log") + log_files = sorted(glob.glob(pattern)) + + files_processed = 0 + rows_inserted = 0 + today = date.today() + + for filepath in log_files: + # Extract date from filename + basename = os.path.basename(filepath) + try: + date_str = basename.replace("access-", "").replace(".log", "") + file_date = date.fromisoformat(date_str) + except ValueError: + continue + + # Only process files strictly before today (file must be complete) + if file_date >= today: + continue + + # Only process files after the last parsed date + if file_date <= last_parsed: + continue + + # Parse this file + try: + stats = parse_log_file(filepath) + except Exception as e: + print(f" ERROR parsing {filepath}: {e}") + continue + + # Store aggregates in database + async with async_session() as session: + from sqlalchemy import select + + # Upsert daily stats + result = await session.execute( + select(VisitorStatDaily).where(VisitorStatDaily.date == file_date) + ) + daily = result.scalar_one_or_none() + if daily is None: + daily = VisitorStatDaily(date=file_date) + session.add(daily) + + daily.unique_visitors = len(stats["unique_ips"]) + + # Geo stats + for geo_key, count in stats["geo_counts"].items(): + geo_row = VisitorStatGeo( + date=file_date, + country_code=geo_key[0], + country=geo_key[1], + city=geo_key[2], + latitude=geo_key[3], + longitude=geo_key[4], + visitor_count=count, + ) + session.add(geo_row) + rows_inserted += 1 + + # Advance parse state + state = ( + await session.execute( + select(LogParseState).where(LogParseState.key == "default") + ) + ).scalar_one() + state.last_parsed_date = file_date + state.updated_at = datetime.utcnow() + + await session.commit() + + files_processed += 1 + + # Report progress via callback (for SSE streaming) + if progress_callback: + cb_result = progress_callback( + file_date.isoformat(), files_processed, len(log_files) + ) + # Support both sync and async callbacks + if cb_result is not None and asyncio.iscoroutine(cb_result): + await cb_result + + return {"files_processed": files_processed, "rows_inserted": rows_inserted} diff --git a/backend/app/main.py b/backend/app/main.py index 1ac648a..e6564b6 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1,4 +1,6 @@ from contextlib import asynccontextmanager +import asyncio +import os from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware @@ -8,7 +10,7 @@ from app.config import settings from app.database import async_session, engine from app.models import Base, Show, StationConfig, HistoryEntry, TeamMember, CommunityHighlight, Underwriter from app.user_models import User -from app.api import events, auth, station_config, history, team, community, shows, storage, underwriters +from app.api import events, auth, station_config, history, team, community, shows, storage, underwriters, stats def _migrate_add_missing_columns(sync_conn): @@ -117,6 +119,27 @@ async def lifespan(app: FastAPI): await session.commit() print(f" → Bootstrap admin created: {settings.ADMIN_USERNAME}") + # Check GeoLite2 database + geo_db = settings.GEOLITE2_DB_PATH + if os.path.exists(geo_db): + print(f" ✓ GeoLite2 database loaded from {geo_db}") + else: + print(f" WARNING: GeoLite2 database not found at {geo_db} — geo lookups will be disabled") + + # Auto-parse any unprocessed logs on startup (background task) + async def _startup_parse(): + try: + from app.log_parser import process_unparsed_logs + result = await process_unparsed_logs() + if result.get("files_processed"): + print(f" ✓ Parsed {result['files_processed']} log files ({result['rows_inserted']} geo rows)") + else: + print(" No unprocessed log files to parse") + except Exception as e: + print(f" ERROR during startup log parse: {e}") + + asyncio.create_task(_startup_parse()) + yield @@ -145,6 +168,7 @@ app.include_router(community.router, prefix="/api/community", tags=["community"] app.include_router(shows.router, prefix="/api/shows", tags=["shows"]) app.include_router(storage.router, prefix="/api/storage", tags=["storage"]) app.include_router(underwriters.router, prefix="/api/underwriters", tags=["underwriters"]) +app.include_router(stats.router, prefix="/api/stats", tags=["stats"]) # Dev-only admin router (disabled in production) if settings.ENV != "production": diff --git a/backend/app/models.py b/backend/app/models.py index 12cb0f5..50d24f8 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -3,6 +3,7 @@ from datetime import datetime from sqlalchemy import ( Column, DateTime, + Float, Integer, LargeBinary, String, @@ -11,6 +12,7 @@ from sqlalchemy import ( Date, ForeignKey, UniqueConstraint, + Index, ) from sqlalchemy.orm import DeclarativeBase, relationship @@ -159,3 +161,47 @@ class StorageBlob(Base): size = Column(Integer, nullable=False) data = Column(LargeBinary, nullable=False) created_at = Column(DateTime, nullable=False, default=datetime.utcnow) + + +# ── Visitor Stats ────────────────────────────────────────────── + +class VisitorStatDaily(Base): + """Daily aggregated visitor statistics.""" + __tablename__ = "visitor_stats_daily" + + id = Column(Integer, primary_key=True, autoincrement=True) + date = Column(Date, nullable=False, unique=True) + unique_visitors = Column(Integer, nullable=False, default=0) + + __table_args__ = ( + Index("idx_visitor_stats_date", "date"), + ) + + +class VisitorStatGeo(Base): + """Geographic breakdown of visitors per day.""" + __tablename__ = "visitor_stats_geo" + + id = Column(Integer, primary_key=True, autoincrement=True) + date = Column(Date, nullable=False) + country = Column(String(100), nullable=True) + country_code = Column(String(10), nullable=True) + city = Column(String(200), nullable=True) + latitude = Column(Float, nullable=True) + longitude = Column(Float, nullable=True) + visitor_count = Column(Integer, nullable=False, default=0) + + __table_args__ = ( + Index("idx_visitor_stats_geo_date", "date"), + Index("idx_visitor_stats_geo_country", "country_code"), + ) + + +class LogParseState(Base): + """Tracks the last successfully parsed log date.""" + __tablename__ = "log_parse_state" + + id = Column(Integer, primary_key=True, autoincrement=True) + key = Column(String(20), unique=True, nullable=False, default="default") + last_parsed_date = Column(Date, nullable=False) + updated_at = Column(DateTime, nullable=False, default=datetime.utcnow) diff --git a/backend/app/schemas.py b/backend/app/schemas.py index f8515ae..6c80ce7 100644 --- a/backend/app/schemas.py +++ b/backend/app/schemas.py @@ -1,7 +1,7 @@ from datetime import date as _date from typing import Optional -from pydantic import BaseModel, computed_field, field_validator +from pydantic import BaseModel, computed_field # ── Shared ─────────────────────────────────────────────── @@ -284,3 +284,27 @@ class ShowUpdate(BaseModel): display_order: Optional[int] = None active: Optional[bool] = None schedules: Optional[list[ShowScheduleCreate]] = None + + +# ── Stats ──────────────────────────────────────────────────────── + +class StatsSummaryResponse(BaseModel): + total_unique_visitors: int + + +class TimeSeriesPoint(BaseModel): + date: str + unique_visitors: int + + +class GeoStatResponse(BaseModel): + country: str + country_code: str + latitude: Optional[float] = None + longitude: Optional[float] = None + visitors: int + + +class ParseResultResponse(BaseModel): + files_processed: int + rows_inserted: int diff --git a/backend/geo/README.md b/backend/geo/README.md new file mode 100644 index 0000000..f7e9281 --- /dev/null +++ b/backend/geo/README.md @@ -0,0 +1,15 @@ +# GeoLite2 City Database + +The visitor stats dashboard uses MaxMind GeoLite2 for IP geolocation. + +## Setup + +1. Create a free account at [MaxMind GeoLite2](https://dev.maxmind.com/geoip/geolite2-free-geolocation-data) +2. Download the **GeoLite2 City** database (`.mmdb` format, not CSV) +3. Place the downloaded file here as `GeoLite2-City.mmdb` + +The file is git-ignored (~60 MB, licensed). The Docker build copies it into the container at `/app/geo/GeoLite2-City.mmdb`. + +## Without the database + +The backend starts but geo lookups are disabled. The dashboard shows visitor counts without country/region breakdown. diff --git a/backend/inject_test_logs.py b/backend/inject_test_logs.py new file mode 100644 index 0000000..795ebd1 --- /dev/null +++ b/backend/inject_test_logs.py @@ -0,0 +1,169 @@ +"""Inject test visitor log entries into the shared logs directory. + +Writes JSON-formatted access log lines (matching the nginx kmtn_json format) +for multiple past days so the log parser will pick them up — data is only +aggregated a day in arrears, so all dates are strictly before today. + +Usage: + python inject_test_logs.py # writes to /logs (default) + python inject_test_logs.py /tmp/logs # writes to custom path + python inject_test_logs.py /logs 14 # writes 14 days of data +""" + +import json +import os +import sys +import hashlib +import struct +from datetime import date, timedelta, timezone + +# Real, well-known public IPs that resolve in MaxMind GeoLite2 City. +# These are actual infrastructure IPs (DNS, CDN, search, cloud) — +# not random addresses, so they're far more likely to be in the DB. +# Each tuple: (ip, expected_country, expected_city) +TEST_IPS = [ + # United States + ("8.8.8.8", "US", "Mountain View"), # Google DNS + ("8.8.4.4", "US", "Mountain View"), # Google DNS + ("216.58.214.206", "US", "Mountain View"), # Google + ("142.250.80.46", "US", "Mountain View"), # Google + ("13.107.42.14", "US", "Redmond"), # Microsoft + ("20.190.128.1", "US", "Boydton"), # Microsoft Azure + ("208.67.222.222", "US", "San Francisco"), # OpenDNS + ("208.67.220.220", "US", "San Francisco"), # OpenDNS + ("151.101.1.69", "US", "San Francisco"), # Fastly + ("52.84.74.1", "US", "Seattle"), # AWS CloudFront + ("99.86.0.1", "US", "Seattle"), # AWS + ("54.239.28.85", "US", "Seattle"), # AWS + ("185.199.108.1", "US", "San Francisco"), # GitHub Pages + # United Kingdom + ("212.58.244.3", "GB", "London"), # Virgin Media + ("178.173.57.1", "GB", "London"), # DigitalOcean + ("51.15.48.1", "GB", "London"), # OVH UK + # France + ("51.140.137.15", "FR", "Paris"), # OVH Paris + ("193.70.150.1", "FR", "Paris"), # Free + # Germany + ("217.160.0.1", "DE", "Frankfurt"), # Telekom + ("212.227.156.1", "DE", "Munich"), # Deutsche Telekom + ("217.69.70.1", "DE", "Frankfurt"), # Freenet + # Japan + ("210.170.232.46", "JP", "Tokyo"), # KDDI + ("202.214.69.1", "JP", "Tokyo"), # NTT + ("210.188.206.1", "JP", "Tokyo"), # Softbank + # Australia + ("1.35.128.1", "AU", "Sydney"), # Telstra + ("203.50.224.1", "AU", "Melbourne"), # iiNet + # Brazil + ("177.72.120.1", "BR", "Sao Paulo"), # Vivo + ("200.149.130.1", "BR", "Rio de Janeiro"), # Telefonica Brasil + ("189.75.1.1", "BR", "Sao Paulo"), # Claro + # Canada + ("76.69.248.1", "CA", "Toronto"), # Bell + ("24.199.12.1", "CA", "Vancouver"), # Shaw + ("99.248.1.1", "CA", "Montreal"), # Bell + # India + ("117.239.1.1", "IN", "Mumbai"), # Reliance Jio + ("180.151.1.1", "IN", "New Delhi"), # Airtel + ("106.192.1.1", "IN", "Bangalore"), # BSNL + # South Korea + ("1.235.198.1", "KR", "Seoul"), # KT + ("211.36.1.1", "KR", "Seoul"), # SK Telecom + # Netherlands + ("185.30.1.1", "NL", "Amsterdam"), # KPN + ("94.23.1.1", "NL", "Amsterdam"), # XS4ALL + # Mexico + ("189.203.1.1", "MX", "Mexico City"), # Telmex + ("200.44.1.1", "MX", "Guadalajara"), # Telcel + # Italy + ("217.20.1.1", "IT", "Rome"), # Telecom Italia + ("93.45.1.1", "IT", "Milan"), # TIM + # Spain + ("213.99.1.1", "ES", "Madrid"), # Telefonica Espana + ("88.0.1.1", "ES", "Madrid"), # Movistar + # Sweden + ("213.248.1.1", "SE", "Stockholm"), # Telia + # Poland + ("79.133.1.1", "PL", "Warsaw"), # Orange Polska + # Argentina + ("190.15.1.1", "AR", "Buenos Aires"), # Telecom Argentina + # South Africa + ("196.201.1.1", "ZA", "Johannesburg"), # Telkom SA + # Egypt + ("213.53.1.1", "EG", "Cairo"), # Telecom Egypt + # Turkey + ("37.146.1.1", "TR", "Istanbul"), # Turk Telekom + # Russia + ("77.88.55.1", "RU", "Moscow"), # Yandex DNS + ("5.255.255.1", "RU", "Moscow"), # Rostelecom + # Singapore + ("182.16.1.1", "SG", "Singapore"), # Singtel + # New Zealand + ("210.55.1.1", "NZ", "Auckland"), # Telecom NZ +] + + +def _deterministic_count(seed: str) -> int: + """Return a pseudo-random visit count (1-8) for a given seed.""" + h = hashlib.md5(seed.encode()).digest() + return struct.unpack("I", h[:4])[0] % 8 + 1 + + +def _deterministic_hour(seed: str) -> int: + """Return a pseudo-random hour (0-23) for a given seed.""" + h = hashlib.md5(seed.encode()).digest() + return struct.unpack("I", h[:4])[0] % 24 + + +def write_test_logs(logs_dir: str, days: int = 14): + """Write test log entries for the given number of past days. + + Skips today (parser only processes files strictly before today). + Writes a realistic mix of visitors per day with varying counts. + """ + os.makedirs(logs_dir, exist_ok=True) + + today = date.today() + total_entries = 0 + files_written = 0 + + # Write for the last `days` days, skipping today + for offset in range(1, days + 1): + file_date = today - timedelta(days=offset) + + filename = f"access-{file_date.isoformat()}.log" + filepath = os.path.join(logs_dir, filename) + + entries: list[str] = [] + for ip, country, city in TEST_IPS: + seed = f"{file_date}-{ip}" + count = _deterministic_count(seed) + hour = _deterministic_hour(seed) + + for j in range(count): + # Deterministic minute spread (hash() is non-deterministic across runs) + ip_hash = struct.unpack("I", hashlib.md5(ip.encode()).digest()[4:8])[0] + minute = (j * 7 + ip_hash % 60) % 60 + entry = { + "time": f"{file_date}T{hour}:{minute:02d}:00+00:00", + "remote_addr": ip, + "x_forwarded_for": "-", + } + entries.append(json.dumps(entry)) + + with open(filepath, "a") as f: + for line in entries: + f.write(line + "\n") + + total_entries += len(entries) + files_written += 1 + + print(f"Wrote {total_entries} log entries across {files_written} daily files ({len(TEST_IPS)} unique IPs)") + print(f"Date range: {(today - timedelta(days=days)).isoformat()} to {(today - timedelta(days=1)).isoformat()}") + print(f"Directory: {logs_dir}") + + +if __name__ == "__main__": + target = sys.argv[1] if len(sys.argv) > 1 else "/logs" + num_days = int(sys.argv[2]) if len(sys.argv) > 2 else 14 + write_test_logs(target, days=num_days) diff --git a/backend/probe_geo.py b/backend/probe_geo.py new file mode 100644 index 0000000..74c41b2 --- /dev/null +++ b/backend/probe_geo.py @@ -0,0 +1,200 @@ +"""Probe the GeoLite2 database to find IPs that resolve to real countries. + +Run inside the api container: + docker compose exec api python /app/probe_geo.py + +Or locally if the DB is present: + python probe_geo.py /path/to/GeoLite2-City.mmdb + +Tests a curated list of well-known public IPs and prints the ones that resolve. +""" + +import sys +import json +import os +import maxminddb + +# Well-known public IPs from various countries +CANDIDATE_IPS = [ + # United States + ("23.21.30.100", "US"), + ("72.14.198.234", "US"), + ("68.49.71.1", "US"), + ("208.67.222.222", "US"), # OpenDNS + ("74.125.200.100", "US"), # Google + ("151.101.1.69", "US"), # Fastly + ("142.250.80.46", "US"), # Google + ("216.58.214.206", "US"), # Google + ("172.217.164.110", "US"), # Google + ("13.107.42.14", "US"), # Microsoft + ("20.190.128.1", "US"), # Microsoft + ("52.84.74.1", "US"), # AWS CloudFront + ("99.86.0.1", "US"), # AWS + # United Kingdom + ("81.2.69.142", "GB"), + ("212.58.224.10", "GB"), + ("51.140.137.15", "GB"), + ("178.173.57.1", "GB"), + # France + ("51.140.137.15", "FR"), + ("193.70.150.1", "FR"), + # Japan + ("210.170.232.46", "JP"), + ("133.244.200.1", "JP"), + ("202.214.69.1", "JP"), # NTT + ("210.188.206.1", "JP"), # Softbank + # Germany + ("217.160.0.1", "DE"), + ("212.227.156.1", "DE"), + ("2a02:2e0::1", "DE"), # IPv6 - may not resolve + ("217.69.70.1", "DE"), + # Australia + ("101.168.100.1", "AU"), + ("203.50.224.1", "AU"), + ("1.35.128.1", "AU"), # Telstra + # Brazil + ("177.72.120.1", "BR"), + ("200.149.130.1", "BR"), + ("189.75.1.1", "BR"), + # Canada + ("76.69.248.1", "CA"), + ("24.199.12.1", "CA"), + ("99.248.1.1", "CA"), # Bell + # India + ("117.239.1.1", "IN"), + ("180.151.1.1", "IN"), + ("106.192.1.1", "IN"), + # South Korea + ("1.235.198.1", "KR"), + ("211.36.1.1", "KR"), + # Netherlands + ("185.30.1.1", "NL"), + ("94.23.1.1", "NL"), + # Mexico + ("189.203.1.1", "MX"), + ("200.44.1.1", "MX"), + # Italy + ("217.20.1.1", "IT"), + ("93.45.1.1", "IT"), + # Spain + ("213.99.1.1", "ES"), + ("88.0.1.1", "ES"), + # Sweden + ("213.248.1.1", "SE"), + # Poland + ("79.133.1.1", "PL"), + # Argentina + ("190.15.1.1", "AR"), + # South Africa + ("196.201.1.1", "ZA"), + # Egypt + ("213.53.1.1", "EG"), + # Turkey + ("37.146.1.1", "TR"), + # Russia + ("77.88.55.1", "RU"), # Yandex DNS + ("5.255.255.1", "RU"), +] + + +def probe(db_path: str): + """Test each candidate IP against the GeoLite2 database.""" + reader = maxminddb.open_database(db_path) + results = [] + + # Deduplicate IPs but keep all expected countries + seen_ips = set() + for ip, expected_country in CANDIDATE_IPS: + if ip in seen_ips: + continue + seen_ips.add(ip) + + try: + result = reader.lookup(ip) + if result is None: + continue + + country = result.get("country", {}) + city = result.get("city", {}) + location = result.get("location", {}) + + country_code = country.get("iso_code", "") + country_name = country.get("names", {}).get("en", country_code) + city_name = city.get("names", {}).get("en", "") + lat = location.get("latitude") + lon = location.get("longitude") + + results.append({ + "ip": ip, + "country": country_name, + "country_code": country_code, + "city": city_name, + "latitude": lat, + "longitude": lon, + }) + except Exception as e: + print(f" ERROR looking up {ip}: {e}") + + reader.close() + return results + + +def main(): + # Determine DB path + db_path = None + + if len(sys.argv) > 1: + # Explicit path provided + db_path = sys.argv[1] + else: + # Try common locations + candidates = [ + "/app/geo/GeoLite2-City.mmdb", # container path + os.path.join(os.path.dirname(__file__), "geo", "GeoLite2-City.mmdb"), # local + ] + for p in candidates: + if os.path.exists(p): + db_path = p + break + + if db_path is None: + print("GeoLite2-City.mmdb not found.") + print("Usage:") + print(" docker compose exec api python /app/probe_geo.py") + print(" python probe_geo.py /path/to/GeoLite2-City.mmdb") + sys.exit(1) + + print(f"Probing GeoLite2 at: {db_path}") + results = probe(db_path) + + if not results: + print("No IPs resolved! The database might be empty or outdated.") + sys.exit(1) + + print(f"\nResolved {len(results)} IPs:\n") + + # Group by country + by_country = {} + for r in results: + by_country.setdefault(r["country_code"], []).append(r) + + for cc in sorted(by_country.keys()): + entries = by_country[cc] + print(f" {cc} ({entries[0]['country']}):") + for e in entries: + print(f" {e['ip']:20s} -> {e['city'] or 'N/A':20s} ({e['latitude']}, {e['longitude']})") + + # Output as Python tuples for inject_test_logs.py + print(f"\n\nTuples for inject_test_logs.py TEST_IPS:") + print("TEST_IPS = [") + for r in results: + print(f' ("{r["ip"]}", "{r["country"]}", "{r["city"]}"),') + print("]") + + # Also output JSON for convenience + print(f"\nJSON output:") + print(json.dumps(results, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/backend/requirements.txt b/backend/requirements.txt index f37a45f..2bf94a6 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -11,3 +11,4 @@ pydantic==2.10.4 pydantic-settings==2.7.1 python-jose[cryptography]==3.3.0 httpx==0.27.2 +maxminddb==2.6.2 diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 1ec448a..79da340 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -10,6 +10,8 @@ services: KMTN_ENV: production # Override with your actual production domain(s) KMTN_CORS_ORIGINS: '["https://yourdomain.com"]' + volumes: + - logs:/logs # No port mapping — traffic arrives via reverse proxy / load balancer frontend: @@ -18,4 +20,6 @@ services: API_UPSTREAM: https://api.yourdomain.com # Browser-facing API URL. Empty = proxy mode. Set to direct URL if ingress breaks proxy chain. API_PUBLIC_URL: "" + volumes: + - logs:/logs # No port mapping — traffic arrives via reverse proxy / load balancer diff --git a/docker-compose.yml b/docker-compose.yml index a76cfb9..42f4104 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,6 +28,8 @@ services: KMTN_CORS_ORIGINS: '["http://localhost:4200", "http://localhost"]' ports: - "8000:8000" + volumes: + - logs:/logs depends_on: db: condition: service_healthy @@ -42,8 +44,11 @@ services: API_PUBLIC_URL: "" ports: - "4200:80" + volumes: + - logs:/logs depends_on: - api volumes: pgdata: + logs: diff --git a/nginx.conf.template b/nginx.conf.template index 59cf4c0..deec1ff 100644 --- a/nginx.conf.template +++ b/nginx.conf.template @@ -1,3 +1,18 @@ +# Map ISO timestamp to date-only string for daily log rotation +map $time_iso8601 $log_date { + ~^(?\d{4}-\d{2}-\d{2}) $date; + default "unknown"; +} + +log_format kmtn_json escape=json + '{' + '"time":"$time_iso8601",' + '"remote_addr":"$remote_addr",' + '"x_forwarded_for":"$http_x_forwarded_for"' + '}'; + +access_log /logs/access-$log_date.log kmtn_json; + server { listen 80; server_name _; diff --git a/package-lock.json b/package-lock.json index 118a06a..382cb78 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,12 +8,18 @@ "name": "radio-station", "version": "0.0.0", "dependencies": { + "@angular/cdk": "^21.2.0", "@angular/common": "^21.2.0", "@angular/compiler": "^21.2.0", "@angular/core": "^21.2.0", "@angular/forms": "^21.2.0", "@angular/platform-browser": "^21.2.0", "@angular/router": "^21.2.0", + "@types/leaflet": "^1.9.17", + "chart.js": "^4.4.0", + "leaflet": "^1.9.4", + "ng2-charts": "^10.0.0", + "ngx-leaflet": "^0.0.16", "rxjs": "~7.8.0", "tslib": "^2.3.0", "zone.js": "^0.16.2" @@ -415,6 +421,22 @@ } } }, + "node_modules/@angular/cdk": { + "version": "21.2.14", + "resolved": "https://registry.npmjs.org/@angular/cdk/-/cdk-21.2.14.tgz", + "integrity": "sha512-806REq/CLf37nEhmmd8Q+ILN8z/RVG2vk2n8YZ/4TdHpcBCi5ux4AxLbpMmduLwGPOzPagJ6ggRzE5fnX0rmcQ==", + "license": "MIT", + "dependencies": { + "parse5": "^8.0.0", + "tslib": "^2.3.0" + }, + "peerDependencies": { + "@angular/common": "^21.0.0 || ^22.0.0", + "@angular/core": "^21.0.0 || ^22.0.0", + "@angular/platform-browser": "^21.0.0 || ^22.0.0", + "rxjs": "^6.5.3 || ^7.4.0" + } + }, "node_modules/@angular/cli": { "version": "21.2.14", "resolved": "https://registry.npmjs.org/@angular/cli/-/cli-21.2.14.tgz", @@ -1821,6 +1843,12 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@kurkle/color": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/@kurkle/color/-/color-0.3.4.tgz", + "integrity": "sha512-M5UknZPHRu3DEDWoipU6sE8PdkZ6Z/S+v4dD+Ke8IaNlpdSQah50lz1KtcFBa2vsdOnwbbnxJwVM4wty6udA5w==", + "license": "MIT" + }, "node_modules/@listr2/prompt-adapter-inquirer": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/@listr2/prompt-adapter-inquirer/-/prompt-adapter-inquirer-3.0.5.tgz", @@ -3662,6 +3690,21 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/geojson": { + "version": "7946.0.16", + "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.16.tgz", + "integrity": "sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg==", + "license": "MIT" + }, + "node_modules/@types/leaflet": { + "version": "1.9.21", + "resolved": "https://registry.npmjs.org/@types/leaflet/-/leaflet-1.9.21.tgz", + "integrity": "sha512-TbAd9DaPGSnzp6QvtYngntMZgcRk+igFELwR2N99XZn7RXUdKgsXMR+28bUO0rPsWp8MIu/f47luLIQuSLYv/w==", + "license": "MIT", + "dependencies": { + "@types/geojson": "*" + } + }, "node_modules/@vitejs/plugin-basic-ssl": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/@vitejs/plugin-basic-ssl/-/plugin-basic-ssl-2.1.4.tgz", @@ -4063,6 +4106,18 @@ "dev": true, "license": "MIT" }, + "node_modules/chart.js": { + "version": "4.5.1", + "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.5.1.tgz", + "integrity": "sha512-GIjfiT9dbmHRiYi6Nl2yFCq7kkwdkp1W/lp2J99rX0yo9tgJGn3lKQATztIjb5tVtevcBtIdICNWqlq5+E8/Pw==", + "license": "MIT", + "dependencies": { + "@kurkle/color": "^0.3.0" + }, + "engines": { + "pnpm": ">=8" + } + }, "node_modules/chokidar": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-5.0.0.tgz", @@ -4557,6 +4612,16 @@ "node": ">= 0.4" } }, + "node_modules/es-toolkit": { + "version": "1.49.0", + "resolved": "https://registry.npmjs.org/es-toolkit/-/es-toolkit-1.49.0.tgz", + "integrity": "sha512-G5iZ6Pc/FNRY/soKZHC+TxGDD83rHUDXxzaWhGCX44vAv/tMs56WMusnm/KMNK+luUPsgA9U28cGr4RDlSzL2g==", + "license": "MIT", + "workspaces": [ + "docs", + "benchmarks" + ] + }, "node_modules/esbuild": { "version": "0.27.3", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.3.tgz", @@ -5379,6 +5444,12 @@ ], "license": "MIT" }, + "node_modules/leaflet": { + "version": "1.9.4", + "resolved": "https://registry.npmjs.org/leaflet/-/leaflet-1.9.4.tgz", + "integrity": "sha512-nxS1ynzJOmOlHp+iL3FyWqK89GtNL8U8rvlMOsQdTTssxZwCXh8N2NB3GDQOL+YR3XnWyZAxwQixURb+FA74PA==", + "license": "BSD-2-Clause" + }, "node_modules/listr2": { "version": "9.0.5", "resolved": "https://registry.npmjs.org/listr2/-/listr2-9.0.5.tgz", @@ -5908,6 +5979,38 @@ "node": ">= 0.6" } }, + "node_modules/ng2-charts": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/ng2-charts/-/ng2-charts-10.0.0.tgz", + "integrity": "sha512-mdL75XJrk/0s0YO2ySPQpAHPja85ECDEGNWFlcElJiy/bYliTNGEpeCtctAqZuozTff/E2CwGjyfPFM1ScP2og==", + "license": "MIT", + "dependencies": { + "es-toolkit": "^1.39.7", + "tslib": "^2.3.0" + }, + "peerDependencies": { + "@angular/cdk": ">=21.0.0", + "@angular/common": ">=21.0.0", + "@angular/core": ">=21.0.0", + "@angular/platform-browser": ">=21.0.0", + "chart.js": "^3.4.0 || ^4.0.0", + "rxjs": "^6.5.3 || ^7.4.0" + } + }, + "node_modules/ngx-leaflet": { + "version": "0.0.16", + "resolved": "https://registry.npmjs.org/ngx-leaflet/-/ngx-leaflet-0.0.16.tgz", + "integrity": "sha512-4cm06T9ZrD3i0VI+/fghrfx/Oe3UkOcYsDYKLFgh+XkoBvCt3oXIgdrABjoXIgMLcKtzS/oDHcVgRkDy/x5/MA==", + "dependencies": { + "tslib": "^2.3.0" + }, + "peerDependencies": { + "@angular/common": "*", + "@angular/core": "*", + "@types/leaflet": "^1.9.0", + "leaflet": "^1.9.3" + } + }, "node_modules/node-addon-api": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz", @@ -6276,7 +6379,6 @@ "version": "8.0.1", "resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.1.tgz", "integrity": "sha512-z1e/HMG90obSGeidlli3hj7cbocou0/wa5HacvI3ASx34PecNjNQeaHNo5WIZpWofN9kgkqV1q5YvXe3F0FoPw==", - "dev": true, "license": "MIT", "dependencies": { "entities": "^8.0.0" @@ -6330,7 +6432,6 @@ "version": "8.0.0", "resolved": "https://registry.npmjs.org/entities/-/entities-8.0.0.tgz", "integrity": "sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA==", - "dev": true, "license": "BSD-2-Clause", "engines": { "node": ">=20.19.0" diff --git a/package.json b/package.json index 78d8389..2233974 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,12 @@ "@angular/forms": "^21.2.0", "@angular/platform-browser": "^21.2.0", "@angular/router": "^21.2.0", + "@types/leaflet": "^1.9.17", + "@angular/cdk": "^21.2.0", + "chart.js": "^4.4.0", + "leaflet": "^1.9.4", + "ng2-charts": "^10.0.0", + "ngx-leaflet": "^0.0.16", "rxjs": "~7.8.0", "tslib": "^2.3.0", "zone.js": "^0.16.2" diff --git a/src/app/admin/admin-stats-dashboard.component.html b/src/app/admin/admin-stats-dashboard.component.html new file mode 100644 index 0000000..f330183 --- /dev/null +++ b/src/app/admin/admin-stats-dashboard.component.html @@ -0,0 +1,103 @@ +
+ +
+
+ @for (preset of presets; track preset.value) { + + } +
+ + @if (selectedPreset === 'custom') { +
+ + +
+ } + + +
+ + @if (parseProgress) { +
{{ parseProgress }}
+ } + + @if (error) { +
{{ error }}
+ } + + @if (loading) { +
Loading stats...
+ } @else { + +
+
+
{{ summary?.total_unique_visitors ?? 0 }}
+
Unique Visitors
+
+
+
{{ geoStats.length }}
+
Countries
+
+
+ + +
+

Visitors by Location

+
+
+
+
+ + +
+

Visitors Over Time

+
+ +
+
+ + +
+

Visitors by Country

+ + + + + + + + + + @for (geo of geoStats; track geo.country_code) { + + + + + + } @empty { + + } + +
CountryCodeVisitors
{{ geo.country }}{{ geo.country_code }}{{ geo.visitors }}
No geo data available yet
+
+ } +
\ No newline at end of file diff --git a/src/app/admin/admin-stats-dashboard.component.scss b/src/app/admin/admin-stats-dashboard.component.scss new file mode 100644 index 0000000..04ed9b4 --- /dev/null +++ b/src/app/admin/admin-stats-dashboard.component.scss @@ -0,0 +1,197 @@ +@use '../../styles/variables' as *; +@use '../../styles/mixins' as *; + +.stats-dashboard { + padding: $spacing-md; +} + +// ── Controls ────────────────────────────────────────────── + +.stats-controls { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: $spacing-md; + margin-bottom: $spacing-lg; +} + +.preset-buttons { + display: flex; + flex-wrap: wrap; + gap: $spacing-xs; +} + +.preset-btn { + background: $neutral-white; + border: 1px solid $neutral-light; + border-radius: $radius-md; + padding: $spacing-xs $spacing-md; + font-size: 0.85rem; + font-weight: 600; + color: $neutral-dark; + cursor: pointer; + transition: all $transition-fast; + + &:hover { + border-color: $primary-blue-muted; + color: $primary-blue; + } + + &.active { + background: $primary-blue; + border-color: $primary-blue; + color: $neutral-white; + } +} + +.custom-dates { + display: flex; + gap: $spacing-sm; + align-items: center; + + label { + font-size: 0.85rem; + color: $neutral-medium; + display: flex; + align-items: center; + gap: $spacing-xs; + } + + input[type="date"] { + padding: $spacing-xs $spacing-sm; + border: 1px solid $neutral-light; + border-radius: $radius-sm; + font-size: 0.85rem; + } +} + +.btn-parse { + @include button-style($success-green, $neutral-white, darken($success-green, 8%)); + margin-left: auto; +} + +.parse-progress { + background: rgba($info-blue, 0.1); + color: $info-blue; + padding: $spacing-sm $spacing-md; + border-radius: $radius-md; + font-size: 0.85rem; + margin-bottom: $spacing-md; +} + +.error-message { + background: rgba(#d32f2f, 0.1); + color: #d32f2f; + padding: $spacing-sm $spacing-md; + border-radius: $radius-md; + font-size: 0.85rem; + margin-bottom: $spacing-md; +} + +// ── Summary cards ───────────────────────────────────────── + +.stats-summary { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); + gap: $spacing-md; + margin-bottom: $spacing-lg; +} + +.stat-card { + @include card-style; + padding: $spacing-lg; + text-align: center; + + .stat-value { + font-family: $font-heading; + font-size: 2rem; + color: $primary-blue; + margin-bottom: $spacing-xs; + } + + .stat-label { + font-size: 0.85rem; + color: $neutral-medium; + text-transform: uppercase; + letter-spacing: 0.5px; + } +} + +// ── Map ─────────────────────────────────────────────────── + +.map-section { + @include card-style; + padding: $spacing-lg; + margin-bottom: $spacing-lg; + + h3 { + font-family: $font-heading; + color: $primary-blue; + margin: 0 0 $spacing-md; + } +} + +.map-container { + height: 400px; + border-radius: $radius-md; + overflow: hidden; + position: relative; + + .map-leaflet { + width: 100%; + height: 100%; + } +} + +// ── Chart ───────────────────────────────────────────────── + +.chart-section { + @include card-style; + padding: $spacing-lg; + margin-bottom: $spacing-lg; + + h3 { + font-family: $font-heading; + color: $primary-blue; + margin: 0 0 $spacing-md; + } +} + +.chart-container { + height: 300px; + position: relative; +} + +// ── Geo table ───────────────────────────────────────────── + +.geo-table-section { + @include card-style; + padding: $spacing-lg; + + h3 { + font-family: $font-heading; + color: $primary-blue; + margin: 0 0 $spacing-md; + } +} + +// ── Responsive ──────────────────────────────────────────── + +@include responsive(md) { + .stats-controls { + flex-direction: column; + align-items: flex-start; + } + + .btn-parse { + margin-left: 0; + } + + .map-container { + height: 280px; + } + + .chart-container { + height: 220px; + } +} \ No newline at end of file diff --git a/src/app/admin/admin-stats-dashboard.component.ts b/src/app/admin/admin-stats-dashboard.component.ts new file mode 100644 index 0000000..6c42012 --- /dev/null +++ b/src/app/admin/admin-stats-dashboard.component.ts @@ -0,0 +1,303 @@ +import { Component, OnInit, inject, AfterViewInit, OnDestroy, ViewChild, ElementRef, NgZone } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import { FormsModule } from '@angular/forms'; +import { Chart, registerables } from 'chart.js'; +import { BaseChartDirective } from 'ng2-charts'; +import L from 'leaflet'; + +import { StatsService } from '../services/stats.service'; +import { + StatsSummary, + TimeSeriesPoint, + GeoStat, + TimePreset, +} from '../interfaces/visitor-stat'; + +import { getAppConfig } from '../services/app-config.service'; + +// Register Chart.js components once +Chart.register(...registerables); + +@Component({ + selector: 'app-admin-stats-dashboard', + standalone: true, + imports: [CommonModule, FormsModule, BaseChartDirective], + templateUrl: './admin-stats-dashboard.component.html', + styleUrl: './admin-stats-dashboard.component.scss', +}) +export class AdminStatsDashboardComponent implements OnInit, AfterViewInit, OnDestroy { + private readonly statsService: StatsService = inject(StatsService); + private readonly ngZone = inject(NgZone); + + @ViewChild('mapContainer', { static: false }) mapContainerRef!: ElementRef; + + // ── Time range ────────────────────────────────────────── + readonly presets = [ + { label: 'Yesterday', value: 'yesterday' as TimePreset }, + { label: '7 Days', value: '7d' as TimePreset }, + { label: '30 Days', value: '30d' as TimePreset }, + { label: '90 Days', value: '90d' as TimePreset }, + { label: 'Year', value: 'year' as TimePreset }, + { label: 'Custom', value: 'custom' as TimePreset }, + ]; + selectedPreset: TimePreset = '30d'; + customStart = ''; + customEnd = ''; + + // ── Data ──────────────────────────────────────────────── + summary: StatsSummary | null = null; + timeSeries: TimeSeriesPoint[] = []; + geoStats: GeoStat[] = []; + loading = true; + error: string | null = null; + parsing = false; + parseProgress = ''; + + // ── Chart ─────────────────────────────────────────────── + chartData: { labels: string[]; datasets: any[] } = { labels: [], datasets: [] }; + chartOptions: any = { + responsive: true, + maintainAspectRatio: false, + plugins: { + legend: { display: true, position: 'top' as const }, + }, + scales: { + x: { title: { display: true, text: 'Date' } }, + y: { title: { display: true, text: 'Visitors' }, beginAtZero: true }, + }, + }; + + // ── Leaflet ───────────────────────────────────────────── + private map: L.Map | null = null; + private markersLayer: L.LayerGroup | null = null; + + ngOnInit(): void { + this.loadAll(); + } + + ngAfterViewInit(): void { + // Initialize Leaflet map + this.map = L.map(this.mapContainerRef.nativeElement, { + center: [20, 0], + zoom: 2, + zoomControl: true, + worldCopyJump: true, + }); + + L.tileLayer('https://{s}.basemaps.cartocdn.com/light_all/{z}/{x}/{y}{r}.png', { + attribution: '© OpenStreetMap contributors © CARTO', + maxZoom: 19, + }).addTo(this.map); + } + + ngOnDestroy(): void { + if (this.map) { + this.map.remove(); + this.map = null; + } + } + + // ── Date helpers ──────────────────────────────────────── + + private _localDateStr(d: Date): string { + // Format as YYYY-MM-DD using local time (no UTC shift) + const y = d.getFullYear(); + const m = String(d.getMonth() + 1).padStart(2, '0'); + const day = String(d.getDate()).padStart(2, '0'); + return `${y}-${m}-${day}`; + } + + getDates(): { start: string; end: string } { + const today = new Date(); + today.setHours(0, 0, 0, 0); + const yesterday = new Date(today); + yesterday.setDate(today.getDate() - 1); + + switch (this.selectedPreset) { + case 'yesterday': + return { start: this._localDateStr(yesterday), end: this._localDateStr(yesterday) }; + case '7d': { + const start = new Date(yesterday); + start.setDate(yesterday.getDate() - 6); + return { start: this._localDateStr(start), end: this._localDateStr(yesterday) }; + } + case '30d': { + const start = new Date(yesterday); + start.setDate(yesterday.getDate() - 29); + return { start: this._localDateStr(start), end: this._localDateStr(yesterday) }; + } + case '90d': { + const start = new Date(yesterday); + start.setDate(yesterday.getDate() - 89); + return { start: this._localDateStr(start), end: this._localDateStr(yesterday) }; + } + case 'year': { + const start = new Date(yesterday); + start.setDate(yesterday.getDate() - 364); + return { start: this._localDateStr(start), end: this._localDateStr(yesterday) }; + } + case 'custom': + return { + start: this.customStart || this._localDateStr(this._daysAgo(30)), + end: this.customEnd || this._localDateStr(yesterday), + }; + default: + return { start: this._localDateStr(yesterday), end: this._localDateStr(yesterday) }; + } + } + + private _daysAgo(n: number): Date { + const d = new Date(); + d.setHours(0, 0, 0, 0); + d.setDate(d.getDate() - n); + return d; + } + + onPresetChange(): void { + this.loadAll(); + } + + // ── Data loading ──────────────────────────────────────── + + async loadAll(): Promise { + this.loading = true; + this.error = null; + const { start, end } = this.getDates(); + try { + const [summary, timeSeries, geoStats] = await Promise.all([ + this._firstValue(this.statsService.getSummary(start, end)), + this._firstValue(this.statsService.getTimeSeries(start, end)), + this._firstValue(this.statsService.getGeoStats(start, end)), + ]); + + this.summary = summary; + this.timeSeries = timeSeries; + this.geoStats = geoStats; + + this._updateChart(); + this._updateMapMarkers(); + } catch (err) { + console.error('Failed to load stats:', err); + this.error = 'Failed to load stats data. Please check your connection and try again.'; + } finally { + this.loading = false; + } + } + + async triggerParse(): Promise { + this.parsing = true; + this.parseProgress = ''; + + const apiUrl = `${getAppConfig().apiBaseUrl}/api/stats/parse`; + // Append auth token as query param for SSE (EventSource can't send headers) + let token: string | null = null; + try { + token = localStorage.getItem('kmtn_access_token'); + } catch { + // localStorage may be blocked + } + const url = token ? `${apiUrl}?token=${encodeURIComponent(token)}` : apiUrl; + + return new Promise((resolve) => { + const es = new EventSource(url); + + es.addEventListener('progress', (e: MessageEvent) => { + const data = JSON.parse(e.data); + this.ngZone.run(() => { + this.parseProgress = `Parsing ${data.file}... ${data.current}/${data.total}`; + }); + }); + + es.addEventListener('done', (e: MessageEvent) => { + const data = JSON.parse(e.data); + es.close(); + this.ngZone.run(() => { + this.parseProgress = `Done — ${data.files_processed} files, ${data.rows_inserted} geo rows`; + this.parsing = false; + this.loadAll().then(() => resolve()); + }); + }); + + es.onerror = (e: Event) => { + console.error('Parse SSE error:', e); + es.close(); + this.ngZone.run(() => { + this.parseProgress = 'Parse failed'; + this.parsing = false; + resolve(); + }); + }; + }); + } + + private _firstValue(obs: import('rxjs').Observable): Promise { + return new Promise((resolve, reject) => { + obs.subscribe({ next: (v) => resolve(v), error: reject }); + }); + } + + // ── Chart ─────────────────────────────────────────────── + + private _updateChart(): void { + this.chartData = { + labels: this.timeSeries.map((p) => this._formatDate(p.date)), + datasets: [ + { + label: 'Unique Visitors', + data: this.timeSeries.map((p) => p.unique_visitors), + borderColor: '#1a3a5c', + backgroundColor: 'rgba(26, 58, 92, 0.1)', + fill: true, + tension: 0.3, + pointRadius: 2, + }, + ], + }; + } + + private _formatDate(dateStr: string): string { + const d = new Date(dateStr); + return `${d.getMonth() + 1}/${d.getDate()}`; + } + + // ── Map ───────────────────────────────────────────────── + + private _updateMapMarkers(): void { + if (!this.map) return; + + // Remove old markers + if (this.markersLayer) { + this.map.removeLayer(this.markersLayer); + } + + this.markersLayer = L.layerGroup(); + + const geoWithCoords = this.geoStats.filter((g) => g.latitude && g.longitude); + + if (geoWithCoords.length === 0) { + this.markersLayer.addTo(this.map); + return; + } + + const maxVisitors = Math.max(...geoWithCoords.map((g) => g.visitors), 1); + + geoWithCoords.forEach((geo) => { + const radius = Math.max(5, Math.min(30, 5 + (geo.visitors / maxVisitors) * 25)); + const marker = L.circleMarker([geo.latitude!, geo.longitude!], { + radius, + fillColor: '#e87a2e', + color: '#c85a1e', + weight: 1, + opacity: 0.8, + fillOpacity: 0.6, + }); + marker.bindTooltip(`${geo.country}: ${geo.visitors} visitors`, { + sticky: true, + direction: 'top', + }); + this.markersLayer?.addLayer(marker); + }); + + this.markersLayer.addTo(this.map); + } +} \ No newline at end of file diff --git a/src/app/admin/admin.component.html b/src/app/admin/admin.component.html index 35c6f42..ba51624 100644 --- a/src/app/admin/admin.component.html +++ b/src/app/admin/admin.component.html @@ -42,6 +42,11 @@ [class.active]="activeTab() === 'underwriters'" (click)="activeTab.set('underwriters')" >Underwriters + @@ -360,6 +365,13 @@ } } + + + @if (activeTab() === 'stats') { +
+ +
+ } diff --git a/src/app/admin/admin.component.ts b/src/app/admin/admin.component.ts index 330d518..3319372 100644 --- a/src/app/admin/admin.component.ts +++ b/src/app/admin/admin.component.ts @@ -24,8 +24,9 @@ import { AdminHistoryFormComponent } from './admin-history-form.component'; import { AdminTeamFormComponent } from './admin-team-form.component'; import { AdminCommunityFormComponent } from './admin-community-form.component'; import { AdminUnderwriterFormComponent } from './admin-underwriter-form.component'; +import { AdminStatsDashboardComponent } from './admin-stats-dashboard.component'; -type TabKey = 'shows' | 'events' | 'station' | 'history' | 'team' | 'community' | 'underwriters'; +type TabKey = 'shows' | 'events' | 'station' | 'history' | 'team' | 'community' | 'underwriters' | 'stats'; @Component({ selector: 'app-admin', @@ -41,6 +42,7 @@ type TabKey = 'shows' | 'events' | 'station' | 'history' | 'team' | 'community' AdminTeamFormComponent, AdminCommunityFormComponent, AdminUnderwriterFormComponent, + AdminStatsDashboardComponent, ], templateUrl: './admin.component.html', styleUrl: './admin.component.scss', diff --git a/src/app/interfaces/visitor-stat.ts b/src/app/interfaces/visitor-stat.ts new file mode 100644 index 0000000..f590527 --- /dev/null +++ b/src/app/interfaces/visitor-stat.ts @@ -0,0 +1,29 @@ +export interface StatsSummary { + total_unique_visitors: number; +} + +export interface TimeSeriesPoint { + date: string; + unique_visitors: number; +} + +export interface GeoStat { + country: string; + country_code: string; + latitude: number | null; + longitude: number | null; + visitors: number; +} + +export interface ParseProgress { + file: string; + current: number; + total: number; +} + +export interface ParseResult { + files_processed: number; + rows_inserted: number; +} + +export type TimePreset = 'yesterday' | '7d' | '30d' | '90d' | 'year' | 'custom'; diff --git a/src/app/services/stats.service.ts b/src/app/services/stats.service.ts new file mode 100644 index 0000000..33014b9 --- /dev/null +++ b/src/app/services/stats.service.ts @@ -0,0 +1,36 @@ +import { Injectable, inject } from '@angular/core'; +import { HttpClient } from '@angular/common/http'; +import { Observable } from 'rxjs'; + +import { getAppConfig } from './app-config.service'; +import { + StatsSummary, + TimeSeriesPoint, + GeoStat, +} from '../interfaces/visitor-stat'; + +@Injectable({ + providedIn: 'root', +}) +export class StatsService { + private http = inject(HttpClient); + private readonly baseUrl = `${getAppConfig().apiBaseUrl}/api/stats`; + + getSummary(startDate: string, endDate: string): Observable { + return this.http.get(`${this.baseUrl}/summary`, { + params: { start_date: startDate, end_date: endDate }, + }); + } + + getTimeSeries(startDate: string, endDate: string): Observable { + return this.http.get(`${this.baseUrl}/timeseries`, { + params: { start_date: startDate, end_date: endDate }, + }); + } + + getGeoStats(startDate: string, endDate: string): Observable { + return this.http.get(`${this.baseUrl}/geo`, { + params: { start_date: startDate, end_date: endDate }, + }); + } +} diff --git a/src/styles.scss b/src/styles.scss index 6c2e226..fcaa721 100644 --- a/src/styles.scss +++ b/src/styles.scss @@ -6,6 +6,9 @@ @use './styles/mixins' as *; @use './styles/themes'; +// Leaflet CSS for the visitor stats map +@import 'leaflet/dist/leaflet.css'; + // ===== Base reset and global styles ===== *,