#!/usr/bin/env python3
"""Build a standalone NCRC traffic-statistics page from Traefik JSON access logs."""
from __future__ import annotations

import argparse
import html
import ipaddress
import json
from collections import Counter, defaultdict
from datetime import datetime
from pathlib import Path
from typing import Any

DEFAULT_LOG = Path("/mnt/user/appdata/traefik/logs/access.json")
DEFAULT_OUTPUT = Path("/mnt/bigstor/www/usecases/web_stat/statistics.html")
DEFAULT_HOST = "ncrc.nl"
DEFAULT_ROUTER = "usecases-https@file"
DEFAULT_SCANNER_THRESHOLD = 5

SUSPICIOUS_PATH_PARTS = (
    "/wp-admin",
    "/wp-content",
    "/wp-includes",
    "/wp-login",
    "/wp-json",
    "/xmlrpc.php",
    "/cgi-bin",
    "/filemanager",
    "/.env",
    "/.git/",
    "/vendor/phpunit",
    "/phpmyadmin",
)

SUSPICIOUS_FILE_NAMES = {
    "/doc.php",
    "/radio.php",
    "/install.php",
    "/themes.php",
    "/ini.php",
    "/moon.php",
    "/bs1.php",
}


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description="Build NCRC traffic statistics HTML from Traefik access logs."
    )
    parser.add_argument("--log", type=Path, default=DEFAULT_LOG)
    parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
    parser.add_argument("--host", default=DEFAULT_HOST)
    parser.add_argument("--router", default=DEFAULT_ROUTER)
    parser.add_argument(
        "--scanner-threshold",
        type=int,
        default=DEFAULT_SCANNER_THRESHOLD,
        help="Suspicious requests required before an IP is classified as scanner",
    )
    parser.add_argument(
        "--show-full-ip",
        action="store_true",
        help="Show full client IP addresses in the generated HTML",
    )
    args = parser.parse_args()
    if args.scanner_threshold < 1:
        parser.error("--scanner-threshold must be at least 1")
    return args


def esc(value: Any) -> str:
    return html.escape(str(value if value is not None else ""), quote=True)


def read_log(path: Path) -> tuple[list[dict[str, Any]], int]:
    records: list[dict[str, Any]] = []
    malformed = 0
    with path.open("r", encoding="utf-8", errors="replace") as handle:
        for line in handle:
            line = line.strip()
            if not line:
                continue
            try:
                record = json.loads(line)
            except json.JSONDecodeError:
                malformed += 1
                continue
            if isinstance(record, dict):
                records.append(record)
            else:
                malformed += 1
    return records, malformed


def parse_time(record: dict[str, Any]) -> datetime | None:
    raw = record.get("StartLocal") or record.get("time") or record.get("StartUTC")
    if not raw:
        return None
    try:
        return datetime.fromisoformat(str(raw).replace("Z", "+00:00"))
    except ValueError:
        return None


def display_time(value: datetime | None) -> str:
    return value.isoformat(timespec="seconds") if value else "-"


def sort_time(record: dict[str, Any]) -> float:
    value = parse_time(record)
    return value.timestamp() if value else 0.0


def mask_ip(value: str) -> str:
    try:
        address = ipaddress.ip_address(value)
    except ValueError:
        return value
    if address.version == 4:
        parts = value.split(".")
        return ".".join(parts[:3] + ["0/24"])
    return str(ipaddress.ip_network(f"{address}/48", strict=False))


def request_status(record: dict[str, Any]) -> int:
    try:
        return int(record.get("DownstreamStatus") or 0)
    except (TypeError, ValueError):
        return 0


def is_html_page(record: dict[str, Any]) -> bool:
    path = str(record.get("RequestPath") or "")
    return path in {"/", "/site", "/site/"} or path.endswith(".html")


def is_success(record: dict[str, Any]) -> bool:
    return request_status(record) in {200, 304}


def is_pageview(record: dict[str, Any]) -> bool:
    return (
        record.get("RequestMethod") == "GET"
        and is_success(record)
        and is_html_page(record)
    )


def is_scanner_request(record: dict[str, Any]) -> bool:
    path = str(record.get("RequestPath") or "").lower()
    if request_status(record) not in {400, 401, 403, 404, 405}:
        return False
    if path in SUSPICIOUS_FILE_NAMES:
        return True
    if any(part in path for part in SUSPICIOUS_PATH_PARTS):
        return True
    return path.endswith(".php") and not path.startswith("/site/")


def count_table(counter: Counter[Any], label: str, limit: int = 30) -> str:
    body = "".join(
        f"<tr><td>{esc(key)}</td><td>{count}</td></tr>"
        for key, count in counter.most_common(limit)
    )
    if not body:
        body = '<tr><td colspan="2">No data</td></tr>'
    return (
        "<table><thead><tr>"
        f"<th>{esc(label)}</th><th>Count</th>"
        "</tr></thead><tbody>"
        f"{body}"
        "</tbody></table>"
    )


def main() -> int:
    args = parse_args()
    records, malformed = read_log(args.log)
    site = [
        record
        for record in records
        if record.get("RequestHost") == args.host
        and record.get("RouterName") == args.router
    ]
    pageviews = [record for record in site if is_pageview(record)]
    errors = [record for record in site if request_status(record) >= 400]
    scanner_requests = [record for record in site if is_scanner_request(record)]

    requests_by_ip = Counter(
        str(record.get("ClientHost"))
        for record in site
        if record.get("ClientHost")
    )
    pageviews_by_ip = Counter(
        str(record.get("ClientHost"))
        for record in pageviews
        if record.get("ClientHost")
    )
    probes_by_ip = Counter(
        str(record.get("ClientHost"))
        for record in scanner_requests
        if record.get("ClientHost")
    )

    scanner_ips = {
        ip
        for ip, count in probes_by_ip.items()
        if count >= args.scanner_threshold
    }
    pageview_ips = set(pageviews_by_ip)
    visitor_ips = pageview_ips - scanner_ips
    mixed_ips = pageview_ips & scanner_ips
    scanner_only_ips = scanner_ips - pageview_ips

    paths = Counter(str(record.get("RequestPath") or "") for record in pageviews)
    statuses = Counter(request_status(record) for record in site)
    threat_paths = Counter(
        str(record.get("RequestPath") or "")
        for record in pageviews
        if "/threats/T-" in str(record.get("RequestPath") or "")
    )
    probe_paths = Counter(
        str(record.get("RequestPath") or "") for record in scanner_requests
    )

    observed_times = [
        parsed for parsed in (parse_time(record) for record in site) if parsed
    ]
    first_seen = min(observed_times) if observed_times else None
    last_seen = max(observed_times) if observed_times else None

    per_ip: dict[str, dict[str, Any]] = defaultdict(
        lambda: {
            "requests": 0,
            "pageviews": 0,
            "probes": 0,
            "errors": 0,
            "first": None,
            "last": None,
            "classification": "other",
        }
    )

    for record in site:
        ip = str(record.get("ClientHost") or "")
        if not ip:
            continue
        item = per_ip[ip]
        item["requests"] += 1
        if is_pageview(record):
            item["pageviews"] += 1
        if is_scanner_request(record):
            item["probes"] += 1
        if request_status(record) >= 400:
            item["errors"] += 1
        timestamp = parse_time(record)
        if timestamp:
            if item["first"] is None or timestamp < item["first"]:
                item["first"] = timestamp
            if item["last"] is None or timestamp > item["last"]:
                item["last"] = timestamp

    for ip, item in per_ip.items():
        if ip in mixed_ips:
            item["classification"] = "mixed"
        elif ip in scanner_ips:
            item["classification"] = "scanner"
        elif ip in pageview_ips:
            item["classification"] = "visitor"
        else:
            item["classification"] = "other"

    def shown_ip(ip: str) -> str:
        return ip if args.show_full_ip else mask_ip(ip)

    ip_rows = "".join(
        "<tr "
        f'data-class="{esc(item["classification"])}" '
        f'data-hay="{esc(shown_ip(ip).lower())}">'
        f"<td>{esc(shown_ip(ip))}</td>"
        f'<td><span class="class-badge {esc(item["classification"])}">'
        f'{esc(item["classification"])}</span></td>'
        f'<td data-sort="{item["pageviews"]}">{item["pageviews"]}</td>'
        f'<td data-sort="{item["requests"]}">{item["requests"]}</td>'
        f'<td data-sort="{item["probes"]}">{item["probes"]}</td>'
        f'<td data-sort="{item["errors"]}">{item["errors"]}</td>'
        f'<td>{esc(display_time(item["first"]))}</td>'
        f'<td>{esc(display_time(item["last"]))}</td>'
        "</tr>"
        for ip, item in sorted(
            per_ip.items(),
            key=lambda pair: (
                pair[1]["pageviews"],
                pair[1]["probes"],
                pair[1]["requests"],
            ),
            reverse=True,
        )
    )
    if not ip_rows:
        ip_rows = '<tr><td colspan="8">No matching traffic</td></tr>'

    error_rows = "".join(
        "<tr>"
        f"<td>{esc(display_time(parse_time(record)))}</td>"
        f'<td>{esc(shown_ip(str(record.get("ClientHost") or "")))}</td>'
        f"<td>{request_status(record)}</td>"
        f'<td>{esc(record.get("RequestMethod"))}</td>'
        f'<td>{esc(record.get("RequestPath"))}</td>'
        "</tr>"
        for record in sorted(errors, key=sort_time, reverse=True)[:50]
    )
    if not error_rows:
        error_rows = '<tr><td colspan="5">No errors</td></tr>'

    generated = datetime.now().astimezone().isoformat(timespec="seconds")
    privacy = (
        "Full IP addresses shown"
        if args.show_full_ip
        else "IP addresses masked to /24 (IPv4) or /48 (IPv6)"
    )

    css = r'''
:root{--bg:#eef1f5;--surface:#fff;--alt:#f7f9fb;--ink:#1b2430;--muted:#647386;--line:#d5dce5;--accent:#1f4e79;--visitor:#2f7d59;--scanner:#a8403c;--mixed:#8a5a23;--other:#647386}
@media(prefers-color-scheme:dark){:root{--bg:#12171e;--surface:#1a212b;--alt:#151c25;--ink:#e6ebf1;--muted:#93a1b3;--line:#2c3644;--accent:#7db4e6;--visitor:#58c493;--scanner:#ef8d89;--mixed:#e0a74d;--other:#93a1b3}}
*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--ink);font:15px/1.45 "Segoe UI",system-ui,sans-serif}main{max-width:1380px;margin:auto;padding:24px}.card{background:var(--surface);border:1px solid var(--line);border-radius:7px;padding:18px;margin-bottom:16px}.metrics,.grid,.controls{display:grid;gap:12px}.metrics{grid-template-columns:repeat(auto-fit,minmax(150px,1fr))}.grid{grid-template-columns:repeat(auto-fit,minmax(410px,1fr))}.metric{background:var(--alt);border:1px solid var(--line);border-radius:6px;padding:13px}.metric b{display:block;font-size:1.55rem;color:var(--accent)}.metric.visitor b{color:var(--visitor)}.metric.scanner b{color:var(--scanner)}.metric.mixed b{color:var(--mixed)}.muted{color:var(--muted)}.small{font-size:.8rem}.scroll{overflow:auto}table{width:100%;border-collapse:collapse}th,td{text-align:left;padding:8px 9px;border-bottom:1px solid var(--line);vertical-align:top}th{color:var(--muted);font-size:.78rem;white-space:nowrap}.controls{grid-template-columns:1fr auto auto;margin-bottom:10px}input,select{padding:8px;background:var(--bg);color:var(--ink);border:1px solid var(--line);border-radius:5px}.class-badge{display:inline-block;padding:2px 7px;border:1px solid currentColor;border-radius:10px;font-size:.68rem;font-weight:700;text-transform:uppercase}.class-badge.visitor{color:var(--visitor)}.class-badge.scanner{color:var(--scanner)}.class-badge.mixed{color:var(--mixed)}.class-badge.other{color:var(--other)}.notice{padding:10px;background:var(--alt);border-left:3px solid var(--accent);font-size:.8rem}@media(max-width:760px){main{padding:10px}.grid,.controls{grid-template-columns:1fr}}
'''

    page = f'''<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>NCRC Traffic Statistics</title><style>{css}</style></head><body><main>
<section class="card"><h1>NCRC Traffic Statistics</h1><p class="muted">Traefik access-log overview for {esc(args.host)} via {esc(args.router)}. Generated {esc(generated)}.</p>
<div class="metrics">
<div class="metric"><b>{len(site)}</b>Total requests</div>
<div class="metric"><b>{len(pageviews)}</b>HTML pageviews</div>
<div class="metric visitor"><b>{len(visitor_ips)}</b>Likely visitors</div>
<div class="metric scanner"><b>{len(scanner_only_ips)}</b>Scanner IPs</div>
<div class="metric mixed"><b>{len(mixed_ips)}</b>Mixed IPs</div>
<div class="metric scanner"><b>{len(scanner_requests)}</b>Probe requests</div>
<div class="metric"><b>{len(errors)}</b>HTTP errors</div>
<div class="metric"><b>{len(threat_paths)}</b>Threat pages visited</div>
</div>
<p class="small muted">Window: {esc(display_time(first_seen))} to {esc(display_time(last_seen))} · {esc(privacy)} · malformed lines skipped: {malformed}</p>
<div class="notice">Classification is heuristic: visitor = successful HTML pageview; scanner = at least {args.scanner_threshold} suspicious requests; mixed = both; other = neither.</div></section>
<section class="card"><h2>Traffic by IP</h2><div class="controls"><input id="ipq" type="search" placeholder="Filter IP"><select id="classFilter"><option value="">All classes</option><option value="visitor">Visitor</option><option value="scanner">Scanner</option><option value="mixed">Mixed</option><option value="other">Other</option></select><select id="sortBy"><option value="2">Pageviews</option><option value="3">Requests</option><option value="4">Probes</option><option value="5">Errors</option></select></div><div class="scroll"><table id="iptable"><thead><tr><th>IP</th><th>Class</th><th>Pageviews</th><th>Requests</th><th>Probes</th><th>Errors</th><th>First seen</th><th>Last seen</th></tr></thead><tbody>{ip_rows}</tbody></table></div></section>
<section class="grid">
<article class="card"><h2>Most visited pages</h2>{count_table(paths, "Page")}</article>
<article class="card"><h2>Most visited threats</h2>{count_table(threat_paths, "Threat page")}</article>
<article class="card"><h2>HTTP status distribution</h2>{count_table(statuses, "Status")}</article>
<article class="card"><h2>Most scanned paths</h2>{count_table(probe_paths, "Probe path")}</article>
<article class="card"><h2>Requests per IP</h2>{count_table(Counter({shown_ip(ip): count for ip, count in requests_by_ip.items()}), "IP")}</article>
<article class="card"><h2>Probe requests per IP</h2>{count_table(Counter({shown_ip(ip): count for ip, count in probes_by_ip.items()}), "IP")}</article>
</section>
<section class="card"><h2>Latest HTTP errors</h2><div class="scroll"><table><thead><tr><th>Time</th><th>IP</th><th>Status</th><th>Method</th><th>Path</th></tr></thead><tbody>{error_rows}</tbody></table></div></section>
<script>
const tbody=document.querySelector('#iptable tbody');
const q=document.getElementById('ipq');
const cf=document.getElementById('classFilter');
const sb=document.getElementById('sortBy');
function apply(){{
  const term=q.value.trim().toLowerCase();
  const cls=cf.value;
  [...tbody.rows].forEach(row=>{{
    const ok=(!term||row.dataset.hay.includes(term))&&(!cls||row.dataset.class===cls);
    row.style.display=ok?'':'none';
  }});
}}
function sortRows(){{
  const column=Number(sb.value);
  const rows=[...tbody.rows];
  rows.sort((a,b)=>Number(b.cells[column]?.dataset.sort||0)-Number(a.cells[column]?.dataset.sort||0));
  rows.forEach(row=>tbody.appendChild(row));
}}
[q,cf].forEach(control=>control.addEventListener('input',apply));
sb.addEventListener('input',sortRows);
apply();sortRows();
</script>
</main></body></html>'''

    args.output.parent.mkdir(parents=True, exist_ok=True)
    args.output.write_text(page, encoding="utf-8")
    print(
        f"Wrote {args.output}: {len(site)} requests, {len(pageviews)} pageviews, "
        f"{len(visitor_ips)} visitors, {len(scanner_only_ips)} scanners, "
        f"{len(mixed_ips)} mixed"
    )
    return 0


if __name__ == "__main__":
    raise SystemExit(main())
