#!/usr/bin/env python3
"""Build a standalone NCRC traffic statistics page from Traefik JSON access logs."""
from __future__ import annotations

import argparse
import html
import ipaddress
import json
from collections import Counter, defaultdict
from datetime import datetime, timezone
from pathlib import Path
from typing import Any

try:
    import geoip2.database
    from geoip2.errors import AddressNotFoundError
except ImportError:
    geoip2 = None
    AddressNotFoundError = Exception

DEFAULT_LOG = Path('/mnt/user/appdata/traefik/logs/access.json')
DEFAULT_OUTPUT = Path('/mnt/bigstor/www/usecases/web_stat/statistics.html')
DEFAULT_GEOIP_DB = Path('/mnt/bigstor/www/usecases/data/GeoLite2-City.mmdb')


def parse_args() -> argparse.Namespace:
    p = argparse.ArgumentParser(description='Build NCRC traffic statistics HTML from Traefik access logs.')
    p.add_argument('--log', type=Path, default=DEFAULT_LOG)
    p.add_argument('--output', type=Path, default=DEFAULT_OUTPUT)
    p.add_argument('--host', default='ncrc.nl')
    p.add_argument('--router', default='usecases-https@file')
    p.add_argument('--show-full-ip', action='store_true', default=True, help='Show full client IP addresses in HTML')
#    p.add_argument('--show-full-ip', action='store_true', help='Show full client IP addresses in HTML')
    p.add_argument('--geoip-db', type=Path, default=DEFAULT_GEOIP_DB, help='Local GeoLite2-City.mmdb database')
    return p.parse_args()


def esc(v: Any) -> str:
    return html.escape(str(v if v is not None else ''), quote=True)


def read_log(path: Path) -> tuple[list[dict[str, Any]], int]:
    records, bad = [], 0
    with path.open('r', encoding='utf-8', errors='replace') as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            try:
                x = json.loads(line)
                if isinstance(x, dict):
                    records.append(x)
            except json.JSONDecodeError:
                bad += 1
    return records, bad


def dt_of(r: dict[str, Any]) -> datetime | None:
    raw = r.get('StartLocal') or r.get('time') or r.get('StartUTC')
    if not raw:
        return None
    try:
        return datetime.fromisoformat(str(raw).replace('Z', '+00:00'))
    except ValueError:
        return None


def mask_ip(value: str) -> str:
    try:
        ip = ipaddress.ip_address(value)
        if ip.version == 4:
            p = value.split('.')
            return '.'.join(p[:3] + ['0/24'])
        return str(ipaddress.ip_network(f'{ip}/48', strict=False))
    except ValueError:
        return value


def is_html_page(r: dict[str, Any]) -> bool:
    p = str(r.get('RequestPath') or '')
    return p in {'/', '/site', '/site/'} or p.endswith('.html')


def is_success(r: dict[str, Any]) -> bool:
    return int(r.get('DownstreamStatus') or 0) in {200, 304}



KNOWN_CRAWLER_TOKENS = ('googlebot','bingbot','duckduckbot','yandexbot','baiduspider','applebot','petalbot','slurp')
AUTOMATION_TOKENS = ('bot','crawler','spider','curl/','wget/','python-requests','python-urllib','go-http-client','aiohttp','okhttp')
SUSPICIOUS_PATH_TOKENS = ('/.env','/.git','/wp-admin','/wp-login','/wp-content','/wordpress','/phpmyadmin','/administrator','/cgi-bin','/vendor/phpunit','/actuator','/server-status','/boaform','/xmlrpc.php','/autodiscover','/owa/','/ecp/')


def user_agent_of(r: dict[str, Any]) -> str:
    for key in ('request_User-Agent','RequestUserAgent','User-Agent','user_agent','userAgent'):
        if r.get(key):
            return str(r[key])
    headers = r.get('RequestHeaders')
    if isinstance(headers, dict):
        value = headers.get('User-Agent') or headers.get('user-agent')
        if isinstance(value, list):
            return ', '.join(str(x) for x in value)
        return str(value or '')
    return ''


def classify(v: dict[str, Any]) -> tuple[str, int, list[str]]:
    req, pages, errors = v['requests'], v['pages'], v['errors']
    ratio = errors / req if req else 0.0
    ua = ' '.join(v['user_agents']).lower()
    reasons, score = [], 0
    if any(x in ua for x in KNOWN_CRAWLER_TOKENS):
        return 'Known crawler', 0, ['declared crawler User-Agent']
    if v['suspicious']:
        score += min(5, 2 + v['suspicious']); reasons.append(f"{v['suspicious']} suspicious path request(s)")
    if req >= 20 and ratio >= .80:
        score += 3; reasons.append(f'{ratio:.0%} errors')
    elif req >= 10 and ratio >= .60:
        score += 2; reasons.append(f'{ratio:.0%} errors')
    if len(v['paths']) >= 25:
        score += 2; reasons.append(f"{len(v['paths'])} unique paths")
    elif len(v['paths']) >= 10:
        score += 1; reasons.append(f"{len(v['paths'])} unique paths")
    if req >= 100:
        score += 2; reasons.append(f'{req} requests')
    elif req >= 40:
        score += 1; reasons.append(f'{req} requests')
    if pages == 0 and req >= 10:
        score += 1; reasons.append('no successful HTML pageviews')
    if any(x in ua for x in AUTOMATION_TOKENS):
        score += 2; reasons.append('automation-like User-Agent')
    if score >= 5:
        return 'Likely scanner', score, reasons
    if score >= 2:
        return 'Likely automation', score, reasons
    if pages and ratio < .5 and not v['suspicious']:
        return 'Human-like', score, ['successful HTML navigation without strong scanner signals']
    return 'Unclassified', score, reasons or ['insufficient evidence']


def content_category(path: str) -> str:
    p = path.lower()
    rules = (
        ('Threat Register', ('threat-register', '/threats/', 'register.html')),
        ('Regulatory', ('regulatory', 'r155', 'r156', '/cra')),
        ('Incidents', ('incident',)),
        ('Signals', ('signal',)),
        ('CVEs', ('cve', 'zero-day', 'zeroday')),
        ('GitHub', ('github',)),
        ('Papers', ('paper', 'research')),
        ('About', ('about',)),
        ('Home', ('/index.html', '/site/index.html', '/')),
    )
    for label, tokens in rules:
        if any(token == p or token in p for token in tokens):
            return label
    return 'Other content'


def probe_category(path: str) -> str:
    p = path.lower()
    rules = (
        ('WordPress', ('wp-admin','wp-login','wp-content','wordpress','xmlrpc.php')),
        ('Secrets/configuration', ('/.env','/config','server-status','backup')),
        ('Git repositories', ('/.git',)),
        ('PHP/phpMyAdmin', ('phpmyadmin','phpunit','.php')),
        ('Admin/login', ('/admin','/administrator','/login')),
        ('Microsoft/Exchange', ('autodiscover','/owa/','/ecp/')),
        ('Network/IoT', ('boaform','hnap1','cgi-bin')),
    )
    for label, tokens in rules:
        if any(token in p for token in tokens):
            return label
    return 'Other probes'

def geo_lookup(reader: Any, ip_value: str) -> tuple[str, str, str]:
    if reader is None:
        return 'ZZ', 'Unknown', 'Unknown'
    try:
        ip = ipaddress.ip_address(ip_value)
        if not ip.is_global:
            return 'ZZ', 'Unknown', 'Unknown'
        response = reader.city(ip_value)
        code = response.country.iso_code or 'ZZ'
        country = response.country.name or code
        region = response.subdivisions.most_specific.name or 'Unknown'
        return code, country, region
    except (ValueError, AddressNotFoundError):
        return 'ZZ', 'Unknown', 'Unknown'


def back_to_top() -> str:
    return '<p class="back"><a href="#top">Back to top ↑</a></p>'


def main() -> int:
    args = parse_args()
    rows, bad = read_log(args.log)
    site = [r for r in rows if r.get('RequestHost') == args.host and r.get('RouterName') == args.router]
    get_rows = [r for r in site if r.get('RequestMethod') == 'GET']
    pages = [r for r in get_rows if is_success(r) and is_html_page(r)]
    ips = Counter(str(r.get('ClientHost') or '') for r in site if r.get('ClientHost'))
    page_ips = Counter(str(r.get('ClientHost') or '') for r in pages if r.get('ClientHost'))
    paths = Counter(str(r.get('RequestPath') or '') for r in pages)
    statuses = Counter(int(r.get('DownstreamStatus') or 0) for r in site)
    errors = [r for r in site if int(r.get('DownstreamStatus') or 0) >= 400]
    threat_paths = Counter(str(r.get('RequestPath')) for r in pages if '/threats/T-' in str(r.get('RequestPath') or ''))
    dts = [d for d in (dt_of(r) for r in site) if d]
    first = min(dts).isoformat(timespec='seconds') if dts else '-'
    last = max(dts).isoformat(timespec='seconds') if dts else '-'

    geo_reader = None
    geo_status = 'disabled: local GeoIP database not found'
    if args.geoip_db.is_file() and geoip2 is not None:
        geo_reader = geoip2.database.Reader(str(args.geoip_db))
        geo_status = f'local database: {args.geoip_db.name}'
    elif args.geoip_db.is_file() and geoip2 is None:
        geo_status = 'disabled: Python package geoip2 is not installed'

    country_unique: Counter[str] = Counter()
    country_pageviews: Counter[str] = Counter()
    nl_region_unique: Counter[str] = Counter()
    nl_region_pageviews: Counter[str] = Counter()
    pageviews_by_ip = Counter(str(r.get('ClientHost') or '') for r in pages if r.get('ClientHost'))
    for ip, pageview_count in pageviews_by_ip.items():
        code, country, region = geo_lookup(geo_reader, ip)
        country_label = f'{country} ({code})' if code != 'ZZ' else 'Unknown'
        country_unique[country_label] += 1
        country_pageviews[country_label] += pageview_count
        if code == 'NL':
            nl_region_unique[region] += 1
            nl_region_pageviews[region] += pageview_count
    if geo_reader is not None:
        geo_reader.close()

    nl_unique = sum(v for label, v in country_unique.items() if label.endswith('(NL)'))
    unknown_unique = country_unique.get('Unknown', 0)
    outside_nl_unique = sum(country_unique.values()) - nl_unique - unknown_unique

    per_ip: dict[str, dict[str, Any]] = defaultdict(lambda: {'requests': 0, 'pages': 0, 'first': None, 'last': None, 'errors': 0, 'paths': set(), 'user_agents': set(), 'suspicious': 0})
    page_record_ids = {id(r) for r in pages}
    suspicious_paths: Counter[str] = Counter()
    for r in site:
        ip = str(r.get('ClientHost') or '')
        if not ip:
            continue
        x = per_ip[ip]
        x['requests'] += 1
        if id(r) in page_record_ids:
            x['pages'] += 1
        if int(r.get('DownstreamStatus') or 0) >= 400:
            x['errors'] += 1
        path = str(r.get('RequestPath') or '')
        x['paths'].add(path)
        ua = user_agent_of(r)
        if ua:
            x['user_agents'].add(ua)
        if any(token.lower() in path.lower() for token in SUSPICIOUS_PATH_TOKENS):
            x['suspicious'] += 1
            suspicious_paths[path] += 1
        d = dt_of(r)
        if d:
            x['first'] = d if x['first'] is None or d < x['first'] else x['first']
            x['last'] = d if x['last'] is None or d > x['last'] else x['last']

    class_ips, class_requests = Counter(), Counter()
    for v in per_ip.values():
        v['classification'], v['score'], v['reasons'] = classify(v)
        class_ips[v['classification']] += 1
        class_requests[v['classification']] += v['requests']


    # A. Traffic over time, grouped by hour and final classification.
    traffic_by_hour: dict[str, Counter[str]] = defaultdict(Counter)
    for r in site:
        ip = str(r.get('ClientHost') or '')
        d = dt_of(r)
        if not ip or not d or ip not in per_ip:
            continue
        traffic_by_hour[d.isoformat(timespec='hours')][per_ip[ip]['classification']] += 1

    # B. Engaged candidates: human-like, at least two successful HTML pageviews
    # across at least two distinct HTML paths.
    engaged_ips: set[str] = set()
    successful_paths_by_ip: dict[str, set[str]] = defaultdict(set)
    for r in pages:
        ip = str(r.get('ClientHost') or '')
        if ip:
            successful_paths_by_ip[ip].add(str(r.get('RequestPath') or ''))
    for ip, v in per_ip.items():
        if v['classification'] == 'Human-like' and v['pages'] >= 2 and len(successful_paths_by_ip[ip]) >= 2:
            engaged_ips.add(ip)

    # C. Content interest, separated into all and human-like/engaged views.
    content_all: Counter[str] = Counter()
    content_human_like: Counter[str] = Counter()
    content_engaged: Counter[str] = Counter()
    for r in pages:
        ip = str(r.get('ClientHost') or '')
        category = content_category(str(r.get('RequestPath') or ''))
        content_all[category] += 1
        if ip in per_ip and per_ip[ip]['classification'] == 'Human-like':
            content_human_like[category] += 1
        if ip in engaged_ips:
            content_engaged[category] += 1

    # D. Group suspicious paths into understandable probe categories.
    probe_categories: Counter[str] = Counter()
    for path, count in suspicious_paths.items():
        probe_categories[probe_category(path)] += count

    def shown_ip(ip: str) -> str:
        return ip if args.show_full_ip else mask_ip(ip)

    ip_table = ''.join(
        f'<tr><td>{esc(shown_ip(ip))}</td><td>{esc(v["classification"])}</td><td>{v["score"]}</td><td>{v["pages"]}</td><td>{v["requests"]}</td><td>{v["errors"]}</td><td>{len(v["paths"])}</td>'
        f'<td>{esc(v["first"].isoformat(timespec="seconds") if v["first"] else "-")}</td><td>{esc(v["last"].isoformat(timespec="seconds") if v["last"] else "-")}</td></tr>'
        for ip, v in sorted(per_ip.items(), key=lambda kv: (kv[1]['score'], kv[1]['pages'], kv[1]['requests']), reverse=True)
    ) or '<tr><td colspan="9">No matching traffic</td></tr>'

    class_order = ('Likely scanner','Likely automation','Known crawler','Human-like','Unclassified')
    classification_table = ''.join(f'<tr><td>{esc(k)}</td><td>{class_ips[k]}</td><td>{class_requests[k]}</td></tr>' for k in class_order)
    scanner_table = ''.join(
        f'<tr><td>{esc(shown_ip(ip))}</td><td>{v["score"]}</td><td>{v["requests"]}</td><td>{v["errors"]}</td><td>{(v["errors"]/v["requests"]):.1%}</td><td>{len(v["paths"])}</td><td>{esc("; ".join(v["reasons"]))}</td></tr>'
        for ip, v in sorted(per_ip.items(), key=lambda kv: (kv[1]['score'], kv[1]['requests']), reverse=True) if v['classification'] == 'Likely scanner'
    ) or '<tr><td colspan="7">No likely scanners detected by the current heuristic</td></tr>'

    def count_table(counter: Counter, label: str, limit: int = 30) -> str:
        body = ''.join(f'<tr><td>{esc(k)}</td><td>{v}</td></tr>' for k, v in counter.most_common(limit))
        if not body:
            body = '<tr><td colspan="2">No data</td></tr>'
        return f'<table><thead><tr><th>{esc(label)}</th><th>Count</th></tr></thead><tbody>{body}</tbody></table>'

    def geo_table(unique_counter: Counter[str], pageview_counter: Counter[str], label: str) -> str:
        keys = sorted(unique_counter, key=lambda k: (unique_counter[k], pageview_counter[k], k), reverse=True)
        body = ''.join(
            f'<tr><td>{esc(k)}</td><td>{unique_counter[k]}</td><td>{pageview_counter[k]}</td></tr>' for k in keys
        ) or '<tr><td colspan="3">No geolocation data</td></tr>'
        return f'<table><thead><tr><th>{esc(label)}</th><th>Unique pageview IPs</th><th>HTML pageviews</th></tr></thead><tbody>{body}</tbody></table>'

    error_table = ''.join(
        f'<tr><td>{esc((dt_of(r).isoformat(timespec="seconds") if dt_of(r) else "-"))}</td>'
        f'<td>{esc(shown_ip(str(r.get("ClientHost") or "")))}</td><td>{esc(r.get("DownstreamStatus"))}</td>'
        f'<td>{esc(r.get("RequestMethod"))}</td><td>{esc(r.get("RequestPath"))}</td></tr>'
        for r in sorted(errors, key=lambda x: dt_of(x) or datetime.min.replace(tzinfo=timezone.utc), reverse=True)[:50]
    ) or '<tr><td colspan="5">No errors</td></tr>'

    timeline_rows = []
    for hour in sorted(traffic_by_hour):
        c = traffic_by_hour[hour]
        total = sum(c.values())
        timeline_rows.append(
            f'<tr><td>{esc(hour)}</td><td>{total}</td><td>{c["Human-like"]}</td>'
            f'<td>{c["Likely automation"]}</td><td>{c["Likely scanner"]}</td>'
            f'<td>{c["Known crawler"]}</td><td>{c["Unclassified"]}</td></tr>'
        )
    timeline_table = ''.join(timeline_rows) or '<tr><td colspan="7">No timestamped traffic</td></tr>'

    content_keys = sorted(set(content_all) | set(content_human_like) | set(content_engaged), key=lambda k: content_all[k], reverse=True)
    content_table = ''.join(
        f'<tr><td>{esc(k)}</td><td>{content_all[k]}</td><td>{content_human_like[k]}</td><td>{content_engaged[k]}</td></tr>'
        for k in content_keys
    ) or '<tr><td colspan="4">No content data</td></tr>'

    generated = datetime.now().astimezone().isoformat(timespec='seconds')
    privacy = 'Full IP addresses shown' if args.show_full_ip else 'IP addresses masked to /24 (IPv4) or /48 (IPv6)'
    css = '''
:root{--bg:#eef1f5;--surface:#fff;--alt:#f7f9fb;--ink:#1b2430;--muted:#647386;--line:#d5dce5;--accent:#1f4e79}@media(prefers-color-scheme:dark){:root{--bg:#12171e;--surface:#1a212b;--alt:#151c25;--ink:#e6ebf1;--muted:#93a1b3;--line:#2c3644;--accent:#7db4e6}}*{box-sizing:border-box}html{scroll-behavior:smooth}body{margin:0;background:var(--bg);color:var(--ink);font:15px/1.45 "Segoe UI",system-ui,sans-serif}main{max-width:1300px;margin:auto;padding:24px}.card{background:var(--surface);border:1px solid var(--line);border-radius:7px;padding:18px;margin-bottom:16px}.metrics,.grid,.controls,.geo-summary{display:grid;gap:12px}.metrics{grid-template-columns:repeat(auto-fit,minmax(155px,1fr))}.grid{grid-template-columns:repeat(auto-fit,minmax(420px,1fr))}.metric,.geo-summary div{background:var(--alt);border:1px solid var(--line);border-radius:6px;padding:13px}.metric b,.geo-summary b{display:block;font-size:1.55rem;color:var(--accent)}.muted{color:var(--muted)}table{width:100%;border-collapse:collapse}th,td{text-align:left;padding:8px 9px;border-bottom:1px solid var(--line);vertical-align:top}th{color:var(--muted);font-size:.78rem}input,select{padding:8px;background:var(--bg);color:var(--ink);border:1px solid var(--line);border-radius:5px}.controls{grid-template-columns:1fr auto;margin-bottom:10px}.scroll{overflow:auto}.small{font-size:.8rem}.jump{display:flex;flex-wrap:wrap;gap:7px;margin-top:16px}.jump a,.back a{display:inline-flex;padding:6px 9px;border:1px solid var(--line);border-radius:5px;background:var(--alt);color:var(--accent);text-decoration:none;font-size:.78rem;font-weight:600}.back{margin:14px 0 0;text-align:right}.geo-summary{grid-template-columns:repeat(3,minmax(0,1fr));margin:12px 0}@media(max-width:700px){main{padding:10px}.grid,.controls,.geo-summary{grid-template-columns:1fr}}
'''

    page = f'''<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>NCRC Traffic Statistics</title><style>{css}</style></head><body><main id="top">
<section class="card"><h1>NCRC Traffic Statistics</h1><p class="muted">Traefik access log overview for {esc(args.host)} via {esc(args.router)}. Generated {esc(generated)}.</p><div class="metrics">
<div class="metric"><b>{len(site)}</b>Total requests</div><div class="metric"><b>{len(engaged_ips)}</b>Engaged candidates</div><div class="metric"><b>{len(pages)}</b>HTML pageviews</div><div class="metric"><b>{len(ips)}</b>Unique request IPs</div><div class="metric"><b>{len(page_ips)}</b>Unique pageview IPs</div><div class="metric"><b>{len(errors)}</b>HTTP errors</div><div class="metric"><b>{len(threat_paths)}</b>Threat pages visited</div></div>
<p class="small muted">Window: {esc(first)} to {esc(last)} · {esc(privacy)} · malformed log lines skipped: {bad}</p>
<nav class="jump" aria-label="Page sections"><a href="#classification">Traffic classification</a><a href="#timeline">Traffic over time</a><a href="#engagement">Engagement</a><a href="#content-interest">Content interest</a><a href="#probe-categories">Probe categories</a><a href="#visitors">Visitors by IP</a><a href="#scanners">Suspected scanners</a><a href="#probes">Suspicious paths</a><a href="#pages">Most visited pages</a><a href="#threats">Most visited threats</a><a href="#statuses">HTTP statuses</a><a href="#requests-per-ip">Requests per IP</a><a href="#geography">Traffic geography</a><a href="#nl-regions">Netherlands by region</a><a href="#errors">Latest errors</a></nav></section>
<section class="card" id="classification"><h2>Traffic classification</h2><p class="muted">Heuristic classification, not proof of identity or intent. Signals combine suspicious paths, error ratio, volume, path diversity and available User-Agent data.</p><div class="scroll"><table><thead><tr><th>Classification</th><th>Unique IPs</th><th>Requests</th></tr></thead><tbody>{classification_table}</tbody></table></div>{back_to_top()}</section>
<section class="card" id="timeline"><h2>Traffic over time</h2><p class="muted">Requests per local-log hour, split by the final per-IP classification.</p><div class="scroll"><table><thead><tr><th>Hour</th><th>Total</th><th>Human-like</th><th>Automation</th><th>Scanners</th><th>Crawlers</th><th>Unclassified</th></tr></thead><tbody>{timeline_table}</tbody></table></div>{back_to_top()}</section>
<section class="card" id="engagement"><h2>Engaged visitor candidates</h2><p class="muted">Human-like IPs with at least two successful HTML pageviews across at least two different paths. This is a behavioral indicator, not proof of a unique person.</p><div class="metrics"><div class="metric"><b>{class_ips["Human-like"]}</b>Human-like IPs</div><div class="metric"><b>{len(engaged_ips)}</b>Engaged candidates</div><div class="metric"><b>{sum(per_ip[ip]["pages"] for ip in engaged_ips)}</b>Engaged pageviews</div></div>{back_to_top()}</section>
<section class="card" id="content-interest"><h2>NCRC content interest</h2><p class="muted">Successful HTML pageviews grouped by NCRC section. Human-like and engaged columns reduce scanner noise.</p><div class="scroll"><table><thead><tr><th>Content area</th><th>All pageviews</th><th>Human-like</th><th>Engaged candidates</th></tr></thead><tbody>{content_table}</tbody></table></div>{back_to_top()}</section>
<section class="card" id="probe-categories"><h2>Probe categories</h2><p class="muted">Suspicious paths grouped into recognizable families.</p><div class="scroll">{count_table(probe_categories, 'Probe category', 30)}</div>{back_to_top()}</section>
<section class="card" id="scanners"><h2>Suspected scanners</h2><div class="scroll"><table><thead><tr><th>IP</th><th>Score</th><th>Requests</th><th>Errors</th><th>Error ratio</th><th>Unique paths</th><th>Signals</th></tr></thead><tbody>{scanner_table}</tbody></table></div>{back_to_top()}</section>
<section class="card" id="probes"><h2>Suspicious or probed paths</h2><p class="muted">Paths matching common CMS, configuration, repository, administration or vulnerability-probe patterns.</p><div class="scroll">{count_table(suspicious_paths, 'Path', 75)}</div>{back_to_top()}</section>
<section class="card" id="visitors"><h2>Visitors by IP</h2><div class="controls"><input id="ipq" type="search" placeholder="Filter IP"><select id="ipsort"><option value="pages">Pageviews</option><option value="requests">Requests</option></select></div><div class="scroll"><table id="iptable"><thead><tr><th>IP</th><th>Classification</th><th>Score</th><th>Pageviews</th><th>Requests</th><th>Errors</th><th>Unique paths</th><th>First seen</th><th>Last seen</th></tr></thead><tbody>{ip_table}</tbody></table></div>{back_to_top()}</section>
<section class="grid"><article class="card" id="pages"><h2>Most visited pages</h2>{count_table(paths, 'Page')}{back_to_top()}</article><article class="card" id="threats"><h2>Most visited threats</h2>{count_table(threat_paths, 'Threat page')}{back_to_top()}</article><article class="card" id="statuses"><h2>HTTP status distribution</h2>{count_table(statuses, 'Status')}{back_to_top()}</article><article class="card" id="requests-per-ip"><h2>Requests per IP</h2>{count_table(Counter(shown_ip(ip) for ip in ips.elements()), 'IP')}{back_to_top()}</article></section>
<section class="card" id="geography"><h2>Traffic geography</h2><p class="muted">Aggregated from unique IP addresses that requested at least one successful HTML page. Geolocation is approximate. Raw IP-to-location mappings are not written to this report.</p><div class="geo-summary"><div><b>{nl_unique}</b>Netherlands</div><div><b>{outside_nl_unique}</b>Outside Netherlands</div><div><b>{unknown_unique}</b>Unknown</div></div><p class="small muted">GeoIP status: {esc(geo_status)}</p><div class="scroll">{geo_table(country_unique, country_pageviews, 'Country')}</div>{back_to_top()}</section>
<section class="card" id="nl-regions"><h2>Netherlands by region</h2><p class="muted">Subdivision-level estimates for Dutch unique pageview IPs. These are network-location estimates, not precise visitor locations.</p><div class="scroll">{geo_table(nl_region_unique, nl_region_pageviews, 'Region')}</div>{back_to_top()}</section>
<section class="card" id="errors"><h2>Latest HTTP errors</h2><div class="scroll"><table><thead><tr><th>Time</th><th>IP</th><th>Status</th><th>Method</th><th>Path</th></tr></thead><tbody>{error_table}</tbody></table></div>{back_to_top()}</section>
<script>const q=document.getElementById('ipq'),rows=[...document.querySelectorAll('#iptable tbody tr')];q.addEventListener('input',()=>{{const v=q.value.toLowerCase();rows.forEach(r=>r.style.display=r.cells[0].textContent.toLowerCase().includes(v)?'':'none')}});</script>
</main></body></html>'''
    args.output.parent.mkdir(parents=True, exist_ok=True)
    args.output.write_text(page, encoding='utf-8')
    print(f'Wrote {args.output}: {len(site)} requests, {len(pages)} pageviews, {len(ips)} unique IPs')
    print(f'Classification: {dict(class_ips)}')
    print(f'Engaged candidates: {len(engaged_ips)}')
    print(f'GeoIP: {geo_status}')
    return 0


if __name__ == '__main__':
    raise SystemExit(main())
