#!/usr/bin/env python3
"""Build a standalone NCRC traffic statistics page from Traefik JSON access logs."""
from __future__ import annotations

import argparse
import html
import ipaddress
import json
import re
from collections import Counter, defaultdict
from datetime import datetime, timezone
from pathlib import Path
from statistics import median
from typing import Any

try:
    import geoip2.database
    from geoip2.errors import AddressNotFoundError
except ImportError:
    geoip2 = None
    AddressNotFoundError = Exception

DEFAULT_LOG = Path('/mnt/user/appdata/traefik/logs/access.json')
DEFAULT_OUTPUT = Path('/mnt/bigstor/www/usecases/web_stat/statistics.html')
DEFAULT_GEOIP_DB = Path('/mnt/bigstor/www/usecases/data/GeoLite2-City.mmdb')


def parse_args() -> argparse.Namespace:
    p = argparse.ArgumentParser(description='Build NCRC traffic statistics HTML from Traefik access logs.')
    p.add_argument('--log', type=Path, default=DEFAULT_LOG)
    p.add_argument('--output', type=Path, default=DEFAULT_OUTPUT)
    p.add_argument('--host', default='ncrc.nl')
    p.add_argument('--router', default='usecases-https@file')
    p.add_argument('--show-full-ip', action='store_true', default=True, help='Show full client IP addresses in HTML')
#    p.add_argument('--show-full-ip', action='store_true', help='Show full client IP addresses in HTML')
    p.add_argument('--geoip-db', type=Path, default=DEFAULT_GEOIP_DB, help='Local GeoLite2-City.mmdb database')
    return p.parse_args()


def esc(v: Any) -> str:
    return html.escape(str(v if v is not None else ''), quote=True)


def read_log(path: Path) -> tuple[list[dict[str, Any]], int]:
    records, bad = [], 0
    with path.open('r', encoding='utf-8', errors='replace') as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            try:
                x = json.loads(line)
                if isinstance(x, dict):
                    records.append(x)
            except json.JSONDecodeError:
                bad += 1
    return records, bad


def dt_of(r: dict[str, Any]) -> datetime | None:
    raw = r.get('StartLocal') or r.get('time') or r.get('StartUTC')
    if not raw:
        return None
    try:
        return datetime.fromisoformat(str(raw).replace('Z', '+00:00'))
    except ValueError:
        return None


def mask_ip(value: str) -> str:
    try:
        ip = ipaddress.ip_address(value)
        if ip.version == 4:
            p = value.split('.')
            return '.'.join(p[:3] + ['0/24'])
        return str(ipaddress.ip_network(f'{ip}/48', strict=False))
    except ValueError:
        return value


def is_html_page(r: dict[str, Any]) -> bool:
    p = str(r.get('RequestPath') or '')
    return p in {'/', '/site', '/site/'} or p.endswith('.html')


def is_success(r: dict[str, Any]) -> bool:
    return int(r.get('DownstreamStatus') or 0) in {200, 304}



KNOWN_CRAWLER_TOKENS = ('googlebot','bingbot','duckduckbot','yandexbot','baiduspider','applebot','petalbot','slurp')
KNOWN_CRAWLER_NETWORKS = ((ipaddress.ip_network('85.208.96.0/24'), 'SEMrush / AS209366'),)
AUTOMATION_TOKENS = ('bot','crawler','spider','curl/','wget/','python-requests','python-urllib','go-http-client','aiohttp','okhttp')
SUSPICIOUS_PATH_TOKENS = ('/.env','/.git','/wp-admin','/wp-login','/wp-content','/wordpress','/phpmyadmin','/administrator','/cgi-bin','/vendor/phpunit','/actuator','/server-status','/boaform','/xmlrpc.php','/autodiscover','/owa/','/ecp/')


def known_crawler_network(ip_value: str) -> str | None:
    try:
        ip = ipaddress.ip_address(ip_value)
    except ValueError:
        return None
    for network, label in KNOWN_CRAWLER_NETWORKS:
        if ip in network:
            return label
    return None

def sequential_threat_run(paths: list[str]) -> int:
    numbers = []
    for path in paths:
        match = re.search(r'/threats/T-(\d+)\.html$', path, re.IGNORECASE)
        if match:
            numbers.append(int(match.group(1)))
    best = current = 1 if numbers else 0
    for previous, current_number in zip(numbers, numbers[1:]):
        current = current + 1 if current_number == previous + 1 else 1
        best = max(best, current)
    return best

def user_agent_of(r: dict[str, Any]) -> str:
    for key in ('request_User-Agent','RequestUserAgent','User-Agent','user_agent','userAgent'):
        if r.get(key):
            return str(r[key])
    headers = r.get('RequestHeaders')
    if isinstance(headers, dict):
        value = headers.get('User-Agent') or headers.get('user-agent')
        if isinstance(value, list):
            return ', '.join(str(x) for x in value)
        return str(value or '')
    return ''


def classify(v: dict[str, Any]) -> tuple[str, int, list[str]]:
    req, pages, errors = v['requests'], v['pages'], v['errors']
    ratio = errors / req if req else 0.0
    ua = ' '.join(v['user_agents']).lower()
    reasons: list[str] = []
    score = 0

    if v.get('crawler_network'):
        return 'Known crawler', 0, [f"known crawler network: {v['crawler_network']}"]
    if any(x in ua for x in KNOWN_CRAWLER_TOKENS):
        return 'Known crawler', 0, ['declared crawler User-Agent']

    page_events = sorted(v.get('page_events', []), key=lambda item: item[0])
    page_times = [item[0] for item in page_events]
    page_paths = [item[1] for item in page_events]
    gaps = [(b - a).total_seconds() for a, b in zip(page_times, page_times[1:])]
    page_span = (page_times[-1] - page_times[0]).total_seconds() if len(page_times) >= 2 else 0.0
    median_gap = median(gaps) if gaps else None
    sequential_run = sequential_threat_run(page_paths)
    discovery_requested = v.get('discovery_requests', 0) > 0

    crawler_signals = []
    if pages >= 8 and page_span <= 10:
        crawler_signals.append(f'{pages} HTML pages in {page_span:.1f}s')
    if pages >= 6 and median_gap is not None and median_gap < 2:
        crawler_signals.append(f'{median_gap:.1f}s median HTML interval')
    if sequential_run >= 5:
        crawler_signals.append(f'{sequential_run} sequential threat pages')
    if discovery_requested and pages >= 6 and page_span <= 30:
        crawler_signals.append('robots/sitemap followed by broad enumeration')
    if crawler_signals:
        return 'Content crawler', 4 + len(crawler_signals), crawler_signals

    if v['suspicious']:
        score += min(5, 2 + v['suspicious'])
        reasons.append(f"{v['suspicious']} suspicious path request(s)")
    if req >= 20 and ratio >= .80:
        score += 3; reasons.append(f'{ratio:.0%} errors')
    elif req >= 10 and ratio >= .60:
        score += 2; reasons.append(f'{ratio:.0%} errors')
    if len(v['paths']) >= 25:
        score += 2; reasons.append(f"{len(v['paths'])} unique paths")
    elif len(v['paths']) >= 10:
        score += 1; reasons.append(f"{len(v['paths'])} unique paths")
    if req >= 100:
        score += 2; reasons.append(f'{req} requests')
    elif req >= 40:
        score += 1; reasons.append(f'{req} requests')
    if pages == 0 and req >= 10:
        score += 1; reasons.append('no successful HTML pageviews')
    if any(x in ua for x in AUTOMATION_TOKENS):
        score += 2; reasons.append('automation-like User-Agent')
    if score >= 5:
        return 'Likely scanner', score, reasons
    if score >= 2:
        return 'Other automation', score, reasons

    distinct_pages = len(set(page_paths))
    if pages >= 3 and distinct_pages >= 3 and ratio < .25 and not v['suspicious']:
        natural_timing = page_span >= 30 or (median_gap is not None and median_gap >= 8)
        if natural_timing:
            details = [f'{pages} successful HTML pageviews', f'{distinct_pages} distinct pages']
            if page_span:
                details.append(f'{page_span:.0f}s observed browsing span')
            if median_gap is not None:
                details.append(f'{median_gap:.1f}s median HTML interval')
            return 'Likely interactive', 1, details
    if pages and ratio < .5 and not v['suspicious']:
        return 'Human-like', score, ['successful HTML navigation without strong automation signals']
    return 'Unclassified', score, reasons or ['insufficient evidence']

def content_category(path: str) -> str:
    p = path.lower()
    rules = (
        ('Threat Register', ('threat-register', '/threats/', 'register.html')),
        ('Regulatory', ('regulatory', 'r155', 'r156', '/cra')),
        ('Incidents', ('incident',)),
        ('Narratives', ('narrative', '/narratives/')),
        ('Signals', ('signal',)),
        ('CVEs', ('cve', 'zero-day', 'zeroday')),
        ('GitHub', ('github',)),
        ('Papers', ('paper', 'research')),
        ('About', ('about',)),
        ('Home', ('/index.html', '/site/index.html', '/')),
    )
    for label, tokens in rules:
        if any(token == p or token in p for token in tokens):
            return label
    return 'Other content'


def probe_category(path: str) -> str:
    p = path.lower()
    rules = (
        ('WordPress', ('wp-admin','wp-login','wp-content','wordpress','xmlrpc.php')),
        ('Secrets/configuration', ('/.env','/config','server-status','backup')),
        ('Git repositories', ('/.git',)),
        ('PHP/phpMyAdmin', ('phpmyadmin','phpunit','.php')),
        ('Admin/login', ('/admin','/administrator','/login')),
        ('Microsoft/Exchange', ('autodiscover','/owa/','/ecp/')),
        ('Network/IoT', ('boaform','hnap1','cgi-bin')),
    )
    for label, tokens in rules:
        if any(token in p for token in tokens):
            return label
    return 'Other probes'

def geo_lookup(reader: Any, ip_value: str) -> tuple[str, str, str]:
    if reader is None:
        return 'ZZ', 'Unknown', 'Unknown'
    try:
        ip = ipaddress.ip_address(ip_value)
        if not ip.is_global:
            return 'ZZ', 'Unknown', 'Unknown'
        response = reader.city(ip_value)
        code = response.country.iso_code or 'ZZ'
        country = response.country.name or code
        region = response.subdivisions.most_specific.name or 'Unknown'
        return code, country, region
    except (ValueError, AddressNotFoundError):
        return 'ZZ', 'Unknown', 'Unknown'


def back_to_top() -> str:
    return '<p class="back"><a href="#top">Back to top ↑</a></p>'


def main() -> int:
    args = parse_args()
    rows, bad = read_log(args.log)
    site = [r for r in rows if r.get('RequestHost') == args.host and r.get('RouterName') == args.router]
    get_rows = [r for r in site if r.get('RequestMethod') == 'GET']
    pages = [r for r in get_rows if is_success(r) and is_html_page(r)]
    ips = Counter(str(r.get('ClientHost') or '') for r in site if r.get('ClientHost'))
    page_ips = Counter(str(r.get('ClientHost') or '') for r in pages if r.get('ClientHost'))
    paths = Counter(str(r.get('RequestPath') or '') for r in pages)
    statuses = Counter(int(r.get('DownstreamStatus') or 0) for r in site)
    errors = [r for r in site if int(r.get('DownstreamStatus') or 0) >= 400]
    threat_paths = Counter(str(r.get('RequestPath')) for r in pages if '/threats/T-' in str(r.get('RequestPath') or ''))
    dts = [d for d in (dt_of(r) for r in site) if d]
    first = min(dts).isoformat(timespec='seconds') if dts else '-'
    last = max(dts).isoformat(timespec='seconds') if dts else '-'

    geo_reader = None
    geo_status = 'disabled: local GeoIP database not found'
    if args.geoip_db.is_file() and geoip2 is not None:
        geo_reader = geoip2.database.Reader(str(args.geoip_db))
        geo_status = f'local database: {args.geoip_db.name}'
    elif args.geoip_db.is_file() and geoip2 is None:
        geo_status = 'disabled: Python package geoip2 is not installed'

    country_unique: Counter[str] = Counter()
    country_pageviews: Counter[str] = Counter()
    nl_region_unique: Counter[str] = Counter()
    nl_region_pageviews: Counter[str] = Counter()
    pageviews_by_ip = Counter(str(r.get('ClientHost') or '') for r in pages if r.get('ClientHost'))
    for ip, pageview_count in pageviews_by_ip.items():
        code, country, region = geo_lookup(geo_reader, ip)
        country_label = f'{country} ({code})' if code != 'ZZ' else 'Unknown'
        country_unique[country_label] += 1
        country_pageviews[country_label] += pageview_count
        if code == 'NL':
            nl_region_unique[region] += 1
            nl_region_pageviews[region] += pageview_count
    if geo_reader is not None:
        geo_reader.close()

    nl_unique = sum(v for label, v in country_unique.items() if label.endswith('(NL)'))
    unknown_unique = country_unique.get('Unknown', 0)
    outside_nl_unique = sum(country_unique.values()) - nl_unique - unknown_unique

    per_ip: dict[str, dict[str, Any]] = defaultdict(lambda: {'requests': 0, 'pages': 0, 'first': None, 'last': None, 'errors': 0, 'paths': set(), 'user_agents': set(), 'suspicious': 0, 'page_events': [], 'discovery_requests': 0, 'crawler_network': None})
    page_record_ids = {id(r) for r in pages}
    suspicious_paths: Counter[str] = Counter()
    for r in site:
        ip = str(r.get('ClientHost') or '')
        if not ip:
            continue
        x = per_ip[ip]
        x['requests'] += 1
        if x['crawler_network'] is None:
            x['crawler_network'] = known_crawler_network(ip)
        if id(r) in page_record_ids:
            x['pages'] += 1
        if int(r.get('DownstreamStatus') or 0) >= 400:
            x['errors'] += 1
        path = str(r.get('RequestPath') or '')
        x['paths'].add(path)
        if path.lower() in {'/robots.txt', '/sitemap.xml'}:
            x['discovery_requests'] += 1
        ua = user_agent_of(r)
        if ua:
            x['user_agents'].add(ua)
        if any(token.lower() in path.lower() for token in SUSPICIOUS_PATH_TOKENS):
            x['suspicious'] += 1
            suspicious_paths[path] += 1
        d = dt_of(r)
        if d:
            if id(r) in page_record_ids:
                x['page_events'].append((d, path))
            x['first'] = d if x['first'] is None or d < x['first'] else x['first']
            x['last'] = d if x['last'] is None or d > x['last'] else x['last']

    class_ips, class_requests = Counter(), Counter()
    for v in per_ip.values():
        v['classification'], v['score'], v['reasons'] = classify(v)
        class_ips[v['classification']] += 1
        class_requests[v['classification']] += v['requests']


    # A. Traffic over time, grouped by hour and final classification.
    traffic_by_hour: dict[str, Counter[str]] = defaultdict(Counter)
    for r in site:
        ip = str(r.get('ClientHost') or '')
        d = dt_of(r)
        if not ip or not d or ip not in per_ip:
            continue
        traffic_by_hour[d.isoformat(timespec='hours')][per_ip[ip]['classification']] += 1

    # B. Engaged candidates: human-like, at least two successful HTML pageviews
    # across at least two distinct HTML paths.
    engaged_ips: set[str] = set()
    successful_paths_by_ip: dict[str, set[str]] = defaultdict(set)
    for r in pages:
        ip = str(r.get('ClientHost') or '')
        if ip:
            successful_paths_by_ip[ip].add(str(r.get('RequestPath') or ''))
    for ip, v in per_ip.items():
        if v['classification'] in {'Human-like', 'Likely interactive'} and v['pages'] >= 2 and len(successful_paths_by_ip[ip]) >= 2:
            engaged_ips.add(ip)

    # C. Content interest, separated into all and human-like/engaged views.
    content_all: Counter[str] = Counter()
    content_human_like: Counter[str] = Counter()
    content_engaged: Counter[str] = Counter()
    for r in pages:
        ip = str(r.get('ClientHost') or '')
        category = content_category(str(r.get('RequestPath') or ''))
        content_all[category] += 1
        if ip in per_ip and per_ip[ip]['classification'] in {'Human-like', 'Likely interactive'}:
            content_human_like[category] += 1
        if ip in engaged_ips:
            content_engaged[category] += 1

    # D. Popular content and recent human-like activity.
    human_paths: Counter[str] = Counter()
    engaged_paths: Counter[str] = Counter()
    for r in pages:
        ip = str(r.get('ClientHost') or '')
        path = str(r.get('RequestPath') or '')
        if ip in per_ip and per_ip[ip]['classification'] in {'Human-like', 'Likely interactive'}:
            human_paths[path] += 1
        if ip in engaged_ips:
            engaged_paths[path] += 1
    popular_keys = sorted(set(human_paths) | set(engaged_paths), key=lambda x: (engaged_paths[x], human_paths[x], x), reverse=True)[:30]
    popular_content_table = ''.join(f'<tr><td>{esc(x)}</td><td>{human_paths[x]}</td><td>{engaged_paths[x]}</td></tr>' for x in popular_keys) or '<tr><td colspan="3">No interactive content activity</td></tr>'
    engaged_core = sum(1 for ip in engaged_ips if any(content_category(path) not in {'Home','About','Other content'} for path in successful_paths_by_ip[ip]))

    # D. Group suspicious paths into understandable probe categories.
    probe_categories: Counter[str] = Counter()
    for path, count in suspicious_paths.items():
        probe_categories[probe_category(path)] += count

    def shown_ip(ip: str) -> str:
        return ip if args.show_full_ip else mask_ip(ip)

    recent_human_table = ''.join(
        f'<tr><td>{esc(v["last"].isoformat(timespec="seconds") if v["last"] else "-")}</td><td>{esc(shown_ip(ip))}</td><td>{v["pages"]}</td><td>{v["requests"]}</td><td>{esc(", ".join(sorted({content_category(path) for path in successful_paths_by_ip[ip]})) or "-")}</td></tr>'
        for ip, v in sorted(per_ip.items(), key=lambda kv: kv[1]['last'] or datetime.min.replace(tzinfo=timezone.utc), reverse=True)
        if v['classification'] in {'Human-like', 'Likely interactive'}
    ) or '<tr><td colspan="5">No recent interactive activity</td></tr>'

    ip_table = ''.join(
        f'<tr><td>{esc(shown_ip(ip))}</td><td>{esc(v["classification"])}</td><td>{esc(v.get("crawler_network") or "-")}</td><td>{v["score"]}</td><td>{v["pages"]}</td><td>{v["requests"]}</td><td>{v["errors"]}</td><td>{len(v["paths"])}</td>'
        f'<td>{esc(v["first"].isoformat(timespec="seconds") if v["first"] else "-")}</td><td>{esc(v["last"].isoformat(timespec="seconds") if v["last"] else "-")}</td></tr>'
        for ip, v in sorted(per_ip.items(), key=lambda kv: (kv[1]['score'], kv[1]['pages'], kv[1]['requests']), reverse=True)
    ) or '<tr><td colspan="10">No matching traffic</td></tr>'

    class_order = ('Likely scanner','Other automation','Known crawler','Content crawler','Likely interactive','Human-like','Unclassified')
    classification_table = ''.join(f'<tr><td>{esc(k)}</td><td>{class_ips[k]}</td><td>{class_requests[k]}</td></tr>' for k in class_order)
    scanner_table = ''.join(
        f'<tr><td>{esc(shown_ip(ip))}</td><td>{v["score"]}</td><td>{v["requests"]}</td><td>{v["errors"]}</td><td>{(v["errors"]/v["requests"]):.1%}</td><td>{len(v["paths"])}</td><td>{esc("; ".join(v["reasons"]))}</td></tr>'
        for ip, v in sorted(per_ip.items(), key=lambda kv: (kv[1]['score'], kv[1]['requests']), reverse=True) if v['classification'] == 'Likely scanner'
    ) or '<tr><td colspan="7">No likely scanners detected by the current heuristic</td></tr>'

    def count_table(counter: Counter, label: str, limit: int = 30) -> str:
        body = ''.join(f'<tr><td>{esc(k)}</td><td>{v}</td></tr>' for k, v in counter.most_common(limit))
        if not body:
            body = '<tr><td colspan="2">No data</td></tr>'
        return f'<table><thead><tr><th>{esc(label)}</th><th>Count</th></tr></thead><tbody>{body}</tbody></table>'

    def geo_table(unique_counter: Counter[str], pageview_counter: Counter[str], label: str) -> str:
        keys = sorted(unique_counter, key=lambda k: (unique_counter[k], pageview_counter[k], k), reverse=True)
        body = ''.join(
            f'<tr><td>{esc(k)}</td><td>{unique_counter[k]}</td><td>{pageview_counter[k]}</td></tr>' for k in keys
        ) or '<tr><td colspan="3">No geolocation data</td></tr>'
        return f'<table><thead><tr><th>{esc(label)}</th><th>Unique pageview IPs</th><th>HTML pageviews</th></tr></thead><tbody>{body}</tbody></table>'

    error_table = ''.join(
        f'<tr><td>{esc((dt_of(r).isoformat(timespec="seconds") if dt_of(r) else "-"))}</td>'
        f'<td>{esc(shown_ip(str(r.get("ClientHost") or "")))}</td><td>{esc(r.get("DownstreamStatus"))}</td>'
        f'<td>{esc(r.get("RequestMethod"))}</td><td>{esc(r.get("RequestPath"))}</td></tr>'
        for r in sorted(errors, key=lambda x: dt_of(x) or datetime.min.replace(tzinfo=timezone.utc), reverse=True)[:50]
    ) or '<tr><td colspan="5">No errors</td></tr>'

    timeline_rows = []
    for hour in sorted(traffic_by_hour):
        c = traffic_by_hour[hour]
        total = sum(c.values())
        timeline_rows.append(
            f'<tr><td>{esc(hour)}</td><td>{total}</td><td>{c["Human-like"] + c["Likely interactive"]}</td>'
            f'<td>{c["Likely interactive"]}</td><td>{c["Content crawler"]}</td>'
            f'<td>{c["Other automation"]}</td><td>{c["Likely scanner"]}</td>'
            f'<td>{c["Known crawler"]}</td><td>{c["Human-like"]}</td><td>{c["Unclassified"]}</td></tr>'
        )
    timeline_table = ''.join(timeline_rows) or '<tr><td colspan="10">No timestamped traffic</td></tr>'

    content_keys = sorted(set(content_all) | set(content_human_like) | set(content_engaged), key=lambda k: content_all[k], reverse=True)
    content_table = ''.join(
        f'<tr><td>{esc(k)}</td><td>{content_all[k]}</td><td>{content_human_like[k]}</td><td>{content_engaged[k]}</td></tr>'
        for k in content_keys
    ) or '<tr><td colspan="4">No content data</td></tr>'

    generated = datetime.now().astimezone().isoformat(timespec='seconds')
    privacy = 'Full IP addresses shown' if args.show_full_ip else 'IP addresses masked to /24 (IPv4) or /48 (IPv6)'
    css = '''
:root{--bg:#eef1f5;--surface:#fff;--alt:#f7f9fb;--ink:#1b2430;--muted:#647386;--line:#d5dce5;--accent:#1f4e79}@media(prefers-color-scheme:dark){:root{--bg:#12171e;--surface:#1a212b;--alt:#151c25;--ink:#e6ebf1;--muted:#93a1b3;--line:#2c3644;--accent:#7db4e6}}*{box-sizing:border-box}html{scroll-behavior:smooth}body{margin:0;background:var(--bg);color:var(--ink);font:15px/1.45 "Segoe UI",system-ui,sans-serif}main{max-width:1300px;margin:auto;padding:24px}.card{background:var(--surface);border:1px solid var(--line);border-radius:7px;padding:18px;margin-bottom:16px}.metrics,.grid,.controls,.geo-summary{display:grid;gap:12px}.metrics{grid-template-columns:repeat(auto-fit,minmax(155px,1fr))}.grid{grid-template-columns:repeat(auto-fit,minmax(420px,1fr))}.metric,.geo-summary div{background:var(--alt);border:1px solid var(--line);border-radius:6px;padding:13px}.metric b,.geo-summary b{display:block;font-size:1.55rem;color:var(--accent)}.muted{color:var(--muted)}table{width:100%;border-collapse:collapse}th,td{text-align:left;padding:8px 9px;border-bottom:1px solid var(--line);vertical-align:top}th{color:var(--muted);font-size:.78rem}input,select{padding:8px;background:var(--bg);color:var(--ink);border:1px solid var(--line);border-radius:5px}.controls{grid-template-columns:1fr auto;margin-bottom:10px}.scroll{overflow:auto}.small{font-size:.8rem}.jump{display:flex;flex-wrap:wrap;gap:7px;margin-top:16px}.jump a,.back a{display:inline-flex;padding:6px 9px;border:1px solid var(--line);border-radius:5px;background:var(--alt);color:var(--accent);text-decoration:none;font-size:.78rem;font-weight:600}.back{margin:14px 0 0;text-align:right}.geo-summary{grid-template-columns:repeat(3,minmax(0,1fr));margin:12px 0}
details.card{padding:0;overflow:hidden}
details.card>summary{cursor:pointer;list-style:none;display:flex;align-items:center;gap:10px;padding:18px;user-select:none}
details.card>summary::-webkit-details-marker{display:none}
details.card>summary::before{content:"▶";color:var(--accent);font-size:.8rem;transition:transform .15s ease}
details.card[open]>summary::before{transform:rotate(90deg)}
details.card>summary h2{margin:0;font-size:1.25rem}
details.card[open]>summary{border-bottom:1px solid var(--line);margin-bottom:0}
details.card>summary~*{margin-left:18px;margin-right:18px}
details.card>summary+*{margin-top:18px}
details.card>.back{margin-bottom:18px}
details.card:target{outline:2px solid var(--accent);outline-offset:2px}
@media(max-width:700px){main{padding:10px}.grid,.controls,.geo-summary{grid-template-columns:1fr}}
'''

    page = f'''<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>NCRC Traffic Statistics</title><style>{css}</style></head><body><main id="top">
<section class="card"><h1>NCRC Traffic Statistics</h1><p class="muted">Traefik access log overview. Generated {esc(generated)}.</p><div class="metrics"><div class="metric"><b>{len(site)}</b>Total requests</div><div class="metric"><b>{len(pages)}</b>HTML pageviews</div><div class="metric"><b>{len(page_ips)}</b>Unique pageview IPs</div><div class="metric"><b>{class_ips["Likely interactive"]}</b>Likely interactive IPs</div><div class="metric"><b>{len(engaged_ips)}</b>Engaged candidates</div><div class="metric"><b>{sum(content_human_like.values())}</b>Interactive/human-like content views</div></div><p class="small muted">Window: {esc(first)} to {esc(last)} · {esc(privacy)} · malformed log lines skipped: {bad}</p><nav class="jump"><a href="#audience">Audience overview</a><a href="#timeline">Traffic over time</a><a href="#funnel">Engagement funnel</a><a href="#content-interest">Content interest</a><a href="#popular-content">Popular content</a><a href="#recent-human">Recent interactive activity</a><a href="#geography">Geography</a><a href="#visitors">Visitors by IP</a><a href="#scanner-activity">Scanner & probe activity</a><a href="#http-health">HTTP health & errors</a></nav></section>
<details class="card" id="audience"><summary><h2>Audience overview</h2></summary><p class="muted">Heuristic classification, not proof of identity or intent.</p><div class="scroll"><table><thead><tr><th>Classification</th><th>Unique IPs</th><th>Requests</th></tr></thead><tbody>{classification_table}</tbody></table></div>{back_to_top()}</details>
<details class="card" id="timeline"><summary><h2>Traffic over time</h2></summary><div class="scroll"><table><thead><tr><th>Hour</th><th>Total</th><th>Human-like total</th><th>Interactive</th><th>Content crawler</th><th>Other automation</th><th>Scanners</th><th>Known crawler</th><th>Human-like basic</th><th>Unclassified</th></tr></thead><tbody>{timeline_table}</tbody></table></div>{back_to_top()}</details>
<details class="card" id="funnel"><summary><h2>Engagement funnel</h2></summary><p class="muted">Behavioral progression through available log signals. IP-based counts are not proof of unique people.</p><div class="metrics"><div class="metric"><b>{len(ips)}</b>Unique request IPs</div><div class="metric"><b>{len(page_ips)}</b>Pageview IPs</div><div class="metric"><b>{class_ips["Likely interactive"]}</b>Likely interactive IPs</div><div class="metric"><b>{len(engaged_ips)}</b>Engaged candidates</div><div class="metric"><b>{engaged_core}</b>Engaged with core content</div></div>{back_to_top()}</details>
<details class="card" id="content-interest"><summary><h2>NCRC content interest</h2></summary><p class="muted">Successful HTML pageviews grouped by NCRC section, including Narratives.</p><div class="scroll"><table><thead><tr><th>Content area</th><th>All pageviews</th><th>Interactive/human-like</th><th>Engaged candidates</th></tr></thead><tbody>{content_table}</tbody></table></div>{back_to_top()}</details>
<details class="card" id="popular-content"><summary><h2>Popular content</h2></summary><p class="muted">Pages viewed by likely interactive/human-like and engaged visitor candidates.</p><div class="scroll"><table><thead><tr><th>Page</th><th>Interactive/human-like views</th><th>Engaged views</th></tr></thead><tbody>{popular_content_table}</tbody></table></div>{back_to_top()}</details>
<details class="card" id="recent-human"><summary><h2>Recent interactive activity</h2></summary><div class="scroll"><table><thead><tr><th>Last seen</th><th>IP</th><th>Pageviews</th><th>Requests</th><th>Content areas</th></tr></thead><tbody>{recent_human_table}</tbody></table></div>{back_to_top()}</details>
<details class="card" id="geography"><summary><h2>Traffic geography</h2></summary><div class="geo-summary"><div><b>{nl_unique}</b>Netherlands</div><div><b>{outside_nl_unique}</b>Outside Netherlands</div><div><b>{unknown_unique}</b>Unknown</div></div><p class="small muted">GeoIP status: {esc(geo_status)}</p><div class="grid"><div class="scroll">{geo_table(country_unique, country_pageviews, 'Country')}</div><div class="scroll">{geo_table(nl_region_unique, nl_region_pageviews, 'Netherlands region')}</div></div>{back_to_top()}</details>
<details class="card" id="visitors"><summary><h2>Visitors by IP</h2></summary><div class="controls"><input id="ipq" type="search" placeholder="Filter IP"><span></span></div><div class="scroll"><table id="iptable"><thead><tr><th>IP</th><th>Classification</th><th>Network context</th><th>Score</th><th>Pageviews</th><th>Requests</th><th>Errors</th><th>Unique paths</th><th>First seen</th><th>Last seen</th></tr></thead><tbody>{ip_table}</tbody></table></div>{back_to_top()}</details>
<details class="card" id="scanner-activity"><summary><h2>Scanner & probe activity</h2></summary><p class="muted">Heuristic indicators for scanner and probe behavior.</p><h3>Suspected scanners</h3><div class="scroll"><table><thead><tr><th>IP</th><th>Score</th><th>Requests</th><th>Errors</th><th>Error ratio</th><th>Unique paths</th><th>Signals</th></tr></thead><tbody>{scanner_table}</tbody></table></div><h3>Probe categories</h3><div class="scroll">{count_table(probe_categories, 'Probe category', 30)}</div><h3>Suspicious paths</h3><div class="scroll">{count_table(suspicious_paths, 'Path', 75)}</div>{back_to_top()}</details>
<details class="card" id="http-health"><summary><h2>HTTP health & errors</h2></summary><h3>Status distribution</h3><div class="scroll">{count_table(statuses, 'Status')}</div><h3>Latest HTTP errors</h3><div class="scroll"><table><thead><tr><th>Time</th><th>IP</th><th>Status</th><th>Method</th><th>Path</th></tr></thead><tbody>{error_table}</tbody></table></div>{back_to_top()}</details>
<script>const q=document.getElementById('ipq'),rows=[...document.querySelectorAll('#iptable tbody tr')];if(q)q.addEventListener('input',()=>{{const v=q.value.toLowerCase();rows.forEach(r=>r.style.display=r.cells[0].textContent.toLowerCase().includes(v)?'':'none')}});function openSection(id){{const t=document.getElementById(id);if(t&&t.tagName==='DETAILS')t.open=true;}}document.querySelectorAll('.jump a[href^="#"]').forEach(a=>a.addEventListener('click',()=>openSection(a.hash.slice(1))));window.addEventListener('hashchange',()=>openSection(location.hash.slice(1)));if(location.hash)openSection(location.hash.slice(1));</script>
</main></body></html>'''
    args.output.parent.mkdir(parents=True, exist_ok=True)
    args.output.write_text(page, encoding='utf-8')
    print(f'Wrote {args.output}: {len(site)} requests, {len(pages)} pageviews, {len(ips)} unique IPs')
    print(f'Classification: {dict(class_ips)}')
    print(f'Engaged candidates: {len(engaged_ips)}')
    print(f'GeoIP: {geo_status}')
    return 0


if __name__ == '__main__':
    raise SystemExit(main())
