#!/usr/bin/env python3
"""Build a standalone NCRC traffic statistics page from Traefik JSON access logs."""
from __future__ import annotations

import argparse
import html
import ipaddress
import json
import os
import urllib.error
import urllib.parse
import urllib.request
from collections import Counter, defaultdict
from datetime import datetime
from pathlib import Path
from typing import Any

try:
    import geoip2.database
    from geoip2.errors import AddressNotFoundError
except ImportError:
    geoip2 = None
    AddressNotFoundError = Exception

DEFAULT_LOG = Path('/mnt/user/appdata/traefik/logs/access.json')
DEFAULT_OUTPUT = Path('/mnt/bigstor/www/usecases/web_stat/statistics.html')
DEFAULT_GEOIP_DB = Path('/mnt/bigstor/www/usecases/data/GeoLite2-City.mmdb')
DEFAULT_ABUSEIP_OUTPUT = Path('/mnt/bigstor/www/usecases/data/abuse_ip.json')
ABUSEIPDB_CHECK_URL = 'https://api.abuseipdb.com/api/v2/check'


def parse_args() -> argparse.Namespace:
    p = argparse.ArgumentParser(description='Build NCRC traffic statistics HTML from Traefik access logs.')
    p.add_argument('--log', type=Path, default=DEFAULT_LOG)
    p.add_argument('--output', type=Path, default=DEFAULT_OUTPUT)
    p.add_argument('--host', default='ncrc.nl')
    p.add_argument('--router', default='usecases-https@file')
    p.add_argument('--show-full-ip', action='store_true', default=True, help='Show full client IP addresses in HTML')
#    p.add_argument('--show-full-ip', action='store_true', help='Show full client IP addresses in HTML')
    p.add_argument('--geoip-db', type=Path, default=DEFAULT_GEOIP_DB, help='Local GeoLite2-City.mmdb database')
    p.add_argument('--abuseip-output', type=Path, default=DEFAULT_ABUSEIP_OUTPUT)
    p.add_argument('--abuseip-max-age', type=int, default=90)
    p.add_argument('--abuseip-limit', type=int, default=950, help='Maximum new AbuseIPDB checks per run')
    p.add_argument('--abuseip-api-key-env', default='ABUSEIPDB_API_KEY')
    return p.parse_args()


def esc(v: Any) -> str:
    return html.escape(str(v if v is not None else ''), quote=True)


def read_log(path: Path) -> tuple[list[dict[str, Any]], int]:
    records, bad = [], 0
    with path.open('r', encoding='utf-8', errors='replace') as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            try:
                x = json.loads(line)
                if isinstance(x, dict):
                    records.append(x)
            except json.JSONDecodeError:
                bad += 1
    return records, bad


def dt_of(r: dict[str, Any]) -> datetime | None:
    raw = r.get('StartLocal') or r.get('time') or r.get('StartUTC')
    if not raw:
        return None
    try:
        return datetime.fromisoformat(str(raw).replace('Z', '+00:00'))
    except ValueError:
        return None


def mask_ip(value: str) -> str:
    try:
        ip = ipaddress.ip_address(value)
        if ip.version == 4:
            p = value.split('.')
            return '.'.join(p[:3] + ['0/24'])
        return str(ipaddress.ip_network(f'{ip}/48', strict=False))
    except ValueError:
        return value


def is_html_page(r: dict[str, Any]) -> bool:
    p = str(r.get('RequestPath') or '')
    return p in {'/', '/site', '/site/'} or p.endswith('.html')


def is_success(r: dict[str, Any]) -> bool:
    return int(r.get('DownstreamStatus') or 0) in {200, 304}



KNOWN_CRAWLER_TOKENS = ('googlebot','bingbot','duckduckbot','yandexbot','baiduspider','applebot','petalbot','slurp')
AUTOMATION_TOKENS = ('bot','crawler','spider','curl/','wget/','python-requests','python-urllib','go-http-client','aiohttp','okhttp')
SUSPICIOUS_PATH_TOKENS = ('/.env','/.git','/wp-admin','/wp-login','/wp-content','/wordpress','/phpmyadmin','/administrator','/cgi-bin','/vendor/phpunit','/actuator','/server-status','/boaform','/xmlrpc.php','/autodiscover','/owa/','/ecp/')


def user_agent_of(r: dict[str, Any]) -> str:
    for key in ('request_User-Agent','RequestUserAgent','User-Agent','user_agent','userAgent'):
        if r.get(key):
            return str(r[key])
    headers = r.get('RequestHeaders')
    if isinstance(headers, dict):
        value = headers.get('User-Agent') or headers.get('user-agent')
        if isinstance(value, list):
            return ', '.join(str(x) for x in value)
        return str(value or '')
    return ''


def classify(v: dict[str, Any]) -> tuple[str, int, list[str]]:
    req, pages, errors = v['requests'], v['pages'], v['errors']
    ratio = errors / req if req else 0.0
    ua = ' '.join(v['user_agents']).lower()
    reasons, score = [], 0
    if any(x in ua for x in KNOWN_CRAWLER_TOKENS):
        return 'Known crawler', 0, ['declared crawler User-Agent']
    if v['suspicious']:
        score += min(5, 2 + v['suspicious']); reasons.append(f"{v['suspicious']} suspicious path request(s)")
    if req >= 20 and ratio >= .80:
        score += 3; reasons.append(f'{ratio:.0%} errors')
    elif req >= 10 and ratio >= .60:
        score += 2; reasons.append(f'{ratio:.0%} errors')
    if len(v['paths']) >= 25:
        score += 2; reasons.append(f"{len(v['paths'])} unique paths")
    elif len(v['paths']) >= 10:
        score += 1; reasons.append(f"{len(v['paths'])} unique paths")
    if req >= 100:
        score += 2; reasons.append(f'{req} requests')
    elif req >= 40:
        score += 1; reasons.append(f'{req} requests')
    if pages == 0 and req >= 10:
        score += 1; reasons.append('no successful HTML pageviews')
    if any(x in ua for x in AUTOMATION_TOKENS):
        score += 2; reasons.append('automation-like User-Agent')
    if score >= 5:
        return 'Likely scanner', score, reasons
    if score >= 2:
        return 'Likely automation', score, reasons
    if pages and ratio < .5 and not v['suspicious']:
        return 'Human-like', score, ['successful HTML navigation without strong scanner signals']
    return 'Unclassified', score, reasons or ['insufficient evidence']


def content_category(path: str) -> str:
    p = path.lower()
    rules = (
        ('Threat Register', ('threat-register', '/threats/', 'register.html')),
        ('Regulatory', ('regulatory', 'r155', 'r156', '/cra')),
        ('Incidents', ('incident',)),
        ('Signals', ('signal',)),
        ('CVEs', ('cve', 'zero-day', 'zeroday')),
        ('GitHub', ('github',)),
        ('Papers', ('paper', 'research')),
        ('About', ('about',)),
        ('Home', ('/index.html', '/site/index.html', '/')),
    )
    for label, tokens in rules:
        if any(token == p or token in p for token in tokens):
            return label
    return 'Other content'


def probe_category(path: str) -> str:
    p = path.lower()
    rules = (
        ('WordPress', ('wp-admin','wp-login','wp-content','wordpress','xmlrpc.php')),
        ('Secrets/configuration', ('/.env','/config','server-status','backup')),
        ('Git repositories', ('/.git',)),
        ('PHP/phpMyAdmin', ('phpmyadmin','phpunit','.php')),
        ('Admin/login', ('/admin','/administrator','/login')),
        ('Microsoft/Exchange', ('autodiscover','/owa/','/ecp/')),
        ('Network/IoT', ('boaform','hnap1','cgi-bin')),
    )
    for label, tokens in rules:
        if any(token in p for token in tokens):
            return label
    return 'Other probes'

def geo_lookup(reader: Any, ip_value: str) -> tuple[str, str, str]:
    if reader is None:
        return 'ZZ', 'Unknown', 'Unknown'
    try:
        ip = ipaddress.ip_address(ip_value)
        if not ip.is_global:
            return 'ZZ', 'Unknown', 'Unknown'
        response = reader.city(ip_value)
        code = response.country.iso_code or 'ZZ'
        country = response.country.name or code
        region = response.subdivisions.most_specific.name or 'Unknown'
        return code, country, region
    except (ValueError, AddressNotFoundError):
        return 'ZZ', 'Unknown', 'Unknown'


def abuseipdb_check(ip_value: str, api_key: str, max_age: int) -> dict[str, Any]:
    query = urllib.parse.urlencode({'ipAddress': ip_value, 'maxAgeInDays': max_age})
    req = urllib.request.Request(f'{ABUSEIPDB_CHECK_URL}?{query}', headers={'Key': api_key, 'Accept': 'application/json'}, method='GET')
    try:
        with urllib.request.urlopen(req, timeout=15) as response:
            data = json.loads(response.read().decode('utf-8')).get('data', {})
        keep = ('ipAddress','abuseConfidenceScore','totalReports','numDistinctUsers','lastReportedAt','countryCode','usageType','isp','domain','isTor','isWhitelisted')
        result = {k: data.get(k) for k in keep}
        result['checkedAt'] = datetime.now().astimezone().isoformat(timespec='seconds')
        return result
    except urllib.error.HTTPError as e:
        detail = e.read().decode('utf-8', errors='replace')[:500]
        return {'ipAddress': ip_value, 'error': f'HTTP {e.code}', 'detail': detail, '_limit_reached': e.code == 429}
    except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as e:
        return {'ipAddress': ip_value, 'error': str(e)}

def load_abuse_cache(path: Path) -> dict[str, dict[str, Any]]:
    if not path.is_file(): return {}
    try:
        doc = json.loads(path.read_text(encoding='utf-8'))
        results = doc.get('results', {}) if isinstance(doc, dict) else {}
        return results if isinstance(results, dict) else {}
    except (OSError, json.JSONDecodeError):
        return {}

def update_abuse_cache(path: Path, ips: list[str], api_key: str, max_age: int, limit: int) -> tuple[dict[str, dict[str, Any]], dict[str, int | bool]]:
    cache = load_abuse_cache(path)
    public = []
    for value in sorted(set(ips)):
        try:
            if ipaddress.ip_address(value).is_global: public.append(value)
        except ValueError: pass
    pending = [ip for ip in public if ip not in cache]
    checked = 0; limit_reached = False
    if api_key:
        for ip in pending[:max(0, limit)]:
            result = abuseipdb_check(ip, api_key, max_age)
            if result.pop('_limit_reached', False):
                limit_reached = True; break
            cache[ip] = result; checked += 1
    doc = {'generatedAt': datetime.now().astimezone().isoformat(timespec='seconds'), 'maxAgeInDays': max_age, 'cachedIPs': len(cache), 'newChecksThisRun': checked, 'pendingIPs': max(0, len(pending)-checked), 'runLimit': limit, 'apiLimitReached': limit_reached, 'results': cache}
    path.parent.mkdir(parents=True, exist_ok=True); path.write_text(json.dumps(doc, indent=2, ensure_ascii=False)+'\n', encoding='utf-8')
    return cache, {'checked': checked, 'pending': max(0, len(pending)-checked), 'cached': len(cache), 'limit_reached': limit_reached}

def abuse_badge(value: Any) -> str:
    if not isinstance(value, int): return '<span class="abuse-score abuse-na">-</span>'
    cls = 'abuse-clean' if value == 0 else 'abuse-low' if value <= 25 else 'abuse-medium' if value <= 50 else 'abuse-high' if value <= 75 else 'abuse-critical'
    return f'<span class="abuse-score {cls}">{value}</span>'

def back_to_top() -> str:
    return '<p class="back"><a href="#top">Back to top ↑</a></p>'


def main() -> int:
    args = parse_args()
    rows, bad = read_log(args.log)
    site = [r for r in rows if r.get('RequestHost') == args.host and r.get('RouterName') == args.router]
    get_rows = [r for r in site if r.get('RequestMethod') == 'GET']
    pages = [r for r in get_rows if is_success(r) and is_html_page(r)]
    ips = Counter(str(r.get('ClientHost') or '') for r in site if r.get('ClientHost'))
    abuseip_results, abuse_status = update_abuse_cache(args.abuseip_output, list(ips), os.getenv(args.abuseip_api_key_env, '').strip(), args.abuseip_max_age, args.abuseip_limit)
    page_ips = Counter(str(r.get('ClientHost') or '') for r in pages if r.get('ClientHost'))
    paths = Counter(str(r.get('RequestPath') or '') for r in pages)
    statuses = Counter(int(r.get('DownstreamStatus') or 0) for r in site)
    errors = [r for r in site if int(r.get('DownstreamStatus') or 0) >= 400]
    threat_paths = Counter(str(r.get('RequestPath')) for r in pages if '/threats/T-' in str(r.get('RequestPath') or ''))
    dts = [d for d in (dt_of(r) for r in site) if d]
    first = min(dts).isoformat(timespec='seconds') if dts else '-'
    last = max(dts).isoformat(timespec='seconds') if dts else '-'

    geo_reader = None
    geo_status = 'disabled: local GeoIP database not found'
    if args.geoip_db.is_file() and geoip2 is not None:
        geo_reader = geoip2.database.Reader(str(args.geoip_db))
        geo_status = f'local database: {args.geoip_db.name}'
    elif args.geoip_db.is_file() and geoip2 is None:
        geo_status = 'disabled: Python package geoip2 is not installed'

    country_unique: Counter[str] = Counter()
    country_pageviews: Counter[str] = Counter()
    nl_region_unique: Counter[str] = Counter()
    nl_region_pageviews: Counter[str] = Counter()
    pageviews_by_ip = Counter(str(r.get('ClientHost') or '') for r in pages if r.get('ClientHost'))
    for ip, pageview_count in pageviews_by_ip.items():
        code, country, region = geo_lookup(geo_reader, ip)
        country_label = f'{country} ({code})' if code != 'ZZ' else 'Unknown'
        country_unique[country_label] += 1
        country_pageviews[country_label] += pageview_count
        if code == 'NL':
            nl_region_unique[region] += 1
            nl_region_pageviews[region] += pageview_count
    if geo_reader is not None:
        geo_reader.close()

    nl_unique = sum(v for label, v in country_unique.items() if label.endswith('(NL)'))
    unknown_unique = country_unique.get('Unknown', 0)
    outside_nl_unique = sum(country_unique.values()) - nl_unique - unknown_unique

    per_ip: dict[str, dict[str, Any]] = defaultdict(lambda: {'requests': 0, 'pages': 0, 'first': None, 'last': None, 'errors': 0, 'paths': set(), 'user_agents': set(), 'suspicious': 0})
    page_record_ids = {id(r) for r in pages}
    suspicious_paths: Counter[str] = Counter()
    suspicious_by_ip: dict[str, Counter[str]] = defaultdict(Counter)
    for r in site:
        ip = str(r.get('ClientHost') or '')
        if not ip:
            continue
        x = per_ip[ip]
        x['requests'] += 1
        if id(r) in page_record_ids:
            x['pages'] += 1
        if int(r.get('DownstreamStatus') or 0) >= 400:
            x['errors'] += 1
        path = str(r.get('RequestPath') or '')
        x['paths'].add(path)
        ua = user_agent_of(r)
        if ua:
            x['user_agents'].add(ua)
        if any(token.lower() in path.lower() for token in SUSPICIOUS_PATH_TOKENS):
            x['suspicious'] += 1
            suspicious_paths[path] += 1
            suspicious_by_ip[ip][probe_category(path)] += 1
        d = dt_of(r)
        if d:
            x['first'] = d if x['first'] is None or d < x['first'] else x['first']
            x['last'] = d if x['last'] is None or d > x['last'] else x['last']

    class_ips, class_requests = Counter(), Counter()
    for v in per_ip.values():
        v['classification'], v['score'], v['reasons'] = classify(v)
        class_ips[v['classification']] += 1
        class_requests[v['classification']] += v['requests']


    # A. Traffic over time, grouped by hour and final classification.
    traffic_by_hour: dict[str, Counter[str]] = defaultdict(Counter)
    for r in site:
        ip = str(r.get('ClientHost') or '')
        d = dt_of(r)
        if not ip or not d or ip not in per_ip:
            continue
        traffic_by_hour[d.isoformat(timespec='hours')][per_ip[ip]['classification']] += 1

    # B. Engaged candidates: human-like, at least two successful HTML pageviews
    # across at least two distinct HTML paths.
    engaged_ips: set[str] = set()
    successful_paths_by_ip: dict[str, set[str]] = defaultdict(set)
    for r in pages:
        ip = str(r.get('ClientHost') or '')
        if ip:
            successful_paths_by_ip[ip].add(str(r.get('RequestPath') or ''))
    for ip, v in per_ip.items():
        if v['classification'] == 'Human-like' and v['pages'] >= 2 and len(successful_paths_by_ip[ip]) >= 2:
            engaged_ips.add(ip)

    # C. Content interest, separated into all and human-like/engaged views.
    content_all: Counter[str] = Counter()
    content_human_like: Counter[str] = Counter()
    content_engaged: Counter[str] = Counter()
    for r in pages:
        ip = str(r.get('ClientHost') or '')
        category = content_category(str(r.get('RequestPath') or ''))
        content_all[category] += 1
        if ip in per_ip and per_ip[ip]['classification'] == 'Human-like':
            content_human_like[category] += 1
        if ip in engaged_ips:
            content_engaged[category] += 1

    # D. Group suspicious paths into understandable probe categories.
    probe_categories: Counter[str] = Counter()
    for path, count in suspicious_paths.items():
        probe_categories[probe_category(path)] += count

    def shown_ip(ip: str) -> str:
        return ip if args.show_full_ip else mask_ip(ip)

    ip_table = ''.join(
        f'<tr><td>{esc(shown_ip(ip))}</td><td>{esc(v["classification"])}</td><td>{v["score"]}</td><td>{abuse_badge(abuseip_results.get(ip, {}).get("abuseConfidenceScore"))}</td><td>{v["pages"]}</td><td>{v["requests"]}</td><td>{v["errors"]}</td><td>{len(v["paths"])}</td>'
        f'<td>{esc(v["first"].isoformat(timespec="seconds") if v["first"] else "-")}</td><td>{esc(v["last"].isoformat(timespec="seconds") if v["last"] else "-")}</td></tr>'
        for ip, v in sorted(per_ip.items(), key=lambda kv: (kv[1]['score'], kv[1]['pages'], kv[1]['requests']), reverse=True)
    ) or '<tr><td colspan="10">No matching traffic</td></tr>'

    class_order = ('Likely scanner','Likely automation','Known crawler','Human-like','Unclassified')
    classification_table = ''.join(f'<tr><td>{esc(k)}</td><td>{class_ips[k]}</td><td>{class_requests[k]}</td></tr>' for k in class_order)
    scanner_table = ''.join(
        f'<tr><td>{esc(shown_ip(ip))}</td><td>{v["score"]}</td><td>{v["requests"]}</td><td>{v["errors"]}</td><td>{(v["errors"]/v["requests"]):.1%}</td><td>{len(v["paths"])}</td><td>{esc("; ".join(v["reasons"]))}</td></tr>'
        for ip, v in sorted(per_ip.items(), key=lambda kv: (kv[1]['score'], kv[1]['requests']), reverse=True) if v['classification'] == 'Likely scanner'
    ) or '<tr><td colspan="7">No likely scanners detected by the current heuristic</td></tr>'

    def count_table(counter: Counter, label: str, limit: int = 30) -> str:
        body = ''.join(f'<tr><td>{esc(k)}</td><td>{v}</td></tr>' for k, v in counter.most_common(limit))
        if not body:
            body = '<tr><td colspan="2">No data</td></tr>'
        return f'<table><thead><tr><th>{esc(label)}</th><th>Count</th></tr></thead><tbody>{body}</tbody></table>'

    def geo_table(unique_counter: Counter[str], pageview_counter: Counter[str], label: str) -> str:
        keys = sorted(unique_counter, key=lambda k: (unique_counter[k], pageview_counter[k], k), reverse=True)
        body = ''.join(
            f'<tr><td>{esc(k)}</td><td>{unique_counter[k]}</td><td>{pageview_counter[k]}</td></tr>' for k in keys
        ) or '<tr><td colspan="3">No geolocation data</td></tr>'
        return f'<table><thead><tr><th>{esc(label)}</th><th>Unique pageview IPs</th><th>HTML pageviews</th></tr></thead><tbody>{body}</tbody></table>'

    error_table = ''.join(
        f'<tr><td>{esc((dt_of(r).isoformat(timespec="seconds") if dt_of(r) else "-"))}</td>'
        f'<td>{esc(shown_ip(str(r.get("ClientHost") or "")))}</td><td>{esc(r.get("DownstreamStatus"))}</td>'
        f'<td>{esc(r.get("RequestMethod"))}</td><td>{esc(r.get("RequestPath"))}</td></tr>'
        for r in sorted(errors, key=lambda x: dt_of(x) or datetime.min, reverse=True)[:50]
    ) or '<tr><td colspan="5">No errors</td></tr>'

    timeline_rows = []
    for hour in sorted(traffic_by_hour):
        c = traffic_by_hour[hour]
        total = sum(c.values())
        timeline_rows.append(
            f'<tr><td>{esc(hour)}</td><td>{total}</td><td>{c["Human-like"]}</td>'
            f'<td>{c["Likely automation"]}</td><td>{c["Likely scanner"]}</td>'
            f'<td>{c["Known crawler"]}</td><td>{c["Unclassified"]}</td></tr>'
        )
    timeline_table = ''.join(timeline_rows) or '<tr><td colspan="7">No timestamped traffic</td></tr>'

    content_keys = sorted(set(content_all) | set(content_human_like) | set(content_engaged), key=lambda k: content_all[k], reverse=True)
    content_table = ''.join(
        f'<tr><td>{esc(k)}</td><td>{content_all[k]}</td><td>{content_human_like[k]}</td><td>{content_engaged[k]}</td></tr>'
        for k in content_keys
    ) or '<tr><td colspan="4">No content data</td></tr>'

    # E. Scanner campaigns, built directly from the suspicious requests collected above.
    campaign_rows_data = []
    for ip, families in suspicious_by_ip.items():
        v = per_ip[ip]
        suspicious_requests = sum(families.values())
        abuse = abuseip_results.get(ip, {})
        campaign_rows_data.append((ip, v, families, suspicious_requests, abuse))

    campaign_rows_data.sort(
        key=lambda item: (item[3], item[1]['score'], item[4].get('abuseConfidenceScore') or 0, item[1]['requests']),
        reverse=True,
    )
    campaign_rows = ''.join(
        f'<tr><td>{esc(shown_ip(ip))}</td><td>{v["requests"]}</td><td>{suspicious_count}</td>'
        f'<td>{esc(v["classification"])}</td><td>{v["score"]}</td>'
        f'<td>{abuse_badge(abuse.get("abuseConfidenceScore"))}</td>'
        f'<td>{esc(abuse.get("isp") or "-")}</td>'
        f'<td>{esc(", ".join(f"{name}: {count}" for name, count in families.most_common()))}</td></tr>'
        for ip, v, families, suspicious_count, abuse in campaign_rows_data
    ) or '<tr><td colspan="8">No suspicious requests matched the configured probe patterns</td></tr>'

    campaign_family_totals: Counter[str] = Counter()
    for families in suspicious_by_ip.values():
        campaign_family_totals.update(families)

    abuse_rows = ''.join(f'<tr><td>{esc(shown_ip(ip))}</td><td>{abuse_badge(r.get("abuseConfidenceScore"))}</td><td>{esc(r.get("totalReports") if r.get("totalReports") is not None else "-")}</td><td>{esc(r.get("isp") or "-")}</td><td>{esc(r.get("usageType") or "-")}</td><td>{esc(r.get("isTor") if r.get("isTor") is not None else "-")}</td></tr>' for ip, r in sorted(abuseip_results.items(), key=lambda kv: kv[1].get('abuseConfidenceScore') or 0, reverse=True) if (r.get('abuseConfidenceScore') or 0) > 0) or '<tr><td colspan="6">No positive AbuseIPDB scores in cache</td></tr>'
    generated = datetime.now().astimezone().isoformat(timespec='seconds')
    privacy = 'Full IP addresses shown' if args.show_full_ip else 'IP addresses masked to /24 (IPv4) or /48 (IPv6)'
    css = '''
:root{--bg:#eef1f5;--surface:#fff;--alt:#f7f9fb;--ink:#1b2430;--muted:#647386;--line:#d5dce5;--accent:#1f4e79}@media(prefers-color-scheme:dark){:root{--bg:#12171e;--surface:#1a212b;--alt:#151c25;--ink:#e6ebf1;--muted:#93a1b3;--line:#2c3644;--accent:#7db4e6}}*{box-sizing:border-box}html{scroll-behavior:smooth}body{margin:0;background:var(--bg);color:var(--ink);font:15px/1.45 "Segoe UI",system-ui,sans-serif}main{max-width:1300px;margin:auto;padding:24px}.card{background:var(--surface);border:1px solid var(--line);border-radius:7px;padding:18px;margin-bottom:16px}.metrics,.grid,.controls,.geo-summary{display:grid;gap:12px}.metrics{grid-template-columns:repeat(auto-fit,minmax(155px,1fr))}.grid{grid-template-columns:repeat(auto-fit,minmax(420px,1fr))}.metric,.geo-summary div{background:var(--alt);border:1px solid var(--line);border-radius:6px;padding:13px}.metric b,.geo-summary b{display:block;font-size:1.55rem;color:var(--accent)}.muted{color:var(--muted)}table{width:100%;border-collapse:collapse}th,td{text-align:left;padding:8px 9px;border-bottom:1px solid var(--line);vertical-align:top}th{color:var(--muted);font-size:.78rem}input,select{padding:8px;background:var(--bg);color:var(--ink);border:1px solid var(--line);border-radius:5px}.controls{grid-template-columns:1fr auto;margin-bottom:10px}.scroll{overflow:auto}.small{font-size:.8rem}.jump{display:flex;flex-wrap:wrap;gap:7px;margin-top:16px}.jump a,.back a{display:inline-flex;padding:6px 9px;border:1px solid var(--line);border-radius:5px;background:var(--alt);color:var(--accent);text-decoration:none;font-size:.78rem;font-weight:600}.back{margin:14px 0 0;text-align:right}.abuse-score{display:inline-block;min-width:38px;text-align:center;padding:3px 7px;border-radius:999px;font-weight:700}.abuse-clean{background:#dff3e4;color:#176b2c}.abuse-low{background:#fff4c2;color:#715900}.abuse-medium{background:#ffe0ad;color:#7a4300}.abuse-high{background:#ffd0c7;color:#8b240f}.abuse-critical{background:#b42318;color:#fff}.abuse-na{background:var(--alt);color:var(--muted)}.geo-summary{grid-template-columns:repeat(3,minmax(0,1fr));margin:12px 0}@media(max-width:700px){main{padding:10px}.grid,.controls,.geo-summary{grid-template-columns:1fr}}
'''

    page = f'''<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>NCRC Traffic Statistics</title><style>{css}</style></head><body><main id="top">
<section class="card"><h1>NCRC Traffic Statistics</h1><p class="muted">Traefik access log overview for {esc(args.host)} via {esc(args.router)}. Generated {esc(generated)}.</p><div class="metrics">
<div class="metric"><b>{len(site)}</b>Total requests</div><div class="metric"><b>{len(engaged_ips)}</b>Engaged candidates</div><div class="metric"><b>{len(pages)}</b>HTML pageviews</div><div class="metric"><b>{len(ips)}</b>Unique request IPs</div><div class="metric"><b>{len(page_ips)}</b>Unique pageview IPs</div><div class="metric"><b>{len(errors)}</b>HTTP errors</div><div class="metric"><b>{len(threat_paths)}</b>Threat pages visited</div></div>
<p class="small muted">Window: {esc(first)} to {esc(last)} · {esc(privacy)} · malformed log lines skipped: {bad}</p>
<nav class="jump" aria-label="Page sections"><a href="#classification">Traffic classification</a><a href="#timeline">Traffic over time</a><a href="#engagement">Engagement</a><a href="#content-interest">Content interest</a><a href="#scanner-campaigns">Scanner campaigns</a><a href="#probe-categories">Probe categories</a><a href="#abuseipdb">AbuseIPDB</a><a href="#visitors">Visitors by IP</a><a href="#scanners">Suspected scanners</a><a href="#probes">Suspicious paths</a><a href="#pages">Most visited pages</a><a href="#threats">Most visited threats</a><a href="#statuses">HTTP statuses</a><a href="#requests-per-ip">Requests per IP</a><a href="#geography">Traffic geography</a><a href="#nl-regions">Netherlands by region</a><a href="#errors">Latest errors</a></nav></section>
<section class="card" id="classification"><h2>Traffic classification</h2><p class="muted">Heuristic classification, not proof of identity or intent. Signals combine suspicious paths, error ratio, volume, path diversity and available User-Agent data.</p><div class="scroll"><table><thead><tr><th>Classification</th><th>Unique IPs</th><th>Requests</th></tr></thead><tbody>{classification_table}</tbody></table></div>{back_to_top()}</section>
<section class="card" id="timeline"><h2>Traffic over time</h2><p class="muted">Requests per local-log hour, split by the final per-IP classification.</p><div class="scroll"><table><thead><tr><th>Hour</th><th>Total</th><th>Human-like</th><th>Automation</th><th>Scanners</th><th>Crawlers</th><th>Unclassified</th></tr></thead><tbody>{timeline_table}</tbody></table></div>{back_to_top()}</section>
<section class="card" id="engagement"><h2>Engaged visitor candidates</h2><p class="muted">Human-like IPs with at least two successful HTML pageviews across at least two different paths. This is a behavioral indicator, not proof of a unique person.</p><div class="metrics"><div class="metric"><b>{class_ips["Human-like"]}</b>Human-like IPs</div><div class="metric"><b>{len(engaged_ips)}</b>Engaged candidates</div><div class="metric"><b>{sum(per_ip[ip]["pages"] for ip in engaged_ips)}</b>Engaged pageviews</div></div>{back_to_top()}</section>
<section class="card" id="content-interest"><h2>NCRC content interest</h2><p class="muted">Successful HTML pageviews grouped by NCRC section. Human-like and engaged columns reduce scanner noise.</p><div class="scroll"><table><thead><tr><th>Content area</th><th>All pageviews</th><th>Human-like</th><th>Engaged candidates</th></tr></thead><tbody>{content_table}</tbody></table></div>{back_to_top()}</section>
<section class="card" id="scanner-campaigns"><h2>Scanner campaigns</h2><p class="muted">Suspicious requests grouped by source IP and probe family. This combines the local behavioral classification with the cached AbuseIPDB reputation; it remains an indicator rather than proof of intent.</p><div class="metrics"><div class="metric"><b>{len(campaign_rows_data)}</b>Source IPs</div><div class="metric"><b>{sum(row[3] for row in campaign_rows_data)}</b>Suspicious requests</div><div class="metric"><b>{len(campaign_family_totals)}</b>Probe families</div></div><div class="scroll"><table><thead><tr><th>IP</th><th>Requests</th><th>Suspicious</th><th>Classification</th><th>Local score</th><th>Abuse score</th><th>ISP</th><th>Probe families</th></tr></thead><tbody>{campaign_rows}</tbody></table></div><h3>Campaign probe families</h3>{count_table(campaign_family_totals, 'Probe family', 30)}{back_to_top()}</section>
<section class="card" id="probe-categories"><h2>Probe categories</h2><p class="muted">Suspicious paths grouped into recognizable families.</p><div class="scroll">{count_table(probe_categories, 'Probe category', 30)}</div>{back_to_top()}</section>
<section class="card" id="scanners"><h2>Suspected scanners</h2><div class="scroll"><table><thead><tr><th>IP</th><th>Score</th><th>Requests</th><th>Errors</th><th>Error ratio</th><th>Unique paths</th><th>Signals</th></tr></thead><tbody>{scanner_table}</tbody></table></div>{back_to_top()}</section>
<section class="card" id="probes"><h2>Suspicious or probed paths</h2><p class="muted">Paths matching common CMS, configuration, repository, administration or vulnerability-probe patterns.</p><div class="scroll">{count_table(suspicious_paths, 'Path', 75)}</div>{back_to_top()}</section>
<section class="card" id="abuseipdb"><h2>AbuseIPDB reputation</h2><p class="muted">Cached external IP reputation. Existing IPs are reused; only previously unseen public IPs are queried.</p><div class="metrics"><div class="metric"><b>{abuse_status['cached']}</b>Cached IPs</div><div class="metric"><b>{abuse_status['checked']}</b>New checks this run</div><div class="metric"><b>{abuse_status['pending']}</b>Pending IPs</div></div><p class="small"><span class="abuse-score abuse-clean">0</span> clean &nbsp; <span class="abuse-score abuse-low">1-25</span> low &nbsp; <span class="abuse-score abuse-medium">26-50</span> moderate &nbsp; <span class="abuse-score abuse-high">51-75</span> high &nbsp; <span class="abuse-score abuse-critical">76-100</span> very high</p><p class="small muted">Per-run lookup limit: {args.abuseip_limit} · API limit reached: {abuse_status['limit_reached']} · cache: {esc(args.abuseip_output)}</p><div class="scroll"><table><thead><tr><th>IP</th><th>Abuse score</th><th>Reports</th><th>ISP</th><th>Usage type</th><th>Tor</th></tr></thead><tbody>{abuse_rows}</tbody></table></div>{back_to_top()}</section>
<section class="card" id="visitors"><h2>Visitors by IP</h2><div class="controls"><input id="ipq" type="search" placeholder="Filter IP"><select id="ipsort"><option value="pages">Pageviews</option><option value="requests">Requests</option></select></div><div class="scroll"><table id="iptable"><thead><tr><th>IP</th><th>Classification</th><th>Local score</th><th>Abuse score</th><th>Pageviews</th><th>Requests</th><th>Errors</th><th>Unique paths</th><th>First seen</th><th>Last seen</th></tr></thead><tbody>{ip_table}</tbody></table></div>{back_to_top()}</section>
<section class="grid"><article class="card" id="pages"><h2>Most visited pages</h2>{count_table(paths, 'Page')}{back_to_top()}</article><article class="card" id="threats"><h2>Most visited threats</h2>{count_table(threat_paths, 'Threat page')}{back_to_top()}</article><article class="card" id="statuses"><h2>HTTP status distribution</h2>{count_table(statuses, 'Status')}{back_to_top()}</article><article class="card" id="requests-per-ip"><h2>Requests per IP</h2>{count_table(Counter(shown_ip(ip) for ip in ips.elements()), 'IP')}{back_to_top()}</article></section>
<section class="card" id="geography"><h2>Traffic geography</h2><p class="muted">Aggregated from unique IP addresses that requested at least one successful HTML page. Geolocation is approximate. Raw IP-to-location mappings are not written to this report.</p><div class="geo-summary"><div><b>{nl_unique}</b>Netherlands</div><div><b>{outside_nl_unique}</b>Outside Netherlands</div><div><b>{unknown_unique}</b>Unknown</div></div><p class="small muted">GeoIP status: {esc(geo_status)}</p><div class="scroll">{geo_table(country_unique, country_pageviews, 'Country')}</div>{back_to_top()}</section>
<section class="card" id="nl-regions"><h2>Netherlands by region</h2><p class="muted">Subdivision-level estimates for Dutch unique pageview IPs. These are network-location estimates, not precise visitor locations.</p><div class="scroll">{geo_table(nl_region_unique, nl_region_pageviews, 'Region')}</div>{back_to_top()}</section>
<section class="card" id="errors"><h2>Latest HTTP errors</h2><div class="scroll"><table><thead><tr><th>Time</th><th>IP</th><th>Status</th><th>Method</th><th>Path</th></tr></thead><tbody>{error_table}</tbody></table></div>{back_to_top()}</section>
<script>const q=document.getElementById('ipq'),rows=[...document.querySelectorAll('#iptable tbody tr')];q.addEventListener('input',()=>{{const v=q.value.toLowerCase();rows.forEach(r=>r.style.display=r.cells[0].textContent.toLowerCase().includes(v)?'':'none')}});</script>
</main></body></html>'''
    args.output.parent.mkdir(parents=True, exist_ok=True)
    args.output.write_text(page, encoding='utf-8')
    print(f'Wrote {args.output}: {len(site)} requests, {len(pages)} pageviews, {len(ips)} unique IPs')
    print(f'Classification: {dict(class_ips)}')
    print(f'Engaged candidates: {len(engaged_ips)}')
    print(f'GeoIP: {geo_status}')
    print(f"AbuseIPDB: cache={abuse_status['cached']}, new={abuse_status['checked']}, pending={abuse_status['pending']}, API-limit={abuse_status['limit_reached']}")
    print(f'Scanner campaigns: {len(campaign_rows_data)} IPs, {sum(row[3] for row in campaign_rows_data)} suspicious requests')
    return 0


if __name__ == '__main__':
    raise SystemExit(main())
