#!/usr/bin/env python3
"""Collect, classify, deduplicate and persist public cyber incident records.

The script deliberately separates real-world incidents from vulnerability-only
advisories and strategic publications. It uses deterministic rules only; it does
not invent facts that are absent from source material.

Default output:
    incidents/incidents.json

Examples:
    python3 incidents_intel.py
    python3 incidents_intel.py --days 31 --verbose
    python3 incidents_intel.py --feed https://example.org/feed.xml
    python3 incidents_intel.py --input-json incidents/raw/manual.json
    python3 incidents_intel.py --dry-run --pretty
    python3 incidents_intel.py --validate-only
"""

from __future__ import annotations

import argparse
import calendar
import dataclasses
import datetime as dt
import email.utils
import hashlib
import html
import json
import logging
import os
import re
import shutil
import sys
import tempfile
import time
import unicodedata
import urllib.error
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
from collections import Counter, defaultdict
from pathlib import Path
from typing import Any, Iterable, Iterator, Optional, Sequence
from html.parser import HTMLParser

SCRIPT_VERSION = "1.0.0"
SCHEMA_VERSION = "1.0"
DEFAULT_OUTPUT = Path("incidents/incidents.json")
DEFAULT_TIMEOUT = 25
DEFAULT_DAYS = 45
DEFAULT_USER_AGENT = f"NCRC-incidents-intel/{SCRIPT_VERSION} (+public-source-research)"

LOG = logging.getLogger("incidents_intel")

# Keep this list small and authoritative. Additional feeds can be supplied with
# --feed or in a JSON config file via --feeds-file.
DEFAULT_FEEDS = [
    {
        "name": "NCSC-NL News",
        "url": "https://feeds.ncsc.nl/nieuws.rss",
        "publisher": "Nationaal Cyber Security Centrum (NCSC-NL)",
        "source_type": "government_news",
        "authority_level": "high",
        "enabled": True,
    },



    {
        "name": "CERT-EU Threat Intelligence",
        "url": "https://cert.europa.eu/publications/threat-intelligence-rss",
        "publisher": "CERT-EU",
        "source_type": "government_threat_intelligence",
        "authority_level": "high",
        "enabled": True,
    },
    {
        "name": "CISA Alerts",
        "url": "https://www.cisa.gov/cybersecurity-advisories/alerts.xml",
        "publisher": "Cybersecurity and Infrastructure Security Agency (CISA)",
        "source_type": "government_alert",
        "authority_level": "high",
        "enabled": True,
    },
    {
        "name": "CISA ICS Advisories",
        "url": "https://www.cisa.gov/cybersecurity-advisories/ics-advisories.xml",
        "publisher": "Cybersecurity and Infrastructure Security Agency (CISA)",
        "source_type": "government_advisory",
        "authority_level": "high",
        # Disabled by default because most ICS advisories describe a possible
        # vulnerability, not an observed incident. Enable explicitly if wanted.
        "enabled": False,
    },
]

AIVD_SOURCES = [
    {
        "name": "AIVD News",
        "url": "https://www.aivd.nl/actueel/nieuws",
        "publisher": (
            "Algemene Inlichtingen- en "
            "Veiligheidsdienst (AIVD)"
        ),
        "source_type": "government_intelligence_news",
        "authority_level": "high",
        "path_prefix": "/actueel/nieuws/",
        "enabled": True,
    },
    {
        "name": "AIVD Documents",
        "url": "https://www.aivd.nl/documenten",
        "publisher": (
            "Algemene Inlichtingen- en "
            "Veiligheidsdienst (AIVD)"
        ),
        "source_type": (
            "government_intelligence_publication"
        ),
        "authority_level": "high",
        "path_prefix": "/documenten/",
        "enabled": False,
    },
]

INCIDENT_SIGNAL_PATTERNS: dict[str, list[str]] = {
    "ransomware": [
        r"\bransomware\b", r"\bgijzelsoftware\b", r"\bransom demand\b",
        r"\bdouble extortion\b", r"\bencrypted (?:systems|files|servers)\b",
    ],
    "data_breach": [
        r"\bdata breach\b", r"\bdatalek\b", r"\bdata (?:was )?(?:stolen|exfiltrated|leaked)\b",
        r"\bgegevens (?:zijn )?(?:gestolen|buitgemaakt|gelekt)\b",
        r"\bpersonal data (?:exposed|compromised)\b",
    ],
    "cyber_physical_attack": [
        r"\bcyber[- ]physical\b", r"\bphysical process\b", r"\bphysical damage\b",
        r"\btemperature(?:s)? (?:changed|raised|set)\b", r"\btemperatuur .*?(?:gewijzigd|verhoogd)\b",
        r"\bdefrost mode\b", r"\bontdooi(?:modus|cyclus)\b",
        r"\bcontroller settings (?:changed|modified)\b",
        r"\boperational technology\b", r"\bindustrial control system\b",
        r"\b(?:plc|scada|ics|ot) (?:attack|incident|compromise)\b",
        r"\bkoelinstallaties?\b",
        r"\bkoelsystemen?\b",
        r"\bkoelcontrollers?\b",
        r"\bvriesinstallaties?\b",
        r"\bvriezers?\b",
        r"\bkoelkasten?\b",
        r"\brefrigeration systems?\b",
        r"\brefrigeration controllers?\b",
        r"\btemperature manipulation\b",
        r"\btemperatuurmanipulatie\b",
    ],
    "supply_chain_attack": [
        r"\bsupply[- ]chain attack\b", r"\bsoftware supply chain\b",
        r"\bleveranciersaanval\b", r"\bthird[- ]party compromise\b",
        r"\btrusted update (?:was )?compromised\b",
    ],
    "espionage": [
        r"\bcyber espionage\b", r"\bcyberspionage\b", r"\bespionage campaign\b",
        r"\bsteal(?:ing)? sensitive information\b", r"\binformation gathering\b",
    ],
    "credential_compromise": [
        r"\bcredential(?:s)? (?:stolen|compromised|theft)\b",
        r"\binloggegevens (?:gestolen|buitgemaakt)\b", r"\baccount takeover\b",
        r"\bcompromised account\b",
    ],
    "service_disruption": [
        r"\bservice disruption\b", r"\boperational disruption\b", r"\boperations disrupted\b",
        r"\bsystems? (?:were )?(?:taken|knocked|forced) offline\b",
        r"\bsystemen .*?(?:offline|uitgeschakeld)\b", r"\bdenial[- ]of[- ]service attack\b",
        r"\bddos attack\b", r"\bdienstverlening .*?(?:verstoord|stilgelegd)\b",
    ],
    "sabotage": [
        r"\bcyber sabotage\b", r"\bsabotage\b", r"\bdestructive attack\b",
        r"\bwiper malware\b", r"\bdestroyed (?:data|systems|equipment)\b",
    ],
    "malware_attack": [
        r"\bmalware attack\b", r"\bmalware campaign\b", r"\bmalware infection\b",
        r"\bmalwarebesmetting\b", r"\btrojan(?:ized)?\b", r"\bbackdoor deployed\b",
    ],
    "unauthorized_access": [
        r"\bunauthorized access\b", r"\bunauthorised access\b", r"\bongeautoriseerde toegang\b",
        r"\bnetwork (?:was )?compromised\b", r"\binfrastructure (?:was )?compromised\b",
        r"\bsuspicious activity .*? malicious\b", r"\bkwaadwillende acties\b",
    ],
}

GENERIC_INCIDENT_PATTERNS = [
    r"\bcyber ?attack(?:s)?\b",
    r"\bcyberaanval(?:len)?\b",
    r"\bcyber incident\b",
    r"\bcybersecurity incident\b",
    r"\bsecurity incident\b",
    r"\bcompromis(?:e|ed)\b",
    r"\bhack(?:ed|ing|ers?)?\b",
    r"\bintrusion\b",
    r"\bbreach\b",
    r"\battackers? (?:accessed|targeted|exploited)\b",
    r"\bthreat actors? (?:accessed|targeted|compromised|exploited|deployed|stole)\b",
]

# Strong indicators that an item is an advisory, report or forecast rather than
# an observed event. They reduce the score but do not automatically discard an
# item if strong incident language is also present.
NON_INCIDENT_PATTERNS = [
    r"\bmay allow\b", r"\bcould allow\b", r"\bcan allow\b",
    r"\bkwetsbaarheid(?:heden)? (?:gevonden|verholpen)\b", r"\bvulnerabilit(?:y|ies)\b",
    r"\bsecurity update\b", r"\bpatch(?:ing)? advisory\b", r"\bproof of concept\b",
    r"\bthreat landscape report\b", r"\bannual report\b", r"\bcyber brief\b",
    r"\bguidance\b", r"\bbest practices\b", r"\bwebinar\b",
]


VULNERABILITY_ONLY_PATTERNS = [
    r"\bknown exploited vulnerabilities?\b",
    r"\badds? (?:one|two|three|four|five|six|\d+) known exploited\b",
    r"\bkev catalog\b",
    r"\bkev-catalogus\b",
    r"\bkwetsbaarheid(?:heden)?\b",
    r"\bzero[- ]?day[- ]kwetsbaarheid\b",
    r"\bzeroday[- ]kwetsbaarheid\b",
    r"\bupdate (?:nu|direct)\b",
    r"\bpatch(?:en|ing)?\b",
    r"\bsecurity advisory\b",
    r"\bcve-\d{4}-\d+\b",
]

STRATEGIC_CONTENT_PATTERNS = [
    # General strategic assessments
    r"\bcybersecuritybeeld\b",
    r"\bthreat landscape\b",
    r"\bcyber brief\b",
    r"\bstrategic assessment\b",
    r"\bdreigingsbeeld\b",
    r"\bdreigingsanalyse\b",
    r"\bcyberadvies\b",

    # Increasing or changing threat
    r"\bverwacht(?:en)? (?:een )?toename\b",
    r"\btoename van cyberaanvallen\b",
    r"\bcyberaanvallen .*?(?:toenemen|toename)\b",
    r"\bdreiging .*?(?:neemt toe|groeit|versnelt)\b",
    r"\bdreigingen .*?(?:nemen toe|groeien|versnellen)\b",
    r"\bai versnelt\b",
    r"\bweerbaarheid\b",
    r"\bnu handelen noodzakelijk\b",

    # Intelligence / state actor observations
    r"\bstatelijke actoren?\b",
    r"\bstatelijke cyberactoren?\b",
    r"\bcyberactoren?\b",
    r"\bChinese cyberactoren?\b",
    r"\bChinese hackers?\b",
    r"\bRussische hackers?\b",
    r"\binlichtingendiensten?\b",

    # Espionage and intelligence risks
    r"\bspionagerisico(?:'s|s)?\b",
    r"\bspionage\b",
    r"\bcyberspionage\b",
    r"\bdigitale spionage\b",
    r"\bspionageoperatie\b",

    # Digital and connected-system threat assessments
    r"\bdigitale dreiging(?:en)?\b",
    r"\bveiligheidsrisico(?:'s|s)?\b",
    r"\bcyberrisico(?:'s|s)?\b",

    # Connected vehicles / automotive intelligence
    r"\bmoderne voertuigen\b",
    r"\bslimme voertuigen\b",
    r"\bconnected vehicles?\b",
    r"\bconnected cars?\b",
    r"\bvoertuigdata\b",
    r"\bdata over inzittenden\b",
    r"\bdigitale systemen .*? voertuigen\b",

    # Internet-connected and edge infrastructure
    r"\bedge devices?\b",
    r"\binternet verbonden\b",
    r"\bverbonden (?:met|aan) (?:het )?internet\b",
]


ORGANIZATIONAL_NEWS_PATTERNS = [
    r"\bblijft csirt\b",
    r"\bwordt csirt\b",
    r"\bsluit zich aan bij\b",
    r"\bmaakt afspraken over samenwerking\b",
    r"\bsamenwerking bij de cyberbeveiligingswet\b",
]

CONCRETE_INCIDENT_PATTERNS = [
    r"\bcyberaanvallen? op\b",
    r"\bcyberaanvallen? tegen\b",
    r"\bwerd(?:en)? getroffen\b",
    r"\bgetroffen door\b",
    r"\bwas attacked\b",
    r"\bwere attacked\b",
    r"\bwas hacked\b",
    r"\bhas been hacked\b",
    r"\bsystems? (?:were )?(?:taken|forced|knocked) offline\b",
    r"\bsystemen? .*?(?:offline gehaald|uitgeschakeld)\b",
    r"\boperations? (?:were )?disrupted\b",
    r"\bbedrijfsvoering .*?(?:verstoord|stilgelegd)\b",
    r"\bdata (?:was |were )?(?:stolen|exfiltrated|leaked)\b",
    r"\bgegevens .*?(?:gestolen|buitgemaakt|gelekt)\b",
    r"\bfiled a police report\b",
    r"\baangifte gedaan\b",
    r"\bincident response\b",
    r"\bmalicious actions\b",
    r"\bkwaadwillende acties\b",
    r"\btemperature(?:s)? .*?(?:raised|changed|set)\b",
    r"\btemperatu(?:ur|ren) .*?(?:verhoogd|gewijzigd|aangepast)\b",
    r"\bdefrost mode\b",
    r"\bontdooi(?:modus|cyclus|cycli)\b",
]


OBSERVED_PATTERNS = [
    r"\bconfirmed\b",
    r"\bbevestigd\b",
    r"\bwas attacked\b",
    r"\bwere attacked\b",
    r"\bgetroffen door\b",
    r"\bhas been hacked\b",
    r"\bwas hacked\b",
    r"\bdetected (?:suspicious|malicious) activity\b",
    r"\bmalicious actions\b",
    r"\bactive exploitation\b",
    r"\bactively exploited\b",
    r"\bactief misbruikt\b",
    r"\bincident response\b",
    r"\bfiled a police report\b",
    r"\baangifte\b",
    r"\bcaused\b",
    r"\bresulted in\b",
    r"\bleidde tot\b",

    # Dutch observed incident language
    r"\bcyberaanvallen? op\b",
    r"\bcyberaanvallen? (?:vinden|vonden) plaats\b",
    r"\bcyberaanvallen? (?:gericht op|tegen)\b",
    r"\bhackers? (?:hebben|hadden|kunnen) .*?(?:toegang|aangepast|gemanipuleerd)\b",
    r"\bwerd(?:en)? getroffen\b",
    r"\bgetroffen (?:door|bij)\b",
    r"\bincident (?:vond|heeft) plaats\b",
    r"\bmisbruik (?:vond|heeft) plaats\b",
]


SPECULATION_PATTERNS = [
    r"\bsuspected\b", r"\bvermoedelijk\b", r"\bpossibly\b", r"\bmogelijk\b",
    r"\breportedly\b", r"\bnaar verluidt\b", r"\bunconfirmed\b", r"\bniet bevestigd\b",
]

CVE_RE = re.compile(r"\bCVE-\d{4}-\d{4,7}\b", re.I)
URL_RE = re.compile(r"https?://[^\s<>\"]+", re.I)
TAG_RE = re.compile(r"<[^>]+>")
SPACE_RE = re.compile(r"\s+")

COUNTRY_TERMS = {
    "Netherlands": ["netherlands", "nederland", "dutch"],
    "Belgium": ["belgium", "belgie", "belgië", "belgian"],
    "Luxembourg": ["luxembourg", "luxemburg"],
    "France": ["france", "frankrijk", "french"],
    "Germany": ["germany", "duitsland", "german"],
    "Sweden": ["sweden", "zweden", "swedish"],
    "Finland": ["finland", "finnish"],
    "Ireland": ["ireland", "ierland", "irish"],
    "Estonia": ["estonia", "estland", "estonian"],
    "United Kingdom": ["united kingdom", "britain", "british", " uk "],
    "United States": ["united states", "u.s.", " us ", "american"],
    "Ukraine": ["ukraine", "ukrainian", "oekraine"],
    "Poland": ["poland", "polen", "polish"],
    "Denmark": ["denmark", "denemarken", "danish"],
    "Norway": ["norway", "noorwegen", "norwegian"],
    "Spain": ["spain", "spanje", "spanish"],
    "Italy": ["italy", "italie", "italië", "italian"],
}

SECTOR_TERMS = {
    "automotive": ["automotive", "vehicle", "vehicles", "car manufacturer",  "connected car",
    "connected vehicle", "connected vehicles", "voertuig", "voertuigen", "moderne auto",
    "moderne voertuigen", "slimme voertuigen", "voertuigdata"],
    "transportation": ["transport", "logistics", "railway", "airport", "maritime", "shipping"],
    "retail": ["retail", "supermarket", "shop", "store", "winkel", "supermarkt"],
    "food": ["food supply", "food safety", "groceries", "refrigeration", "koelinstallatie"],
    "energy": ["energy", "electricity", "power grid", "oil", "gas pipeline"],
    "healthcare": ["healthcare", "hospital", "medical", "patient", "zorg", "ziekenhuis"],
    "government": ["government", "ministry", "municipality", "public authority", "overheid", "gemeente"],
    "finance": ["bank", "financial", "insurance", "payment", "financiele", "financiële"],
    "telecommunications": ["telecom", "telecommunications", "mobile operator", "internet provider"],
    "manufacturing": ["manufacturing", "factory", "industrial production", "fabriek"],
    "water": ["water utility", "water treatment", "wastewater", "drinkwater"],
    "education": ["university", "school", "education", "universiteit", "onderwijs"],
    "technology": ["software provider", "cloud provider", "technology company", "it provider"],
    "OT": ["operational technology", "industrial control", "scada", " plc ", "ics", "controller"],
}

THEME_TERMS = {
    "OT": ["operational technology", "industrial control", "scada", " plc ", "ics"],
    "IoT": ["internet of things", "iot", "connected device", "internet-connected"],
    "remote_access": ["remote access", "remotely accessed", "op afstand", "remote control"],
    "internet_exposure": ["internet-exposed", "internet facing", "publicly accessible", "rechtstreeks toegankelijk via het internet"],
    "identity": ["credential", "account takeover", "identity", "phishing", "inloggegevens"],
    "supply_chain": ["supply chain", "third party", "supplier", "leverancier"],
    "cyber_physical": ["cyber-physical", "physical process", "physical damage", "temperature", "defrost", "controller"],
    "AI": ["artificial intelligence", " ai ", "machine learning", "kunstmatige intelligentie"],
    "state_actor": ["state-sponsored", "state actor", "government-linked", "statelijke actor"],
    "automotive": [
        "automotive",
        "vehicle",
        "vehicles",
        "voertuig",
        "voertuigen",
        "moderne auto",
        "slimme voertuigen",
        "connected car",
        "connected vehicle",
    ],

    "espionage": [
        "espionage",
        "spionage",
        "spionagerisico",
        "spionagerisico's",
        "cyberspionage",
        "digitale spionage",
    ],

    "data_collection": [
        "data verzamelen",
        "dataverzameling",
        "vehicle data",
        "voertuigdata",
        "data over inzittenden",
    ],
}

IMPACT_TERMS = {
    "operational_disruption": ["operational disruption", "operations disrupted", "offline", "stilgelegd", "uitgeschakeld"],
    "data_theft": ["data stolen", "data exfiltration", "gestolen gegevens", "buitgemaakt"],
    "data_exposure": ["data leak", "data exposed", "datalek", "gelekt"],
    "financial_loss": ["financial loss", "million euros", "miljoenen euro", "economic damage"],
    "physical_process_manipulation": ["physical process", "temperature", "defrost", "controller settings", "ontdooicyclus"],
    "equipment_damage": ["equipment damage", "fire", "brand", "destroyed equipment"],
    "product_loss": ["spoiled food", "food loss", "products destroyed", "bedorven", "weggegooid"],
    "service_unavailability": ["service unavailable", "website unavailable", "services offline", "niet beschikbaar"],
    "safety_risk": ["safety risk", "food safety", "patient safety", "public safety", "veiligheid"],
}

ATTACK_VECTOR_TERMS = {
    "internet_exposed_service": ["internet-exposed", "internet facing", "publicly accessible"],
    "remote_access": ["remote access", "remotely accessed", "op afstand"],
    "phishing": ["phishing", "spearphishing"],
    "credential_abuse": ["stolen credentials", "compromised credentials", "default password", "weak password", "standaardwachtwoord"],
    "vulnerability_exploitation": ["exploited vulnerability", "exploitation of", "misbruik van een kwetsbaarheid", "zero-day"],
    "third_party": ["third party", "supplier", "service provider", "leverancier"],
}

AUTHENTICATION_TERMS = {
    "default_credentials": ["default credentials", "default password", "standaardwachtwoord"],
    "weak_credentials": ["weak password", "weak credentials", "zwak wachtwoord"],
    "stolen_credentials": ["stolen credentials", "credential theft", "gestolen inloggegevens"],
    "missing_mfa": ["without multi-factor", "no mfa", "missing mfa", "geen mfa"],
}

SOURCE_AUTHORITY_WEIGHTS = {"high": 0.22, "medium": 0.12, "low": 0.05, "unknown": 0.0}


@dataclasses.dataclass
class RawItem:
    feed_name: str
    publisher: str
    source_type: str
    authority_level: str
    title: str
    url: str
    summary: str
    content: str
    published: Optional[str]
    updated: Optional[str]
    guid: Optional[str] = None
    record_type: Optional[str] = None

    @property
    def text(self) -> str:
        return clean_text(
            " ".join(
                x
                for x in [
                    self.title,
                    self.summary,
                    self.content,
                ]
                if x
            )
        )

class CollectorError(RuntimeError):
    pass


def utc_now() -> dt.datetime:
    return dt.datetime.now(dt.timezone.utc)


def iso_z(value: Optional[dt.datetime] = None) -> str:
    value = value or utc_now()
    return value.astimezone(dt.timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")


def clean_text(value: Any) -> str:
    if value is None:
        return ""
    text = html.unescape(str(value))
    text = TAG_RE.sub(" ", text)
    text = unicodedata.normalize("NFKC", text)
    return SPACE_RE.sub(" ", text).strip()


def normalize_url(url: str) -> str:
    url = html.unescape((url or "").strip())
    if not url:
        return ""
    try:
        parts = urllib.parse.urlsplit(url)
        query = urllib.parse.parse_qsl(parts.query, keep_blank_values=True)
        query = [(k, v) for k, v in query if not k.lower().startswith("utm_") and k.lower() not in {"fbclid", "gclid"}]
        path = re.sub(r"/{2,}", "/", parts.path or "/")
        return urllib.parse.urlunsplit((parts.scheme.lower(), parts.netloc.lower(), path.rstrip("/") or "/", urllib.parse.urlencode(query), ""))
    except Exception:
        return url


def slug_tokens(text: str) -> list[str]:
    text = unicodedata.normalize("NFKD", text.lower())
    text = "".join(ch for ch in text if not unicodedata.combining(ch))
    return re.findall(r"[a-z0-9]{3,}", text)


def stable_hash(*parts: str, length: int = 20) -> str:
    material = "\x1f".join((p or "").strip().lower() for p in parts)
    return hashlib.sha256(material.encode("utf-8")).hexdigest()[:length]


def parse_date(value: Any) -> Optional[dt.datetime]:
    if not value:
        return None
    if isinstance(value, dt.datetime):
        return value if value.tzinfo else value.replace(tzinfo=dt.timezone.utc)
    if isinstance(value, dt.date):
        return dt.datetime.combine(value, dt.time.min, tzinfo=dt.timezone.utc)
    text = clean_text(value)
    if not text:
        return None
    try:
        parsed = email.utils.parsedate_to_datetime(text)
        if parsed:
            return parsed if parsed.tzinfo else parsed.replace(tzinfo=dt.timezone.utc)
    except (TypeError, ValueError, OverflowError):
        pass
    candidate = text.replace("Z", "+00:00")
    try:
        parsed = dt.datetime.fromisoformat(candidate)
        return parsed if parsed.tzinfo else parsed.replace(tzinfo=dt.timezone.utc)
    except ValueError:
        pass
    for fmt in ("%Y-%m-%d", "%d-%m-%Y", "%Y/%m/%d", "%d/%m/%Y"):
        try:
            return dt.datetime.strptime(text[:10], fmt).replace(tzinfo=dt.timezone.utc)
        except ValueError:
            continue
    return None


def to_iso_date(value: Any) -> Optional[str]:
    parsed = parse_date(value)
    return parsed.date().isoformat() if parsed else None


def is_recent(value: Optional[str], days: int, now: Optional[dt.datetime] = None) -> bool:
    if days <= 0 or not value:
        return True
    parsed = parse_date(value)
    if not parsed:
        return True
    now = now or utc_now()
    return parsed >= now - dt.timedelta(days=days)


def compile_patterns(patterns: Iterable[str]) -> list[re.Pattern[str]]:
    return [re.compile(p, re.I) for p in patterns]


COMPILED_INCIDENT = {
    key: compile_patterns(values)
    for key, values in INCIDENT_SIGNAL_PATTERNS.items()
}

COMPILED_GENERIC = compile_patterns(GENERIC_INCIDENT_PATTERNS)
COMPILED_NON_INCIDENT = compile_patterns(NON_INCIDENT_PATTERNS)
COMPILED_OBSERVED = compile_patterns(OBSERVED_PATTERNS)
COMPILED_SPECULATION = compile_patterns(SPECULATION_PATTERNS)

COMPILED_VULNERABILITY_ONLY = compile_patterns(
    VULNERABILITY_ONLY_PATTERNS
)

COMPILED_STRATEGIC_CONTENT = compile_patterns(
    STRATEGIC_CONTENT_PATTERNS
)

COMPILED_ORGANIZATIONAL_NEWS = compile_patterns(
    ORGANIZATIONAL_NEWS_PATTERNS
)

COMPILED_CONCRETE_INCIDENT = compile_patterns(
    CONCRETE_INCIDENT_PATTERNS
)

def match_count(text: str, patterns: Sequence[re.Pattern[str]]) -> int:
    return sum(1 for pattern in patterns if pattern.search(text))


def matched_terms(text: str, mapping: dict[str, list[str]]) -> list[str]:
    lower = f" {text.lower()} "
    result = []
    for label, terms in mapping.items():
        if any(term.lower() in lower for term in terms):
            result.append(label)
    return sorted(set(result))


def fetch_bytes(url: str, timeout: int, user_agent: str, retries: int = 2) -> bytes:
    request = urllib.request.Request(
        url,
        headers={
            "User-Agent": user_agent,
            "Accept": "application/rss+xml, application/atom+xml, application/xml, text/xml, application/json;q=0.9, */*;q=0.5",
        },
    )
    last_error: Optional[Exception] = None
    for attempt in range(retries + 1):
        try:
            with urllib.request.urlopen(request, timeout=timeout) as response:
                return response.read()
        except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, OSError) as exc:
            last_error = exc
            if attempt < retries:
                time.sleep(1.0 * (attempt + 1))
    raise CollectorError(f"Could not fetch {url}: {last_error}")


def child_text(node: ET.Element, names: Sequence[str]) -> str:
    wanted = {name.lower() for name in names}
    for child in list(node):
        local = child.tag.rsplit("}", 1)[-1].lower()
        if local in wanted:
            return clean_text("".join(child.itertext()))
    return ""


def child_link(node: ET.Element) -> str:
    for child in list(node):
        local = child.tag.rsplit("}", 1)[-1].lower()
        if local != "link":
            continue
        href = child.attrib.get("href", "")
        rel = child.attrib.get("rel", "alternate")
        if href and rel in {"alternate", ""}:
            return href
        text = clean_text("".join(child.itertext()))
        if text.startswith("http"):
            return text
    return ""


def parse_feed_xml(data: bytes, feed: dict[str, Any]) -> list[RawItem]:
    try:
        root = ET.fromstring(data)
    except ET.ParseError as exc:
        raise CollectorError(f"Invalid XML from {feed['url']}: {exc}") from exc

    root_name = root.tag.rsplit("}", 1)[-1].lower()
    if root_name == "rss":
        channel = next((x for x in root if x.tag.rsplit("}", 1)[-1].lower() == "channel"), root)
        nodes = [x for x in channel if x.tag.rsplit("}", 1)[-1].lower() == "item"]
    elif root_name == "feed":
        nodes = [x for x in root if x.tag.rsplit("}", 1)[-1].lower() == "entry"]
    else:
        nodes = [x for x in root.iter() if x.tag.rsplit("}", 1)[-1].lower() in {"item", "entry"}]

    results: list[RawItem] = []
    for node in nodes:
        title = child_text(node, ["title"])
        link = child_link(node) or child_text(node, ["link"])
        summary = child_text(node, ["description", "summary"])
        content = child_text(node, ["encoded", "content"])
        published = child_text(node, ["pubdate", "published", "date", "issued"])
        updated = child_text(node, ["updated", "modified"])
        guid = child_text(node, ["guid", "id"])
        if not title and not link:
            continue
        results.append(
            RawItem(
                feed_name=feed.get("name", feed.get("url", "feed")),
                publisher=feed.get("publisher", feed.get("name", "Unknown")),
                source_type=feed.get("source_type", "rss"),
                authority_level=feed.get("authority_level", "unknown"),
                title=title or "Untitled source item",
                url=normalize_url(link),
                summary=summary,
                content=content,
                published=to_iso_date(published),
                updated=to_iso_date(updated),
                guid=guid or None,
            )
        )
    return results


class AIVDOverviewParser(HTMLParser):
    """
    Extract dated AIVD publication links from an overview page.

    This parser deliberately uses only Python's standard library.
    The detail pages are fetched separately for their summary and
    publication date.
    """

    def __init__(
        self,
        base_url: str,
        path_prefix: str,
    ) -> None:
        super().__init__()

        self.base_url = base_url
        self.path_prefix = path_prefix

        self.links: list[dict[str, str]] = []

        self._inside_link = False
        self._current_href = ""
        self._current_text: list[str] = []

    def handle_starttag(
        self,
        tag: str,
        attrs: list[tuple[str, Optional[str]]],
    ) -> None:
        if tag.lower() != "a":
            return

        attributes = dict(attrs)
        href = attributes.get("href") or ""

        absolute_url = urllib.parse.urljoin(
            self.base_url,
            href,
        )

        parsed = urllib.parse.urlsplit(
            absolute_url,
        )

        if parsed.netloc.lower() != "www.aivd.nl":
            return

        if self.path_prefix not in parsed.path:
            return

        # Only accept dated detail pages:
        # /documenten/YYYY/MM/DD/...
        # /actueel/nieuws/YYYY/MM/DD/...
        date_pattern = (
            re.escape(self.path_prefix)
            + r"\d{4}/\d{2}/\d{2}/"
        )

        if not re.search(date_pattern, parsed.path):
            return

        self._inside_link = True
        self._current_href = normalize_url(
            absolute_url,
        )
        self._current_text = []

    def handle_data(self, data: str) -> None:
        if self._inside_link:
            self._current_text.append(data)

    def handle_endtag(self, tag: str) -> None:
        if tag.lower() != "a":
            return

        if not self._inside_link:
            return

        title = clean_text(
            " ".join(self._current_text)
        )

        if title and self._current_href:
            self.links.append(
                {
                    "title": title,
                    "url": self._current_href,
                }
            )

        self._inside_link = False
        self._current_href = ""
        self._current_text = []

META_TAG_RE = re.compile(
    r"""<meta\s+
        (?=[^>]*\b(?:name|property)=["']
        (?P<name>[^"']+)["'])
        (?=[^>]*\bcontent=["']
        (?P<content>[^"']*)["'])
        [^>]*>
    """,
    re.IGNORECASE | re.VERBOSE,
)


TIME_TAG_RE = re.compile(
    r"""<time\b[^>]*datetime=["']
        (?P<datetime>[^"']+)
        ["'][^>]*>
    """,
    re.IGNORECASE | re.VERBOSE,
)


def extract_html_metadata(
    html_text: str,
) -> dict[str, str]:
    """
    Extract useful metadata from HTML meta elements.

    Returns a dictionary indexed by the lower-case value
    of the meta element's name or property attribute.
    """

    metadata: dict[str, str] = {}

    for match in META_TAG_RE.finditer(html_text):
        name = clean_text(
            match.group("name")
        ).lower()

        content = clean_text(
            match.group("content")
        )

        if name and content:
            metadata[name] = content

    return metadata


def extract_aivd_date_from_url(
    url: str,
) -> Optional[str]:
    """
    Extract the publication date from an AIVD detail-page URL.

    Expected URL structures:

        /documenten/YYYY/MM/DD/...
        /actueel/nieuws/YYYY/MM/DD/...

    Returns YYYY-MM-DD or None.
    """

    path = urllib.parse.urlsplit(url).path

    match = re.search(
        r"/(\d{4})/(\d{2})/(\d{2})/",
        path,
    )

    if not match:
        return None

    year, month, day = match.groups()

    try:
        value = dt.date(
            int(year),
            int(month),
            int(day),
        )
    except ValueError:
        return None

    return value.isoformat()


def extract_aivd_detail(
    url: str,
    timeout: int,
    user_agent: str,
) -> dict[str, Optional[str]]:
    """
    Fetch an AIVD detail page and extract basic metadata.

    Metadata in the page is preferred. If no publication date
    is available in the metadata or a time element, the date
    encoded in the URL is used as fallback.
    """

    data = fetch_bytes(
        url,
        timeout=timeout,
        user_agent=user_agent,
    )

    html_text = data.decode(
        "utf-8",
        errors="replace",
    )

    metadata = extract_html_metadata(
        html_text,
    )

    title = (
        metadata.get("og:title")
        or metadata.get("twitter:title")
        or metadata.get("title")
    )

    description = (
        metadata.get("description")
        or metadata.get("og:description")
        or metadata.get("twitter:description")
        or ""
    )

    published = (
        metadata.get("article:published_time")
        or metadata.get("date")
        or metadata.get("dcterms.date")
    )

    updated = (
        metadata.get("article:modified_time")
        or metadata.get("last-modified")
    )

    if not published:
        time_match = TIME_TAG_RE.search(
            html_text,
        )

        if time_match:
            published = time_match.group(
                "datetime"
            )
    published_date = (
        to_iso_date(published)
        or extract_aivd_date_from_url(url)
    )

    updated_date = to_iso_date(
        updated
    )

    return {
        "title": clean_text(title),
        "description": clean_text(description),
        "published": published_date,
        "updated": updated_date,
    }
   

def collect_aivd_source(
    source: dict[str, Any],
    timeout: int,
    user_agent: str,
    days: int,
) -> list[RawItem]:
    LOG.info(
        "Fetching %s: %s",
        source["name"],
        source["url"],
    )

    data = fetch_bytes(
        source["url"],
        timeout=timeout,
        user_agent=user_agent,
    )

    html_text = data.decode(
        "utf-8",
        errors="replace",
    )

    parser = AIVDOverviewParser(
        base_url=source["url"],
        path_prefix=source["path_prefix"],
    )

    parser.feed(html_text)

    unique_links: dict[str, dict[str, str]] = {}

    for link in parser.links:
        unique_links[link["url"]] = link

    LOG.info(
        "Discovered %d AIVD detail links from %s",
        len(unique_links),
        source["name"],
    )

    items: list[RawItem] = []

    for link in unique_links.values():
        url_date = extract_aivd_date_from_url(
            link["url"],
        )

        if not is_recent(url_date, days):
            continue

        try:
            detail = extract_aivd_detail(
                link["url"],
                timeout=timeout,
                user_agent=user_agent,
            )
        except Exception as exc:
            LOG.warning(
                "Could not fetch AIVD detail page %s: %s",
                link["url"],
                exc,
            )

            detail = {
                "title": link["title"],
                "description": "",
                "published": url_date,
                "updated": None,
            }

        title = (
            detail.get("title")
            or link["title"]
            or "Untitled AIVD publication"
        )

        title = (
            detail.get("title")
            or link["title"]
            or "Untitled AIVD publication"
        )

        items.append(
            RawItem(
                feed_name=source["name"],
                publisher=source["publisher"],
                source_type=source["source_type"],
                authority_level=source["authority_level"],
                title=clean_text(title),
                url=link["url"],
                summary=clean_text(
                    detail.get("description")
                ),
                content="",
                published=(
                    detail.get("published")
                    or url_date
                ),
                updated=detail.get("updated"),
                guid=link["url"],
            )
        )

    LOG.info(
        "Collected %d source items from %s",
        len(items),
        source["name"],
    )

    return items
    LOG.info(
        "Collected %d source items from %s",
        len(items),
        source["name"],
    )

    return items



def collect_feed(feed: dict[str, Any], timeout: int, user_agent: str) -> list[RawItem]:
    LOG.info("Fetching %s: %s", feed.get("name", "feed"), feed["url"])
    data = fetch_bytes(feed["url"], timeout=timeout, user_agent=user_agent)
    items = parse_feed_xml(data, feed)
    LOG.info("Collected %d source items from %s", len(items), feed.get("name", "feed"))
    return items






def load_manual_json(path: Path) -> list[RawItem]:
    with path.open("r", encoding="utf-8") as handle:
        payload = json.load(handle)
    rows = payload.get("items", payload) if isinstance(payload, dict) else payload
    if not isinstance(rows, list):
        raise ValueError(f"Manual input {path} must contain a JSON list or an object with an 'items' list")
    items = []
    for index, row in enumerate(rows):
        if not isinstance(row, dict):
            LOG.warning("Skipping non-object row %d in %s", index, path)
            continue
        items.append(
            RawItem(
                feed_name=clean_text(row.get("feed_name") or path.name),
                publisher=clean_text(row.get("publisher") or "Manual input"),
                source_type=clean_text(row.get("source_type") or "manual"),
                authority_level=clean_text(row.get("authority_level") or "unknown"),
                title=clean_text(row.get("title") or "Untitled source item"),
                url=normalize_url(row.get("url") or ""),
                summary=clean_text(row.get("summary") or row.get("description") or ""),
                content=clean_text(row.get("content") or row.get("body") or ""),
                published=to_iso_date(row.get("published") or row.get("date")),
                updated=to_iso_date(row.get("updated")),
                guid=clean_text(row.get("guid") or row.get("id")) or None,
                record_type=clean_text(
                    row.get("record_type")
                ) or None,
            )
        )
    return items


def evaluate_incident_gate(
    item: RawItem,
) -> tuple[bool, list[str], dict[str, int]]:
    """
    Determine whether a source item describes a concrete, observed incident.

    A score alone is not sufficient. Strategic publications, organizational
    news and vulnerability-only advisories must not become incident records
    unless the text also contains clear evidence of a concrete incident.
    """

    text = item.text

    concrete_matches = match_count(
        text,
        COMPILED_CONCRETE_INCIDENT,
    )

    observed_matches = match_count(
        text,
        COMPILED_OBSERVED,
    )

    vulnerability_matches = match_count(
        text,
        COMPILED_VULNERABILITY_ONLY,
    )

    strategic_matches = match_count(
        text,
        COMPILED_STRATEGIC_CONTENT,
    )

    organizational_matches = match_count(
        text,
        COMPILED_ORGANIZATIONAL_NEWS,
    )

    typed_matches = sum(
        match_count(text, patterns)
        for patterns in COMPILED_INCIDENT.values()
    )

    reasons: list[str] = []

    explicit_record_type = clean_text(
        item.record_type
    ).lower()

    explicit_evidence_record = explicit_record_type in {
        "strategic_assessment",
        "threat_observation",
        "exploitation_activity",
    }

    if organizational_matches > 0 and typed_matches == 0:
        reasons.append("organizational_news_without_incident")


    if (
        vulnerability_matches > 0
        and concrete_matches == 0
    ):
        reasons.append("vulnerability_only_without_incident")

    if (
        not explicit_evidence_record
        and concrete_matches == 0
        and observed_matches == 0
        and strategic_matches == 0
    ):
        reasons.append("no_relevant_incident_or_intelligence")

    if (
        not explicit_evidence_record
        and concrete_matches == 0
        and typed_matches == 0
        and strategic_matches == 0
    ):
        reasons.append("no_incident_or_strategic_evidence")
    accepted = not reasons

    details = {
        "concrete_matches": concrete_matches,
        "observed_matches": observed_matches,
        "vulnerability_matches": vulnerability_matches,
        "strategic_matches": strategic_matches,
        "organizational_matches": organizational_matches,
        "typed_matches": typed_matches,
    }

    return accepted, reasons, details


def incident_score(item: RawItem) -> tuple[float, dict[str, Any]]:
    text = item.text
    generic = match_count(text, COMPILED_GENERIC)
    observed = match_count(text, COMPILED_OBSERVED)
    speculative = match_count(text, COMPILED_SPECULATION)
    non_incident = match_count(text, COMPILED_NON_INCIDENT)
    typed = {kind: match_count(text, patterns) for kind, patterns in COMPILED_INCIDENT.items()}
    typed = {kind: count for kind, count in typed.items() if count}

    score = 0.0
    score += min(generic, 3) * 0.18
    score += min(observed, 3) * 0.16
    score += min(sum(typed.values()), 4) * 0.13
    score += SOURCE_AUTHORITY_WEIGHTS.get(item.authority_level, 0.0)
    score -= min(non_incident, 3) * 0.13
    score -= min(speculative, 2) * 0.04

    # Vulnerability/advisory-only records should not pass because they mention
    # "attackers could". Active exploitation can still make them evidence of an
    # observed exploitation event, but not necessarily a named victim incident.
    if non_incident and not observed and generic == 0:
        score -= 0.30
    if "active exploitation" in text.lower() or "actief misbruikt" in text.lower():
        score += 0.15

    score = max(0.0, min(1.0, score))
    details = {
        "generic_matches": generic,
        "observed_matches": observed,
        "speculation_matches": speculative,
        "non_incident_matches": non_incident,
        "typed_matches": typed,
    }
    return round(score, 4), details


def classify_event(text: str) -> str:
    scores = {kind: match_count(text, patterns) for kind, patterns in COMPILED_INCIDENT.items()}
    priority = [
        "cyber_physical_attack", "ransomware", "supply_chain_attack", "sabotage",
        "espionage", "data_breach", "credential_compromise", "service_disruption",
        "malware_attack", "unauthorized_access",
    ]
    best = sorted(scores, key=lambda kind: (scores[kind], -priority.index(kind)), reverse=True)[0]
    if scores[best] > 0:
        return best
    return "cyber_attack"


def infer_status(text: str, authority_level: str) -> str:
    observed = match_count(text, COMPILED_OBSERVED)
    speculative = match_count(text, COMPILED_SPECULATION)
    if observed and authority_level == "high" and speculative == 0:
        return "confirmed"
    if observed:
        return "reported"
    if speculative:
        return "suspected"
    return "reported"


def infer_geography(text: str) -> dict[str, Any]:
    lower = f" {text.lower()} "
    countries = sorted({country for country, terms in COUNTRY_TERMS.items() if any(term in lower for term in terms)})
    if len(countries) > 1:
        scope = "multi_country"
    elif len(countries) == 1:
        scope = "country"
    elif any(term in lower for term in ["europe", "european", "eu-wide", "europa"]):
        scope = "regional"
    else:
        scope = "unknown"
    regions = ["Europe"] if any(term in lower for term in ["europe", "european", "europa"]) else []
    return {"scope": scope, "countries": countries, "regions": regions}


def infer_impact(text: str, event_type: str) -> dict[str, Any]:
    categories = matched_terms(text, IMPACT_TERMS)
    lower = text.lower()
    physical = event_type == "cyber_physical_attack" or any(x in categories for x in ["physical_process_manipulation", "equipment_damage", "product_loss"])
    financial = True if "financial_loss" in categories else None
    safety = True if "safety_risk" in categories else None
    availability = True if any(x in categories for x in ["operational_disruption", "service_unavailability"]) else None
    confidentiality = True if any(x in categories for x in ["data_theft", "data_exposure"]) else None
    integrity = True if any(x in categories for x in ["physical_process_manipulation", "data_theft"]) else None

    severity = "unknown"
    if physical or safety is True or len(categories) >= 4:
        severity = "high"
    elif len(categories) >= 2 or event_type in {"ransomware", "sabotage", "supply_chain_attack"}:
        severity = "medium"
    elif categories:
        severity = "low"

    descriptions = []
    for sentence in re.split(r"(?<=[.!?])\s+", text):
        sentence_l = sentence.lower()
        if any(term in sentence_l for terms in IMPACT_TERMS.values() for term in terms):
            descriptions.append(sentence.strip())
        if len(descriptions) == 2:
            break

    return {
        "categories": categories,
        "description": " ".join(descriptions)[:1200] or None,
        "physical": physical,
        "financial": financial,
        "safety": safety,
        "availability": availability,
        "confidentiality": confidentiality,
        "integrity": integrity,
        "severity": severity,
    }


def infer_cyber_physical(text: str, impact: dict[str, Any]) -> dict[str, Any]:
    is_cp = bool(impact.get("physical") or "cyber_physical" in matched_terms(text, THEME_TERMS))
    digital_action = None
    physical_effect = None
    lower = text.lower()
    if is_cp:
        if any(x in lower for x in ["remote access", "remotely", "op afstand"]):
            digital_action = "Reported remote access or remote manipulation of a connected system"
        elif any(x in lower for x in ["controller", "scada", " plc ", "ics"]):
            digital_action = "Reported manipulation or compromise of a control system"
        if any(x in lower for x in ["temperature", "defrost", "ontdooi"]):
            physical_effect = "Reported modification of a temperature-controlled physical process"
        elif "physical damage" in lower or "equipment damage" in lower or "fire" in lower:
            physical_effect = "Reported physical or equipment impact"
    return {
        "is_cyber_physical": is_cp,
        "digital_action": digital_action,
        "physical_effect": physical_effect,
        "potential_consequence": None,
    }


def infer_automotive_relevance(text: str, themes: list[str], sectors: list[str], cyber_physical: dict[str, Any]) -> dict[str, Any]:
    lower = text.lower()
    direct_terms = ["automotive", "vehicle", "vehicles", "connected car", "connected vehicle", "connected vehicles",
                   "car manufacturer", "voertuig", "voertuigen", "moderne auto", "moderne voertuigen", "voertuigdata"]
    direct = "automotive" in sectors or any(term in lower for term in direct_terms)
    score = 0.0
    reasons = []
    if direct:
        score += 0.60
        reasons.append("The source explicitly concerns vehicles or the automotive sector")
    if cyber_physical.get("is_cyber_physical"):
        score += 0.18
        reasons.append("The incident concerns digital influence on a physical process")
    if "remote_access" in themes:
        score += 0.08
        reasons.append("Remote access or remote manipulation is reported")
    if "OT" in themes or "IoT" in themes:
        score += 0.07
        reasons.append("The affected environment includes connected or operational technology")
    if "supply_chain" in themes:
        score += 0.05
        reasons.append("The incident includes a supply-chain or third-party dimension")
    score = round(min(score, 1.0), 2)
    if direct:
        relevance = "direct"
    elif score >= 0.25:
        relevance = "strongly_related"
    elif score >= 0.15:
        relevance = "analogous"
    elif score > 0:
        relevance = "contextual"
    else:
        relevance = "none"
    return {"direct": direct, "relevance": relevance, "score": score, "reason": "; ".join(reasons) or None}


def make_summary(item: RawItem, max_length: int = 850) -> str:
    summary = clean_text(item.summary or item.content)
    if not summary:
        return item.title
    if len(summary) <= max_length:
        return summary
    clipped = summary[:max_length].rsplit(" ", 1)[0]
    return clipped.rstrip(" ,;:") + "..."


def determine_record_type(
    item: RawItem,
    score_details: dict[str, Any],
) -> str:

    explicit_record_type = clean_text(
        item.record_type
    ).lower()

    if explicit_record_type in {
        "incident",
        "strategic_assessment",
        "threat_observation",
        "exploitation_activity",
    }:
        return explicit_record_type

    gate = score_details.get(
        "incident_gate",
        {},
    )

    # bestaande functie blijft hieronder staan



    gate = score_details.get(
        "incident_gate",
        {},
    )

    concrete_matches = gate.get(
        "concrete_matches",
        0,
    )

    strategic_matches = gate.get(
        "strategic_matches",
        0,
    )

    organizational_matches = gate.get(
        "organizational_matches",
        0,
    )

    vulnerability_matches = gate.get(
        "vulnerability_matches",
        0,
    )

    observed_matches = gate.get(
        "observed_matches",
        0,
    )

    typed_matches = gate.get(
        "typed_matches",
        0,
    )

    # Strategic publications remain strategic unless there is
    # specific incident-type evidence.
    if (
        strategic_matches > 0
        and typed_matches == 0
    ):
        return "strategic_assessment"

    # Organizational publications are not incidents merely
    # because their body refers to previous cyber incidents.
    if (
        organizational_matches > 0
        and typed_matches == 0
    ):
        return "organizational"

    if concrete_matches > 0:
        return "incident"

    if (
        vulnerability_matches > 0
        and observed_matches > 0
    ):
        return "exploitation_activity"

    if observed_matches > 0:
        return "threat_observation"

    return "unknown"

def build_record(item: RawItem, score: float, score_details: dict[str, Any], run_time: str) -> dict[str, Any]:
    text = item.text
    event_type = classify_event(text)

    record_type = determine_record_type(
       item,
       score_details,
    )

    if record_type != "incident":
       event_type = None

    status = infer_status(
        text,
        item.authority_level,
    )

    sectors = matched_terms(text, SECTOR_TERMS)
    themes = matched_terms(text, THEME_TERMS)
    if event_type == "cyber_physical_attack" and "cyber_physical" not in themes:
        themes.append("cyber_physical")
    themes = sorted(set(themes))
    impact = infer_impact(text, event_type)
    cp = infer_cyber_physical(text, impact)
    automotive = infer_automotive_relevance(text, themes, sectors, cp)
    attack_vectors = matched_terms(text, ATTACK_VECTOR_TERMS)
    auth_issues = matched_terms(text, AUTHENTICATION_TERMS)
    cves = sorted({match.upper() for match in CVE_RE.findall(text)})
    internet_exposed: Optional[bool] = True if "internet_exposed_service" in attack_vectors or "internet_exposure" in themes else None

    source_id = "SRC-" + stable_hash(item.publisher, item.url or item.guid or item.title, length=12).upper()
    title_key = " ".join(slug_tokens(item.title)[:18])
    deduplication_key = stable_hash(
        title_key,
        item.published or "",
        item.publisher or "",
        item.url or item.guid or "",
        length=24,
    )

    year = (item.published or str(utc_now().year))[:4]

    incident_id = (
        f"INC-{year}-"
        f"{stable_hash(deduplication_key, length=12).upper()}"
    )
    confidence_score = score
    verification = {
        "official_source": item.authority_level == "high",
        "multiple_independent_sources": False,
        "technical_analysis_available": item.source_type in {"threat_intelligence", "government_threat_intelligence", "technical_analysis"},
    }

    supports = ["incident_exists"]
    if sectors:
        supports.append("target_sector")
    if impact["categories"]:
        supports.append("impact")
    if attack_vectors:
        supports.append("attack_vector")
    if cves:
        supports.append("vulnerability")
    if cp["is_cyber_physical"]:
        supports.append("physical_impact")

    return {
        "incident_id": incident_id,
        "title": item.title,
        "summary": make_summary(item),
        "record_type": record_type,
        "status": status,
        "event_type": event_type,
        "dates": {
            "first_observed": None,
            "last_observed": None,
            "published": item.published,
            "updated": item.updated,
        },
        "geography": infer_geography(text),
        "sectors": sectors,
        "targets": [],
        "organizations": [],
        "attack": {
            "initial_access": [],
            "techniques": [],
            "attack_vector": attack_vectors,
            "vulnerabilities": [],
            "cves": cves,
            "authentication_issues": auth_issues,
            "internet_exposed": internet_exposed,
            "description": None,
        },
        "impact": impact,
        "cyber_physical": cp,
        "actors": {
            "known": False,
            "names": [],
            "type": [],
            "motivation": [],
            "attribution": None,
            "attribution_confidence": "unknown",
        },
        "themes": themes,
        "source_types": [item.source_type],
        "sources": [
            {
                "source_id": source_id,
                "publisher": item.publisher,
                "title": item.title,
                "url": item.url or None,
                "published": item.published,
                "source_type": item.source_type,
                "authority_level": item.authority_level,
                "supports": supports,
            }
        ],
        "evidence": {
            "confidence": status,
            "confidence_score": confidence_score,
            "verification": verification,
            "selection_details": score_details,
        },
        "correlation": {
            "threat_matches": [],
            "cve_matches": [],
            "paper_matches": [],
            "github_matches": [],
            "signal_matches": [],
            "narrative_matches": [],
        },
        "automotive_relevance": automotive,
        "campaign": {"campaign_id": None, "campaign_name": None, "related_incidents": []},

        "tags": sorted(
           set(
               tag
               for tag in (
                   ["real_world", record_type]
                   + ([event_type] if event_type else [])
                   + themes
               )
               if tag
           )
         ),


        "processing": {
            "created_at": run_time,
            "updated_at": run_time,
            "discovered_by": "incidents_intel.py",
            "deduplication_key": deduplication_key,
            "review_status": "pending",
            "reviewed_by": None,
        },
    }


def title_similarity(a: str, b: str) -> float:
    ta, tb = set(slug_tokens(a)), set(slug_tokens(b))
    if not ta or not tb:
        return 0.0
    return len(ta & tb) / len(ta | tb)


def source_identity(source: dict[str, Any]) -> str:
    return source.get("url") or source.get("source_id") or stable_hash(source.get("publisher", ""), source.get("title", ""))


def merge_records(primary: dict[str, Any], secondary: dict[str, Any], run_time: str) -> dict[str, Any]:
    merged = json.loads(json.dumps(primary))
    known_sources = {source_identity(source) for source in merged["sources"]}
    for source in secondary["sources"]:
        if source_identity(source) not in known_sources:
            merged["sources"].append(source)
            known_sources.add(source_identity(source))

    for field in ["sectors", "themes", "source_types", "tags", "organizations"]:
        merged[field] = sorted(set(merged.get(field, []) + secondary.get(field, [])))
    for field in ["countries", "regions"]:
        merged["geography"][field] = sorted(set(merged["geography"].get(field, []) + secondary["geography"].get(field, [])))
    if len(merged["geography"]["countries"]) > 1:
        merged["geography"]["scope"] = "multi_country"

    for field in ["attack_vector", "vulnerabilities", "cves", "authentication_issues", "initial_access", "techniques"]:
        merged["attack"][field] = sorted(set(merged["attack"].get(field, []) + secondary["attack"].get(field, [])))
    merged["impact"]["categories"] = sorted(set(merged["impact"].get("categories", []) + secondary["impact"].get("categories", [])))

    status_rank = {"unverified": 0, "suspected": 1, "reported": 2, "confirmed": 3}
    if status_rank.get(secondary["status"], 0) > status_rank.get(merged["status"], 0):
        merged["status"] = secondary["status"]
        merged["evidence"]["confidence"] = secondary["status"]

    merged["evidence"]["confidence_score"] = round(max(
        float(merged["evidence"].get("confidence_score") or 0),
        float(secondary["evidence"].get("confidence_score") or 0),
    ), 4)
    publishers = {source.get("publisher") for source in merged["sources"] if source.get("publisher")}
    merged["evidence"]["verification"]["multiple_independent_sources"] = len(publishers) > 1
    merged["evidence"]["verification"]["official_source"] = any(source.get("authority_level") == "high" for source in merged["sources"])
    merged["evidence"]["verification"]["technical_analysis_available"] = any(
        source.get("source_type") in {"threat_intelligence", "government_threat_intelligence", "technical_analysis"}
        for source in merged["sources"]
    )

    if len(secondary.get("summary", "")) > len(merged.get("summary", "")):
        merged["summary"] = secondary["summary"]
    merged["processing"]["updated_at"] = run_time
    return merged


def deduplicate(records: list[dict[str, Any]], threshold: float = 0.52) -> list[dict[str, Any]]:
    groups: list[dict[str, Any]] = []
    by_url: dict[str, int] = {}
    by_key: dict[str, int] = {}
    run_time = iso_z()

    for record in records:
        urls = [source.get("url") for source in record["sources"] if source.get("url")]
        exact_index = next((by_url[url] for url in urls if url in by_url), None)
        key = record["processing"].get("deduplication_key")
        if exact_index is None and key in by_key:
            exact_index = by_key[key]

        candidate_index = exact_index
        if candidate_index is None:
            published = record["dates"].get("published")
            for index, existing in enumerate(groups):
                existing_date = existing["dates"].get("published")
                if published and existing_date:
                    try:
                        delta = abs((dt.date.fromisoformat(published) - dt.date.fromisoformat(existing_date)).days)
                    except ValueError:
                        delta = 999
                    if delta > 14:
                        continue
                similarity = title_similarity(record["title"], existing["title"])
                shared_cves = set(record["attack"]["cves"]) & set(existing["attack"]["cves"])
                if similarity >= threshold or (shared_cves and similarity >= 0.30):
                    candidate_index = index
                    break

        if candidate_index is None:
            groups.append(record)
            index = len(groups) - 1
            for url in urls:
                by_url[url] = index
            if key:
                by_key[key] = index
        else:
            groups[candidate_index] = merge_records(groups[candidate_index], record, run_time)
            for url in urls:
                by_url[url] = candidate_index
            if key:
                by_key[key] = candidate_index

    return groups


def load_existing(path: Path) -> list[dict[str, Any]]:
    if not path.exists():
        return []
    try:
        with path.open("r", encoding="utf-8") as handle:
            payload = json.load(handle)
        incidents = payload.get("incidents", [])
        if not isinstance(incidents, list):
            raise ValueError("'incidents' is not a list")
        return incidents
    except Exception as exc:
        raise ValueError(f"Could not load existing output {path}: {exc}") from exc


def preserve_review_fields(new: dict[str, Any], old: dict[str, Any]) -> dict[str, Any]:
    result = json.loads(json.dumps(new))
    result["incident_id"] = old.get("incident_id", new["incident_id"])
    result["processing"]["created_at"] = old.get("processing", {}).get("created_at") or new["processing"]["created_at"]
    for field in ["review_status", "reviewed_by"]:
        old_value = old.get("processing", {}).get(field)
        if old_value not in (None, "", "pending"):
            result["processing"][field] = old_value
    for field in ["targets", "organizations"]:
        if old.get(field):
            result[field] = old[field]
    for section in ["actors", "campaign"]:
        if old.get(section) and any(v not in (None, False, [], {}, "unknown") for v in old[section].values()):
            result[section] = old[section]
    if old.get("correlation"):
        result["correlation"] = old["correlation"]
    return result


def merge_with_existing(fresh: list[dict[str, Any]], existing: list[dict[str, Any]], run_time: str) -> list[dict[str, Any]]:
    old_by_key = {x.get("processing", {}).get("deduplication_key"): x for x in existing if x.get("processing", {}).get("deduplication_key")}
    old_by_url = {}
    for old in existing:
        for source in old.get("sources", []):
            if source.get("url"):
                old_by_url[source["url"]] = old

    result = []
    consumed_old_ids = set()
    for record in fresh:
        old = old_by_key.get(record["processing"].get("deduplication_key"))
        if old is None:
            old = next((old_by_url.get(source.get("url")) for source in record["sources"] if source.get("url") in old_by_url), None)
        if old:
            record = preserve_review_fields(record, old)
            record["processing"]["updated_at"] = run_time
            consumed_old_ids.add(old.get("incident_id"))
        result.append(record)

    # Retain older records that are outside the current collection window.
    for old in existing:
        if old.get("incident_id") not in consumed_old_ids:
            result.append(old)
    return deduplicate(result)


def build_payload(incidents: list[dict[str, Any]], run_time: str, statistics: dict[str, Any]) -> dict[str, Any]:
    incidents.sort(key=lambda x: (x.get("dates", {}).get("published") or "", x.get("title") or ""), reverse=True)


    by_record_type = Counter(
        x.get("record_type") or "unknown"
        for x in incidents
    )

    by_type = Counter(
        x.get("event_type") or "not_applicable"
        for x in incidents
    )   

    by_status = Counter(
        x.get("status") or "unknown"
        for x in incidents
    )

    by_sector = Counter(
        sector
        for x in incidents
        for sector in x.get("sectors", [])
    )
    return {
        "metadata": {
            "schema_version": SCHEMA_VERSION,
            "script_version": SCRIPT_VERSION,
            "generated_at": run_time,
            "description": "Real-world cybersecurity incidents and cyber-physical events used as evidence for the NCRC Threat Register.",
            "record_count": len(incidents),
            "statistics": {
                **statistics,
                "records_by_type": dict(sorted(by_record_type.items())),
                "incidents_by_type": dict(sorted(by_type.items())),
                "incidents_by_status": dict(by_status.items()),
            },
        },
        "incidents": incidents,
    }


def validate_payload(payload: dict[str, Any]) -> list[str]:
    errors: list[str] = []
    if not isinstance(payload, dict):
        return ["Root must be a JSON object"]
    if not isinstance(payload.get("metadata"), dict):
        errors.append("metadata must be an object")
    incidents = payload.get("incidents")
    if not isinstance(incidents, list):
        errors.append("incidents must be a list")
        return errors
    seen_ids = set()
    for index, record in enumerate(incidents):
        prefix = f"incidents[{index}]"
        if not isinstance(record, dict):
            errors.append(f"{prefix} must be an object")
            continue
        for required in ["incident_id", "title", "status", "event_type", "sources", "processing"]:
            if required not in record:
                errors.append(f"{prefix}.{required} is required")
        incident_id = record.get("incident_id")
        if incident_id in seen_ids:
            errors.append(f"Duplicate incident_id: {incident_id}")
        seen_ids.add(incident_id)
        if record.get("status") not in {"confirmed", "reported", "suspected", "unverified"}:
            errors.append(f"{prefix}.status has invalid value {record.get('status')!r}")
        if not isinstance(record.get("sources"), list) or not record.get("sources"):
            errors.append(f"{prefix}.sources must contain at least one source")
        score = record.get("evidence", {}).get("confidence_score")
        if score is not None and not isinstance(score, (int, float)):
            errors.append(f"{prefix}.evidence.confidence_score must be numeric or null")
        elif isinstance(score, (int, float)) and not 0 <= score <= 1:
            errors.append(f"{prefix}.evidence.confidence_score must be between 0 and 1")
    count = payload.get("metadata", {}).get("record_count")
    if count is not None and count != len(incidents):
        errors.append(f"metadata.record_count={count} but incidents contains {len(incidents)} records")
    return errors


def atomic_write_json(path: Path, payload: dict[str, Any], pretty: bool = True, backup: bool = True) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    if backup and path.exists():
        backup_path = path.with_suffix(path.suffix + ".bak")
        shutil.copy2(path, backup_path)
    fd, temporary_name = tempfile.mkstemp(prefix=path.name + ".", suffix=".tmp", dir=str(path.parent))
    try:
        with os.fdopen(fd, "w", encoding="utf-8") as handle:
            json.dump(payload, handle, ensure_ascii=False, indent=2 if pretty else None, sort_keys=False)
            handle.write("\n")
            handle.flush()
            os.fsync(handle.fileno())
        os.replace(temporary_name, path)
    except Exception:
        try:
            os.unlink(temporary_name)
        except OSError:
            pass
        raise


def load_feeds(args: argparse.Namespace) -> list[dict[str, Any]]:
    feeds = [] if args.no_default_feeds else [dict(feed) for feed in DEFAULT_FEEDS if feed.get("enabled", True)]
    if args.feeds_file:
        with Path(args.feeds_file).open("r", encoding="utf-8") as handle:
            extra = json.load(handle)
        if isinstance(extra, dict):
            extra = extra.get("feeds", [])
        if not isinstance(extra, list):
            raise ValueError("--feeds-file must contain a list or an object with a 'feeds' list")
        feeds.extend(x for x in extra if isinstance(x, dict) and x.get("enabled", True))
    for number, url in enumerate(args.feed or [], start=1):
        feeds.append({
            "name": f"CLI feed {number}", "url": url, "publisher": urllib.parse.urlsplit(url).netloc,
            "source_type": "rss", "authority_level": "unknown", "enabled": True,
        })
    unique = {}
    for feed in feeds:
        if not feed.get("url"):
            continue
        unique[normalize_url(feed["url"])] = feed
    return list(unique.values())


def configure_logging(verbose: int) -> None:
    level = logging.WARNING
    if verbose == 1:
        level = logging.INFO
    elif verbose >= 2:
        level = logging.DEBUG
    logging.basicConfig(level=level, format="%(asctime)s %(levelname)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S")


def parse_args(argv: Optional[Sequence[str]] = None) -> argparse.Namespace:
    parser = argparse.ArgumentParser(description="Collect and normalize public real-world cybersecurity incidents")
    parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT, help=f"Output JSON path (default: {DEFAULT_OUTPUT})")
    parser.add_argument("--days", type=int, default=DEFAULT_DAYS, help=f"Include source items from the last N days; 0 disables filtering (default: {DEFAULT_DAYS})")
    parser.add_argument("--min-score", type=float, default=0.44, help="Minimum deterministic incident-selection score between 0 and 1 (default: 0.44)")
    parser.add_argument("--dedup-threshold", type=float, default=0.52, help="Title token similarity threshold for deduplication (default: 0.52)")
    parser.add_argument("--feed", action="append", help="Additional RSS/Atom feed URL; repeatable")
    parser.add_argument("--feeds-file", type=Path, help="JSON file containing additional feed definitions")
    parser.add_argument("--no-default-feeds", action="store_true", help="Do not use built-in feeds")
    parser.add_argument("--input-json", action="append", type=Path, help="Manual source-item JSON file; repeatable")
    parser.add_argument("--replace", action="store_true", help="Replace existing incidents instead of preserving old records")
    parser.add_argument("--validate-only", action="store_true", help="Validate the existing output and exit")
    parser.add_argument("--dry-run", action="store_true", help="Build and validate but do not write output")
    parser.add_argument("--pretty", action=argparse.BooleanOptionalAction, default=True, help="Pretty-print JSON")
    parser.add_argument("--backup", action=argparse.BooleanOptionalAction, default=True, help="Create .bak before replacing output")
    parser.add_argument("--timeout", type=int, default=DEFAULT_TIMEOUT, help=f"HTTP timeout in seconds (default: {DEFAULT_TIMEOUT})")
    parser.add_argument("--user-agent", default=DEFAULT_USER_AGENT, help="HTTP User-Agent")
    parser.add_argument("--strict", action="store_true", help="Abort if any feed cannot be collected")
    parser.add_argument("-v", "--verbose", action="count", default=0, help="Increase logging verbosity; repeat for debug")
    parser.add_argument("--version", action="version", version=f"%(prog)s {SCRIPT_VERSION}")
    args = parser.parse_args(argv)
    if not 0 <= args.min_score <= 1:
        parser.error("--min-score must be between 0 and 1")
    if not 0 <= args.dedup_threshold <= 1:
        parser.error("--dedup-threshold must be between 0 and 1")
    if args.days < 0:
        parser.error("--days cannot be negative")
    return args


def main(
    argv: Optional[Sequence[str]] = None,
) -> int:
    args = parse_args(argv)

    configure_logging(
        args.verbose,
    )

    # ---------------------------------------------------------
    # Validation-only mode
    # ---------------------------------------------------------

    if args.validate_only:
        if not args.output.exists():
            LOG.error(
                "Output does not exist: %s",
                args.output,
            )
            return 2

        with args.output.open(
            "r",
            encoding="utf-8",
        ) as handle:
            payload = json.load(handle)

        errors = validate_payload(
            payload,
        )

        if errors:
            for error in errors:
                LOG.error(
                    "Validation: %s",
                    error,
                )

            return 1

        print(
            f"Valid: {args.output} "
            f"({len(payload['incidents'])} records)"
        )

        return 0

    # ---------------------------------------------------------
    # Initialise collection run
    # ---------------------------------------------------------

    run_time = iso_z()

    feeds = load_feeds(
        args,
    )

    raw_items: list[RawItem] = []

    feed_failures: list[dict[str, str]] = []

    # ---------------------------------------------------------
    # RSS / Atom feeds
    # ---------------------------------------------------------

    for feed in feeds:
        try:
            items = collect_feed(
                feed,
                timeout=args.timeout,
                user_agent=args.user_agent,
            )

            raw_items.extend(
                items,
            )

        except Exception as exc:
            feed_failures.append(
                {
                    "feed": feed.get(
                        "name",
                        feed.get(
                            "url",
                            "unknown",
                        ),
                    ),
                    "error": str(exc),
                }
            )

            LOG.error(
                "Feed failed: %s",
                exc,
            )

            if args.strict:
                return 3

    # ---------------------------------------------------------
    # AIVD HTML sources
    # ---------------------------------------------------------
    #
    # AIVD is handled separately because these sources are
    # collected from the official overview pages rather than
    # through the normal RSS/Atom collector.
    # ---------------------------------------------------------

    enabled_aivd_sources = [
        source
        for source in AIVD_SOURCES
        if source.get(
            "enabled",
            False,
        )
    ]

    for source in enabled_aivd_sources:
        try:
            items = collect_aivd_source(
                source,
                timeout=args.timeout,
                user_agent=args.user_agent,
                days=args.days,
            )

            raw_items.extend(
                items,
            )

        except Exception as exc:
            feed_failures.append(
                {
                    "feed": source.get(
                        "name",
                        source.get(
                            "url",
                            "unknown",
                        ),
                    ),
                    "error": str(exc),
                }
            )

            LOG.error(
                "AIVD source failed: %s",
                exc,
            )

            if args.strict:
                return 3

    # ---------------------------------------------------------
    # Optional manually supplied JSON source items
    # ---------------------------------------------------------

    for input_path in args.input_json or []:
        try:
            manual = load_manual_json(
                input_path,
            )

            raw_items.extend(
                manual,
            )

            LOG.info(
                "Loaded %d manual source items from %s",
                len(manual),
                input_path,
            )

        except Exception as exc:
            LOG.error(
                "Manual input failed for %s: %s",
                input_path,
                exc,
            )

            if args.strict:
                return 3

    # ---------------------------------------------------------
    # Apply time window
    # ---------------------------------------------------------

    persistent_record_types = {
        "strategic_assessment",
        "threat_observation",
        "exploitation_activity",
    }

    recent_items = [
        item
        for item in raw_items
        if (
            clean_text(item.record_type).lower()
            in persistent_record_types
            or is_recent(
                item.published or item.updated,
                args.days,
            )
        )
    ]


    LOG.info(
        "Source items collected: %d",
        len(raw_items),
    )

    LOG.info(
        "Source items inside date window: %d",
        len(recent_items),
    )

    # ---------------------------------------------------------
    # Incident/intelligence selection
    # ---------------------------------------------------------

    candidates: list[dict[str, Any]] = []

    rejected: list[dict[str, Any]] = []

    for item in recent_items:
        gate_accepted, gate_reasons, gate_details = (
            evaluate_incident_gate(
                item,
            )
        )

        score, details = incident_score(
            item,
        )

        details["incident_gate"] = {
            "accepted": gate_accepted,
            "reasons": gate_reasons,
            **gate_details,
        }

        # -----------------------------------------------------
        # Gate rejection
        # -----------------------------------------------------

        if not gate_accepted:
            rejected.append(
                {
                     "score": score,
                    "title": item.title,
                    "reasons": gate_reasons,
                }
            )

            LOG.debug(
                "Rejected by gate %.3f [%s\]: %s",
                score,
                ", ".join(gate_reasons),
              item.title,
            )

            continue

        # -----------------------------------------------------
        # Score rejection
        # -----------------------------------------------------


        record_type = determine_record_type(
            item,
            details,
        )

        score_thresholds = {
            "incident": args.min_score,
            "strategic_assessment": 0.20,
            "threat_observation": 0.30,
            "exploitation_activity": 0.40,
        }

        required_score = score_thresholds.get(
            record_type,
            args.min_score,
        )

        if score < required_score:
            rejected.append(
                {
                    "score": score,
                    "title": item.title,
                    "record_type": record_type,
                    "required_score": required_score,
                    "reasons": [
                        "score_below_threshold",
                    ],
                }
            )

            LOG.debug(
                "Rejected by score %.3f < %.3f "
                "[%s]: %s",
                score,
                required_score,
                record_type,
                item.title,
            )

            continue

        candidates.append(
            build_record(
                item,
                score,
                details,
                run_time,
            )
        )
        

    # ---------------------------------------------------------
    # Deduplicate newly collected records
    # ---------------------------------------------------------

    fresh = deduplicate(
        candidates,
        threshold=args.dedup_threshold,
    )

    # ---------------------------------------------------------
    # Load existing records unless --replace was requested
    # ---------------------------------------------------------

    if args.replace:
        existing: list[dict[str, Any]] = []

    else:
        existing = load_existing(
            args.output,
        )

    # ---------------------------------------------------------
    # Merge current collection with existing intelligence
    # ---------------------------------------------------------

    if existing:
        incidents = merge_with_existing(
            fresh,
            existing,
            run_time,
        )

    else:
        incidents = fresh

    # ---------------------------------------------------------
    # Collection statistics
    # ---------------------------------------------------------

    statistics = {
        "feeds_configured": len(feeds),

        "aivd_sources_configured": len(
            enabled_aivd_sources
        ),

        "feed_failures": feed_failures,

        "source_items_collected": len(
            raw_items
        ),

        "source_items_in_date_window": len(
            recent_items
        ),

        "candidate_records_before_deduplication": len(
            candidates
        ),

        "rejected_source_items": len(
            rejected
        ),

        "fresh_incidents_after_deduplication": len(
            fresh
        ),

        "existing_incidents_loaded": len(
            existing
        ),
    }

    # ---------------------------------------------------------
    # Build final JSON structure
    # ---------------------------------------------------------

    payload = build_payload(
        incidents,
        run_time,
        statistics,
    )

    # ---------------------------------------------------------
    # Validate before writing
    # ---------------------------------------------------------

    errors = validate_payload(
        payload,
    )

    if errors:
        for error in errors:
            LOG.error(
                "Validation: %s",
                error,
            )

        return 1

    # ---------------------------------------------------------
    # Show metadata/statistics
    # ---------------------------------------------------------

    print(
        json.dumps(
            payload["metadata"],
            ensure_ascii=False,
            indent=2,
        )
    )

    # ---------------------------------------------------------
    # Dry-run
    # ---------------------------------------------------------

    if args.dry_run:
        LOG.warning(
            "Dry run: output not written"
        )

        return 0

    # ---------------------------------------------------------
    # Atomic write
    # ---------------------------------------------------------

    atomic_write_json(
        args.output,
        payload,
        pretty=args.pretty,
        backup=args.backup,
    )

    print(
        f"Wrote {args.output}: "
        f"{len(payload['incidents'])} records"
    )

    return 0


if __name__ == "__main__":
    raise SystemExit(
        main()
    )
