#!/usr/bin/env python3
"""
Correlate CVE, paper, incident, assessment and signal evidence with T-xxx threats.

Default inputs:
  data/threats.json
  data/taxonomy.json
  zero_days_threats/zero_day.json
  papers_threats/papers.json

Default outputs:
  evidence/threat_correlations.json
  evidence/evidence_index.json
  evidence/correlation_review.json
  evidence/correlation_run.json

The matcher is deterministic and explainable. It uses normalized lexical
similarity, phrase/feature overlap, threat-specific rules, and contradiction
penalties. No network access, external model, or embeddings are required.
"""

from __future__ import annotations

import argparse
import hashlib
import html
import json
import logging
import math
import os
import re
import sys
import tempfile
import time
import unicodedata
from collections import Counter, defaultdict
from dataclasses import asdict, dataclass, field
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable, Mapping, Sequence


SCHEMA_VERSION = "1.0"
LOGGER = logging.getLogger("correlate_evidence")
THREAT_ID_RE = re.compile(r"^T-\d{3}$", re.IGNORECASE)
CVE_ID_RE = re.compile(r"\bCVE-\d{4}-\d{4,7}\b", re.IGNORECASE)
URL_RE = re.compile(r"https?://[^\s\"'<>]+", re.IGNORECASE)
HTML_TAG_RE = re.compile(r"<[^>]+>")
TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9+.#/_-]*")

DEFAULT_PATHS = {
    "threats": Path("data/threads_register.json"),
    "taxonomy": Path("data/taxonomy.json"),
    "cves": Path("zero_day_threats/zero_day.json"),
    "papers": Path("papers_threats/papers.json"),
    "repositories": Path("github/github.json"),
    "incidents": Path("incidents/incidents.json"),
    "signals": Path("signals/signals.json"),
    "output_dir": Path("evidence"),
}

# Generic words that add little or no discriminative value.
STOPWORDS = {
    "a", "about", "across", "after", "against", "all", "also", "an", "and",
    "any", "are", "as", "at", "attack", "attacker", "attackers", "attacks",
    "automotive", "based", "be", "because", "been", "before", "being", "between",
    "both", "but", "by", "can", "could", "cyber", "cybersecurity", "data", "do",
    "during", "each", "for", "from", "further", "had", "has", "have", "having",
    "if", "in", "including", "into", "is", "it", "its", "may", "more", "most",
    "multiple", "new", "no", "not", "of", "on", "one", "or", "other", "our",
    "over", "paper", "possible", "provide", "risk", "security", "several", "should",
    "software", "some", "such", "system", "systems", "than", "that", "the", "their",
    "these", "they", "this", "through", "to", "under", "use", "used", "using",
    "vehicle", "vehicles", "via", "was", "we", "were", "which", "while", "with",
    "within", "would"
}

# Shared vocabulary used for both threat profiles and evidence records.
FEATURE_TERMS: dict[str, dict[str, list[str]]] = {
    "assets": {
        "ev_charging": [
            "ev charging", "electric vehicle charging", "charging infrastructure",
            "charging station", "charging stations", "charge point", "charge points",
            "charger", "chargers", "evse", "csms", "charging backend"
        ],
        "energy_grid": [
            "electricity grid", "energy grid", "power grid", "smart grid", "grid stability",
            "grid control", "energy management system", "load balancing"
        ],
        "backend_cloud": [
            "backend", "backend service", "cloud service", "cloud platform", "api gateway",
            "management server", "remote service", "fleet platform", "web service"
        ],
        "infotainment": [
            "infotainment", "head unit", "headunit", "ivi", "in-vehicle infotainment",
            "dilink", "telematics control unit", "tcu"
        ],
        "vehicle_ecu": [
            "ecu", "electronic control unit", "vehicle controller", "embedded controller",
            "gateway ecu", "central gateway"
        ],
        "autonomous_adas": [
            "autonomous driving", "automated driving", "adas", "automatic emergency braking",
            "aeb", "perception system", "path planning", "self-driving", "radar object",
            "lidar", "camera perception"
        ],
        "mobile_identity": [
            "mobile app", "mobile application", "customer account", "identity provider",
            "authentication service", "account recovery", "access token"
        ],
        "occupant_sensor": [
            "occupant monitoring", "cabin camera", "cabin sensor", "microphone",
            "biometric", "driver monitoring"
        ],
        "road_infrastructure": [
            "roadside unit", "rsu", "traffic management", "traffic light", "road infrastructure",
            "intelligent transport system", "its"
        ],
        "satellite_gnss": [
            "gnss", "gps", "galileo", "satellite navigation", "satellite communication"
        ],
        "update_system": [
            "software update", "firmware update", "ota update", "over-the-air update",
            "update server", "update process", "code signing"
        ],
        "supply_chain": [
            "supply chain", "supplier", "third-party component", "dependency", "dependencies",
            "open source component", "software bill of materials", "sbom"
        ],
        "vehicle_intelligence": [
            "modern vehicle", "modern vehicles", "moderne auto", "moderne voertuigen",
            "smart vehicle", "smart vehicles", "slimme voertuigen", "connected vehicle",
            "connected vehicles", "connected car", "vehicle data", "voertuigdata"
        ],
        "critical_infrastructure": [
            "critical infrastructure",
            "critical service",
            "critical services",
            "essential service",
            "essential services",
            "vital infrastructure",
            "vitale infrastructuur",
            "kritieke infrastructuur"
        ],
    },
    "protocols": {
        "ocpp": ["ocpp", "open charge point protocol"],
        "v2g": ["v2g", "vehicle-to-grid", "iso 15118"],
        "can": ["can bus", "controller area network", "can-fd", "can fd"],
        "uds": ["uds", "unified diagnostic services", "iso 14229"],
        "doip": ["doip", "diagnostics over internet protocol", "iso 13400"],
        "v2x": ["v2x", "c-v2x", "cv2x", "dsrc", "its-g5"],
        "bluetooth": ["bluetooth", "ble", "bluetooth low energy"],
        "wifi": ["wi-fi", "wifi", "802.11"],
        "cellular": ["cellular", "lte", "4g", "5g", "modem", "baseband"],
        "rke": ["rke", "remote keyless entry", "keyless entry", "key fob"],
        "nfc": ["nfc", "near field communication"],
        "ethernet": ["automotive ethernet", "ethernet", "some/ip", "someip"],
    },
    "attack_vectors": {
        "remote": [
            "remote attack", "remotely exploitable", "network attack", "internet exposed",
            "internet-accessible", "remote code execution", "rce"
        ],
        "authentication": [
            "authentication bypass", "bypass authentication", "brute force", "bruteforce",
            "weak credentials", "default credentials", "account takeover", "broken authentication"
        ],
        "physical": [
            "physical access", "usb", "obd", "obd-ii", "diagnostic port", "removable media"
        ],
        "wireless": [
            "wireless", "radio", "rf", "bluetooth", "wifi", "cellular", "keyless",
            "signal injection", "jamming", "spoofing"
        ],
        "supply_chain": [
            "supply chain attack", "dependency confusion", "malicious update", "third-party compromise",
            "compromised supplier"
        ],
        "privilege": [
            "privilege escalation", "unauthorized access", "access control", "improper authorization"
        ],
        "injection": [
            "injection", "command injection", "sql injection", "object injection",
            "adversarial example", "phantom object"
        ],
        "memory_corruption": [
            "buffer overflow", "heap overflow", "use after free", "out-of-bounds", "memory corruption"
        ],
        "denial_of_service": [
            "denial of service", "dos", "resource exhaustion", "service disruption", "crash"
        ],
        "ai_assisted": [
            "ai-assisted",
            "ai assisted",
            "ai-powered",
            "ai powered",
            "artificial intelligence",
            "machine learning",
            "large language model",
            "llm",
            "automated exploitation",
            "ai exploitation",
            "ai-enabled attack",
            "ai enabled attack"
        ],
    },
    "impacts": {
        "vehicle_control": [
            "vehicle control", "steering", "braking", "acceleration", "unsafe behavior",
            "unsafe behaviour", "loss of control", "control vehicle functions"
        ],
        "availability": [
            "availability", "outage", "unavailable", "service disruption", "denial of service",
            "immobilize", "immobilise", "disable"
        ],
        "privacy": [
            "personal data", "occupant data", "privacy", "surveillance", "tracking",
            "location data", "behavioral data", "behavioural data"
        ],
        "espionage": [
            "espionage", "intelligence gathering", "eavesdropping", "secretly collect",
            "sensitive information"
        ],
        "theft": [
            "vehicle theft", "steal vehicle", "unlock vehicle", "key cloning", "relay attack",
            "replay attack", "rolljam"
        ],
        "grid_disruption": [
            "grid destabilization", "grid destabilisation", "grid disruption", "load manipulation",
            "power outage", "grid-control signals"
        ],
        "fleet_scale": [
            "fleet-wide", "fleet wide", "large scale", "large-scale", "multiple fleets",
            "many vehicles", "mass compromise", "shared dependency"
        ],
        "road_safety": [
            "road safety", "collision", "crash", "fatality", "fatalities", "serious injury",
            "emergency braking", "unsafe driving"
        ],
        "integrity": [
            "integrity", "manipulation", "tampering", "unauthorized modification",
            "false data", "malicious command"
        ],
        "systemic_disruption": [
            "critical infrastructure disruption",
            "critical service disruption",
            "essential service disruption",
            "societal disruption",
            "large-scale disruption",
            "large scale disruption",
            "systemic disruption"
        ],
    },
    "lifecycle": {
        "development_testing": [
            "secure development", "testing", "test coverage", "verification", "validation",
            "fuzzing", "type approval"
        ],
        "monitoring_detection": [
            "monitoring", "detection", "telemetry", "logging", "incident response",
            "anomaly detection", "honeypot", "log correlation"
        ],
        "patching_support": [
            "patching", "security update", "unsupported", "end of support", "end-of-life",
            "vulnerability remediation"
        ],
        "disposal_transfer": [
            "disposal", "sale", "rental", "recycling", "factory reset", "data deletion"
        ],
        "governance": [
            "regulation", "regulatory", "oversight", "governance", "compliance",
            "r155", "r156", "iso/sae 21434", "cyber resilience act"
        ],
    },
}

# Exact or near-exact technical indicators. When present, these can strongly
# raise a specific threat while still allowing related secondary matches.
THREAT_RULES: list[dict[str, Any]] = [
    {
        "name": "ev_charging_general",
        "requires_any": ["ocpp", "evse", "charge point", "charging station", "electric vehicle charging", "csms"],
        "boost": {"T-002": 0.38},
    },
    {
        "name": "v2g_grid_manipulation",
        "requires_any": ["v2g", "vehicle-to-grid", "grid stability", "load manipulation", "aggregator"],
        "boost": {"T-004": 0.25, "T-003": 0.10, "T-002": 0.05},
    },
    {
        "name": "grid_attack_path",
        "requires_any": ["grid destabilization", "grid destabilisation", "grid-control", "energy management system", "power grid"],
        "boost": {"T-003": 0.18, "T-004": 0.14},
    },
    {
        "name": "infotainment_headunit",
        "requires_any": ["infotainment", "head unit", "headunit", "ivi", "dilink"],
        "boost": {"T-023": 0.27, "T-047": 0.08, "T-064": 0.04},
    },
    {
        "name": "gnss_jamming_spoofing",
        "requires_any": ["gnss jamming", "gnss spoofing", "gps jamming", "gps spoofing"],
        "boost": {"T-035": 0.28, "T-066": 0.10, "T-054": 0.04},
    },
    {
        "name": "perception_manipulation",
        "requires_any": ["phantom object", "radar object injection", "sensor spoofing", "adversarial example", "perception attack"],
        "boost": {"T-066": 0.27, "T-012": 0.12, "T-013": 0.07, "T-010": 0.06},
    },
    {
        "name": "digital_vehicle_theft",
        "requires_any": ["vehicle theft", "keyless entry", "relay attack", "rolljam", "key cloning", "unlock vehicle"],
        "boost": {"T-044": 0.28, "T-043": 0.07},
    },
    {
        "name": "vehicle_espionage_assessment",
        "requires_any": [
            "digital espionage", "vehicle espionage", "espionage", "spionage",
            "spionagerisico", "cyberspionage", "intelligence gathering",
            "modern vehicle", "modern vehicles", "moderne auto", "moderne voertuigen",
            "smart vehicle", "smart vehicles", "slimme voertuigen", "connected vehicle",
            "connected vehicles", "vehicle data", "voertuigdata", "data over inzittenden",
            "national security", "nationale veiligheid", "state actor", "statelijke actor"
        ],
        "boost": {"T-017": 0.30, "T-018": 0.24},
    },

    {
        "name": "vehicle_architecture_difference_assessment",
        "requires_any": [
            "vehicle architecture",
            "vehicle architectures",
            "different vehicle architectures",
            "materially different vehicle architectures",
            "architecture segmentation",
            "critical functionality was effectively isolated",
            "critical functionality is online",
            "critical vehicle functionality was online",
            "isolated critical functions",
            "connected critical functions",
            "system integration",
        ],
        "boost": {
            "T-051": 0.34,
            "T-026": 0.08,
        },
    },
    {
        "name": "remote_oem_critical_function_access",
        "requires_any": [
            "remote oem",
            "remote oem access",
            "manufacturer access",
            "manufacturer diagnostics",
            "direct digital access",
            "remote diagnostics",
            "critical functionality",
            "critical vehicle functionality",
            "battery management",
            "power supply",
            "remote software control",
            "remotely change",
            "remotely disable",
            "remote disable",
            "kill switch",
        ],
        "boost": {
            "T-047": 0.28,
            "T-058": 0.10,
            "T-001": 0.06,
        },
    },
    {
        "name": "vehicle_ota_lifecycle_control",
        "requires_any": [
            "ota update",
            "ota updates",
            "ota software update",
            "ota software updates",
            "over-the-air update",
            "over-the-air updates",
            "software update",
            "software updates",
            "post-sale software",
            "post-sale modification",
        ],
        "boost": {
            "T-060": 0.30,
            "T-047": 0.12,
            "T-026": 0.05,
        },
    },


    {
        "name": "vehicle_sensor_supplier_espionage",
        "requires_any": [
            "vehicle sensor", "vehicle sensors", "sensor data", "cabin camera", "microphone",
            "supplier", "leverancier", "foreign supplier", "buitenlandse leverancier",
            "foreign manufacturer", "chinese vehicles", "chinese vehicle", "chinese manufacturer"
        ],
        "boost": {"T-018": 0.22, "T-017": 0.08},
    },
    {
        "name": "vehicle_tracking_privacy",
        "requires_any": ["location tracking", "movement data", "location data", "driver tracking"],
        "boost": {"T-016": 0.19, "T-040": 0.23, "T-036": 0.05},
    },
    {
        "name": "occupant_monitoring",
        "requires_any": ["occupant monitoring", "cabin camera", "driver monitoring", "cabin microphone"],
        "boost": {"T-015": 0.22, "T-036": 0.12, "T-041": 0.08},
    },
    {
        "name": "vehicle_update_compromise",
        "requires_any": ["ota update", "over-the-air update", "update server", "firmware update", "malicious update", "code signing"],
        "boost": {"T-060": 0.24, "T-033": 0.06, "T-047": 0.05},
    },
    {
        "name": "vehicle_communications",
        "requires_any": ["v2x", "c-v2x", "dsrc", "automotive ethernet", "cellular modem", "telematics"],
        "boost": {"T-059": 0.17, "T-054": 0.08, "T-047": 0.04},
    },
    {
        "name": "external_interfaces",
        "requires_any": ["usb", "obd", "obd-ii", "diagnostic port", "external interface"],
        "boost": {"T-042": 0.18, "T-062": 0.19, "T-064": 0.06},
    },
    {
        "name": "identity_keys_trust",
        "requires_any": ["cryptographic key","private key","hardcoded key","hard-coded key","certificate validation",
            "certificate verification","trust anchor","digital certificate","credential storage", "credential exposure",
            "impersonation" ],
        "boost": {"T-065": 0.16, "T-063": 0.09},
    },
    {
        "name": "monitoring_detection",
        "requires_any": ["anomaly detection", "honeypot", "logging", "telemetry", "incident response", "log correlation"],
        "boost": {"T-034": 0.12, "T-057": 0.19},
    },
    {
        "name": "supply_chain_compromise",
        "requires_any": ["supply chain attack", "compromised supplier", "third-party component", "dependency compromise"],
        "boost": {"T-033": 0.23, "T-048": 0.10, "T-038": 0.10, "T-062": 0.05},
    },
    {
        "name": "backend_services",
        "requires_any": ["backend service", "cloud platform", "management server", "fleet platform", "api gateway"],
        "boost": {"T-058": 0.18, "T-067": 0.08, "T-011": 0.05},
    },
]

# Minimum contextual requirements for broad scenarios. A missing requirement
# creates a penalty rather than an automatic rejection.
THREAT_CONTEXT_REQUIREMENTS: dict[str, list[str]] = {
    "T-001": ["fleet", "large-scale", "multiple vehicles", "shared dependency", "mobility service"],
    "T-003": ["grid", "energy management", "power grid", "grid-control", "electricity"],
    "T-004": ["v2g", "vehicle-to-grid", "aggregator", "grid stability", "load manipulation"],
    "T-005": ["sabotage", "targeted attack", "physical damage", "malicious control"],
    "T-006": ["critical infrastructure", "vip", "national security", "protected transport"],
    "T-007": ["cyber-physical", "moving weapon", "physical attack", "remote control"],
    "T-009": ["botnet", "distributed denial", "malicious traffic", "many endpoints"],
    "T-010": ["road safety", "collision", "unsafe driving", "braking", "steering"],
    "T-012": ["autonomous", "automated driving", "adas", "perception", "path planning"],
    "T-013": ["autonomous", "automated driving", "self-driving"],
    "T-017": [
        "espionage", "spionage", "spionagerisico", "cyberspionage", "intelligence",
        "secret collection", "surveillance", "moderne voertuigen", "slimme voertuigen",
        "connected vehicles", "vehicle data", "voertuigdata", "national security",
        "nationale veiligheid", "state actor", "statelijke actor"
    ],
    "T-018": [
        "sensor", "sensor data", "supplier", "leverancier", "foreign supplier",
        "buitenlandse leverancier", "espionage", "spionage", "spionagerisico",
        "intelligence", "moderne voertuigen", "slimme voertuigen", "connected vehicles",
        "vehicle data", "voertuigdata"
    ],
    "T-020": ["societal disruption", "population", "public services", "large-scale"],
    "T-021": ["trust", "public confidence", "repeated incident"],
    "T-028": ["cyber resilience act", "cra", "regulatory uncertainty"],
    "T-029": ["regulation", "legal obligation", "regulatory uncertainty"],
    "T-030": ["oem support", "end of support", "unsupported", "manufacturer insolvency"],
    "T-031": ["supplier failure", "supplier insolvency", "critical supplier", "service withdrawal"],
    "T-037": ["ai-assisted", "ai assisted", "ai-powered", "aai powered","artificial intelligence", "large language model", 
    "machine learning", "automated exploitation"],
    "T-049": ["underestimation", "preparedness", "continuity plan", "fleet-scale"],
    "T-055": ["satellite communication", "foreign", "untrusted party", "sovereignty"],
    "T-056": ["financial health", "insolvency", "bankruptcy", "reduced support"],
    "T-069": ["shared dependency", "single point of failure", "multiple fleets", "multiple providers"],
}


@dataclass
class Threat:
    threat_id: str
    title: str = ""
    title_nl: str = ""
    category: str = ""
    source: str = ""
    primary_domain: str = ""
    secondary_domains: list[str] = field(default_factory=list)
    tags: list[str] = field(default_factory=list)
    terms: list[str] = field(default_factory=list)
    surfaces: list[str] = field(default_factory=list)
    evidence_types: list[str] = field(default_factory=list)
    paper_queries: list[str] = field(default_factory=list)
    signal_queries: list[str] = field(default_factory=list)
    description: str = ""
    domain_rationale: str = ""
    impact_rationale: str = ""
    likelihood_rationale: str = ""
    risk_rationale: str = ""
    comments: str = ""
    additional_comments: str = ""
    risk_rating: str = ""
    search_text: str = ""
    tokens: set[str] = field(default_factory=set)
    features: dict[str, set[str]] = field(default_factory=dict)


@dataclass
class Evidence:
    evidence_id: str
    evidence_type: str
    title: str
    description: str
    published: str = ""
    url: str = ""
    severity: str = ""
    cvss: float | None = None
    cvss_version: str = ""
    pending_analysis: bool = False
    authors: list[str] = field(default_factory=list)
    venue: str = ""
    year: int | None = None
    doi: str = ""
    sources: list[str] = field(default_factory=list)
    matched_terms: list[str] = field(default_factory=list)
    topics: list[str] = field(default_factory=list)
    vendors: list[str] = field(default_factory=list)
    surfaces: list[str] = field(default_factory=list)
    linked_cves: list[str] = field(default_factory=list)
    cited_by: int = 0
    open_access: bool = False
    search_text: str = ""
    tokens: set[str] = field(default_factory=set)
    features: dict[str, set[str]] = field(default_factory=dict)
    raw_record: dict[str, Any] = field(default_factory=dict, repr=False)


@dataclass
class ScoreBreakdown:
    lexical: float = 0.0
    title: float = 0.0
    features: float = 0.0
    assets: float = 0.0
    protocols: float = 0.0
    attack_vectors: float = 0.0
    impacts: float = 0.0
    lifecycle: float = 0.0
    domain: float = 0.0
    rule_bonus: float = 0.0
    context_penalty: float = 0.0
    generic_penalty: float = 0.0
    final: float = 0.0


@dataclass
class MatchResult:
    evidence_id: str
    evidence_type: str
    threat_id: str
    relevance_score: float
    evidence_strength_score: float
    evidence_strength: str
    confidence: str
    status: str
    title: str
    url: str
    published: str
    matched_axes: list[str]
    matched_features: dict[str, list[str]]
    matched_terms: list[str]
    rules_fired: list[str]
    explanation: str
    score_breakdown: dict[str, float]


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description="Correlate CVEs, papers, repositories, incidents, assessments and signals to T-xxx threats using explainable deterministic matching."
    )
    parser.add_argument("--threats", type=Path, default=DEFAULT_PATHS["threats"])
    parser.add_argument("--taxonomy", type=Path, default=DEFAULT_PATHS["taxonomy"])
    parser.add_argument("--cves", type=Path, default=DEFAULT_PATHS["cves"])
    parser.add_argument("--papers", type=Path, default=DEFAULT_PATHS["papers"])
    parser.add_argument(
        "--repositories",
        type=Path,
        default=DEFAULT_PATHS["repositories"],
    )
    parser.add_argument("--incidents", type=Path, default=DEFAULT_PATHS["incidents"])
    parser.add_argument("--signals", type=Path, default=DEFAULT_PATHS["signals"])
    

    parser.add_argument("--output-dir", type=Path, default=DEFAULT_PATHS["output_dir"])
    parser.add_argument("--accept-threshold", type=float, default=0.58)
    parser.add_argument("--review-threshold", type=float, default=0.38)
    parser.add_argument("--top-k", type=int, default=10)
    parser.add_argument("--max-accepted-per-evidence", type=int, default=5)
    parser.add_argument("--threat-id", help="Only process one threat, e.g. T-023")
    parser.add_argument("--evidence-id", help="Only process one evidence ID")
    parser.add_argument("--validate-only", action="store_true")
    parser.add_argument("--include-rejected", action="store_true")
    parser.add_argument("--explain", action="store_true")
    parser.add_argument("--force", action="store_true", help="Overwrite existing output files")
    parser.add_argument("--log-level", choices=["DEBUG", "INFO", "WARNING", "ERROR"], default="INFO")
    args = parser.parse_args()

    if not 0.0 <= args.review_threshold <= 1.0:
        parser.error("--review-threshold must be between 0 and 1")
    if not 0.0 <= args.accept_threshold <= 1.0:
        parser.error("--accept-threshold must be between 0 and 1")
    if args.review_threshold >= args.accept_threshold:
        parser.error("--review-threshold must be lower than --accept-threshold")
    if args.top_k < 1:
        parser.error("--top-k must be at least 1")
    if args.max_accepted_per_evidence < 1:
        parser.error("--max-accepted-per-evidence must be at least 1")
    if args.threat_id:
        args.threat_id = args.threat_id.upper()
        if not THREAT_ID_RE.match(args.threat_id):
            parser.error("--threat-id must match T-xxx")
    return args


def configure_logging(level: str) -> None:
    logging.basicConfig(
        level=getattr(logging, level),
        format="%(asctime)s %(levelname)-8s %(message)s",
        datefmt="%Y-%m-%d %H:%M:%S",
    )


def normalize_whitespace(value: Any) -> str:
    if value is None:
        return ""
    text = html.unescape(str(value))
    text = unicodedata.normalize("NFKC", text)
    text = text.replace("\xa0", " ")
    text = re.sub(r"\s+", " ", text)
    return text.strip()


def normalize_text(value: Any) -> str:
    text = normalize_whitespace(value).lower()
    text = text.replace("vehicle to grid", "vehicle-to-grid")
    text = re.sub(r"[^a-z0-9+.#/_\- ]+", " ", text)
    return re.sub(r"\s+", " ", text).strip()


def extract_url(value: Any) -> str:
    text = html.unescape(str(value or ""))
    match = URL_RE.search(text)
    if match:
        return match.group(0).rstrip(".,);]")
    clean = HTML_TAG_RE.sub(" ", text)
    clean = normalize_whitespace(clean)
    return clean if clean.startswith(("http://", "https://")) else ""


def listify(value: Any, separators: str = r"[;,]") -> list[str]:
    if value is None:
        return []
    if isinstance(value, list):
        items = value
    elif isinstance(value, (tuple, set)):
        items = list(value)
    elif isinstance(value, str):
        items = re.split(separators, value)
    else:
        items = [value]
    result: list[str] = []
    seen: set[str] = set()
    for item in items:
        cleaned = normalize_whitespace(item)
        key = cleaned.casefold()
        if cleaned and key not in seen:
            result.append(cleaned)
            seen.add(key)
    return result


def join_nonempty(values: Iterable[Any]) -> str:
    return "\n".join(normalize_whitespace(v) for v in values if normalize_whitespace(v))


def tokenize(text: str) -> set[str]:
    normalized = normalize_text(text)
    return {
        token for token in TOKEN_RE.findall(normalized)
        if len(token) >= 3 and token not in STOPWORDS and not token.isdigit()
    }


def contains_phrase(text: str, phrase: str) -> bool:
    t = f" {normalize_text(text)} "
    p = normalize_text(phrase)
    if not p:
        return False
    if len(p) <= 3 and p.isalnum():
        return re.search(rf"(?<![a-z0-9]){re.escape(p)}(?![a-z0-9])", t) is not None
    return p in t


def has_any(text: str, phrases: Sequence[str]) -> bool:
    return any(contains_phrase(text, phrase) for phrase in phrases)


def extract_features(text: str) -> dict[str, set[str]]:
    features: dict[str, set[str]] = {axis: set() for axis in FEATURE_TERMS}
    for axis, labels in FEATURE_TERMS.items():
        for label, phrases in labels.items():
            if has_any(text, phrases):
                features[axis].add(label)
    return features


def safe_float(value: Any) -> float | None:
    try:
        if value in (None, ""):
            return None
        return float(value)
    except (TypeError, ValueError):
        return None


def safe_int(value: Any) -> int | None:
    try:
        if value in (None, ""):
            return None
        return int(value)
    except (TypeError, ValueError):
        return None


def load_json(path: Path) -> Any:
    if not path.exists():
        raise FileNotFoundError(f"Input file not found: {path}")
    try:
        with path.open("r", encoding="utf-8") as handle:
            return json.load(handle)
    except json.JSONDecodeError as exc:
        raise ValueError(f"Invalid JSON in {path}: line {exc.lineno}, column {exc.colno}: {exc.msg}") from exc


def records_from_json(data: Any, preferred_keys: Sequence[str]) -> list[dict[str, Any]]:
    if isinstance(data, list):
        return [item for item in data if isinstance(item, dict)]
    if isinstance(data, dict):
        for key in preferred_keys:
            value = data.get(key)
            if isinstance(value, list):
                return [item for item in value if isinstance(item, dict)]
        # Last-resort support for a dict keyed by record ID.
        if data and all(isinstance(v, dict) for v in data.values()):
            return [dict(v, id=v.get("id", k)) for k, v in data.items()]
    raise ValueError(f"Could not find records; expected a list or one of keys: {', '.join(preferred_keys)}")


def merge_text(existing: str, incoming: str) -> str:
    a = normalize_whitespace(existing)
    b = normalize_whitespace(incoming)
    if not a:
        return b
    if not b or b.casefold() == a.casefold():
        return a
    if b.casefold() in a.casefold():
        return a
    if a.casefold() in b.casefold():
        return b
    return f"{a}\n{b}"


def merge_unique(existing: list[str], incoming: Iterable[str]) -> list[str]:
    result = list(existing)
    seen = {item.casefold() for item in result}
    for item in incoming:
        cleaned = normalize_whitespace(item)
        if cleaned and cleaned.casefold() not in seen:
            result.append(cleaned)
            seen.add(cleaned.casefold())
    return result


def load_threats(threats_path: Path, taxonomy_path: Path) -> list[Threat]:
    base_data = load_json(threats_path)
    taxonomy_data = load_json(taxonomy_path)
    base_records = records_from_json(base_data, ("threats", "records"))
    taxonomy_records = records_from_json(taxonomy_data, ("taxonomy", "threats", "records"))

    merged: dict[str, Threat] = {}

    for record in base_records:
        threat_id = normalize_whitespace(record.get("id")).upper()
        if not THREAT_ID_RE.match(threat_id):
            continue
        merged[threat_id] = Threat(
            threat_id=threat_id,
            title=normalize_whitespace(record.get("title")),
            title_nl=normalize_whitespace(record.get("title_nl")),
            category=normalize_whitespace(record.get("category")),
            primary_domain=normalize_whitespace(record.get("primary_domain")),
            secondary_domains=listify(record.get("secondary_domain") or record.get("secondary_domains")),
            tags=listify(record.get("tags")),
            terms=listify(record.get("terms")),
            surfaces=listify(record.get("surfaces")),
            evidence_types=[normalize_text(value) for value in listify(record.get("evidence_types"))],
            paper_queries=listify(record.get("paper_queries")),
            signal_queries=listify(record.get("signal_queries")),
            domain_rationale=normalize_whitespace(record.get("domain_rationale")),
            impact_rationale=normalize_whitespace(record.get("impact_rationale")),
            likelihood_rationale=normalize_whitespace(record.get("likelihood_rationale")),
            risk_rationale=normalize_whitespace(record.get("risk_rationale")),
            risk_rating=normalize_whitespace(record.get("risk_level")),
        )

    for record in taxonomy_records:
        threat_id = normalize_whitespace(record.get("id")).upper()
        if not THREAT_ID_RE.match(threat_id):
            continue
        threat = merged.setdefault(threat_id, Threat(threat_id=threat_id))
        threat.title = merge_text(threat.title, record.get("title", ""))
        threat.category = merge_text(threat.category, record.get("category", ""))
        threat.source = merge_text(threat.source, record.get("source", ""))
        threat.primary_domain = merge_text(threat.primary_domain, record.get("primary_domain", ""))
        threat.secondary_domains = merge_unique(
            threat.secondary_domains, listify(record.get("secondary_domains") or record.get("secondary_domain"))
        )
        threat.tags = merge_unique(threat.tags, listify(record.get("tags")))
        threat.terms = merge_unique(threat.terms, listify(record.get("terms")))
        threat.surfaces = merge_unique(threat.surfaces, listify(record.get("surfaces")))
        threat.evidence_types = merge_unique(
            threat.evidence_types,
            [normalize_text(value) for value in listify(record.get("evidence_types"))],
        )
        threat.paper_queries = merge_unique(threat.paper_queries, listify(record.get("paper_queries")))
        threat.signal_queries = merge_unique(threat.signal_queries, listify(record.get("signal_queries")))
        threat.description = merge_text(threat.description, record.get("description", ""))
        threat.impact_rationale = merge_text(threat.impact_rationale, record.get("impact_rationale", ""))
        threat.likelihood_rationale = merge_text(threat.likelihood_rationale, record.get("likelihood_rationale", ""))
        threat.comments = merge_text(threat.comments, record.get("comments", ""))
        threat.additional_comments = merge_text(threat.additional_comments, record.get("additional_comments", ""))
        threat.risk_rating = merge_text(
            threat.risk_rating,
            record.get("combined_rating") or record.get("risk_rating") or "",
        )

        # Impact dimensions carry useful scenario-specific vocabulary.
        dimensions = join_nonempty([
            record.get("physical_safety"),
            record.get("economic_impact"),
            record.get("social_impact"),
            record.get("territorial_impact"),
        ])
        threat.comments = merge_text(threat.comments, dimensions)

    for threat in merged.values():
        threat.search_text = join_nonempty([
            threat.title,
            threat.title_nl,
            threat.category,
            threat.primary_domain,
            " ".join(threat.secondary_domains),
            " ".join(threat.tags),
            " ".join(threat.terms),
            " ".join(threat.surfaces),
            " ".join(threat.paper_queries),
            " ".join(threat.signal_queries),
            threat.description,
            threat.domain_rationale,
            threat.impact_rationale,
            threat.likelihood_rationale,
            threat.risk_rationale,
            threat.comments,
            threat.additional_comments,
        ])
        threat.tokens = tokenize(threat.search_text)
        threat.features = extract_features(threat.search_text)

    threats = sorted(merged.values(), key=lambda t: int(t.threat_id.split("-")[1]))
    if not threats:
        raise ValueError("No valid T-xxx threats found in threats/taxonomy input")
    return threats


def normalize_cve(record: dict[str, Any]) -> Evidence | None:
    evidence_id = normalize_whitespace(record.get("name") or record.get("id") or record.get("cve_id")).upper()
    if not evidence_id:
        return None
    vulnerability_texts: list[str] = []
    for item in record.get("vulnerabilities", []) or []:
        if isinstance(item, dict):
            text = normalize_whitespace(item.get("text") or item.get("description"))
        else:
            text = normalize_whitespace(item)
        if text:
            vulnerability_texts.append(text)
    description = " ".join(vulnerability_texts)
    matched_terms = listify(record.get("matched_terms"))
    keywords = listify(record.get("keywords"))
    keyword = normalize_whitespace(record.get("keyword"))
    if keyword:
        keywords = merge_unique(keywords, [keyword])
    surfaces = listify(record.get("surfaces"))
    url = extract_url(record.get("url"))
    search_text = join_nonempty([
        evidence_id,
        description,
        " ".join(keywords),
        " ".join(matched_terms),
        " ".join(surfaces),
    ])
    cves = CVE_ID_RE.findall(search_text)
    return Evidence(
        evidence_id=evidence_id,
        evidence_type="cve",
        title=evidence_id,
        description=description,
        published=normalize_whitespace(record.get("published")),
        url=url,
        severity=normalize_whitespace(record.get("score")),
        cvss=safe_float(record.get("cvss")),
        cvss_version=normalize_whitespace(record.get("cvss_version")),
        pending_analysis=bool(record.get("pending_analysis", False)),
        matched_terms=merge_unique(matched_terms, keywords),
        surfaces=surfaces,
        linked_cves=merge_unique([], cves),
        search_text=search_text,
        tokens=tokenize(search_text),
        features=extract_features(search_text),
        raw_record=record,
    )


def normalize_paper(record: dict[str, Any]) -> Evidence | None:
    evidence_id = normalize_whitespace(
        record.get("id") or record.get("doi") or record.get("arxiv_id") or record.get("title")
    )
    if not evidence_id:
        return None
    title = normalize_whitespace(record.get("title"))
    abstract = normalize_whitespace(record.get("abstract"))
    matched_terms = listify(record.get("matched_terms"))
    topics = listify(record.get("topics"))
    vendors = listify(record.get("vendors"))
    explicit_cves = merge_unique(
        listify(record.get("cves")),
        listify(record.get("linked_cves")),
    )
    explicit_cves = merge_unique(explicit_cves, [m.upper() for m in CVE_ID_RE.findall(f"{title} {abstract}")])
    search_text = join_nonempty([
        title,
        abstract,
        " ".join(matched_terms),
        " ".join(topics),
        " ".join(vendors),
        " ".join(explicit_cves),
        record.get("venue"),
    ])
    return Evidence(
        evidence_id=evidence_id,
        evidence_type="paper",
        title=title,
        description=abstract,
        published=normalize_whitespace(record.get("date") or record.get("year")),
        url=extract_url(record.get("url") or record.get("pdf_url")),
        authors=listify(record.get("authors")),
        venue=normalize_whitespace(record.get("venue")),
        year=safe_int(record.get("year")),
        doi=normalize_whitespace(record.get("doi")),
        sources=listify(record.get("sources")),
        matched_terms=matched_terms,
        topics=topics,
        vendors=vendors,
        linked_cves=explicit_cves,
        cited_by=safe_int(record.get("cited_by")) or 0,
        open_access=bool(record.get("open_access", False)),
        search_text=search_text,
        tokens=tokenize(search_text),
        features=extract_features(search_text),
        raw_record=record,
    )


def normalize_repository(
    record: dict[str, Any],
) -> Evidence | None:

    evidence_id = normalize_whitespace(
        record.get("id")
        or (
            f"github:{record.get('full_name')}"
            if record.get("full_name")
            else ""
        )
        or record.get("url")
    )

    if not evidence_id:
        return None

    full_name = normalize_whitespace(
        record.get("full_name")
    )

    title = normalize_whitespace(
        record.get("title")
        or record.get("name")
        or full_name
        or evidence_id
    )

    description = normalize_whitespace(
        record.get("description")
    )

    readme = normalize_whitespace(
        record.get("readme")
    )

    topics = listify(
        record.get("topics")
    )

    classification = record.get("classification")
    if not isinstance(classification, dict):
        classification = {}

    relevance = record.get("relevance")
    if not isinstance(relevance, dict):
        relevance = {}

    discovery = record.get("discovery")
    if not isinstance(discovery, dict):
        discovery = {}

    matched_terms: list[str] = []

    matched_terms = merge_unique(
        matched_terms,
        listify(
            classification.get(
                "automotive_term_hits"
            )
        ),
    )

    matched_terms = merge_unique(
        matched_terms,
        listify(
            classification.get(
                "ev_charging_term_hits"
            )
        ),
    )

    matched_terms = merge_unique(
        matched_terms,
        listify(
            classification.get(
                "security_term_hits"
            )
        ),
    )

    matched_terms = merge_unique(
        matched_terms,
        listify(
            discovery.get(
                "matched_queries"
            )
        ),
    )

    explicit_cves = [
        match.upper()
        for match in CVE_ID_RE.findall(
            join_nonempty([
                title,
                description,
                readme,
            ])
        )
    ]

    search_text = join_nonempty([
        evidence_id,
        full_name,
        title,
        description,
        readme,
        " ".join(topics),
        " ".join(matched_terms),
        " ".join(explicit_cves),
        record.get("language"),
        record.get("license"),
    ])

    return Evidence(
        evidence_id=evidence_id,
        evidence_type="repository",
        title=title,
        description=description,
        published=normalize_whitespace(
            record.get("pushed_at")
            or record.get("updated_at")
            or record.get("created_at")
        ),
        url=extract_url(
            record.get("url")
        ),
        sources=["GitHub"],
        matched_terms=matched_terms,
        topics=topics,
        linked_cves=merge_unique(
            [],
            explicit_cves,
        ),
        search_text=search_text,
        tokens=tokenize(search_text),
        features=extract_features(
            search_text
        ),
        raw_record=record,
    )

def nested_dict(
    record: Mapping[str, Any],
    key: str,
) -> dict[str, Any]:
    value = record.get(key)

    if isinstance(value, dict):
        return value

    return {}


def collect_source_metadata(
    record: Mapping[str, Any],
) -> tuple[list[str], list[str]]:
    source_names: list[str] = []
    source_urls: list[str] = []

    sources = record.get("sources") or []

    if isinstance(sources, dict):
        sources = [sources]

    if not isinstance(sources, list):
        sources = []

    for source in sources:
        if isinstance(source, str):
            cleaned = normalize_whitespace(
                source
            )

            if cleaned.startswith(
                ("http://", "https://")
            ):
                source_urls.append(
                    cleaned
                )
            elif cleaned:
                source_names.append(
                    cleaned
                )

            continue

        if not isinstance(source, dict):
            continue

        publisher = normalize_whitespace(
            source.get("publisher")
            or source.get("name")
            or source.get("organization")
            or source.get("source")
        )

        if publisher:
            source_names.append(
                publisher
            )

        source_url = extract_url(
            source.get("url")
            or source.get("link")
        )

        if source_url:
            source_urls.append(
                source_url
            )

    direct_source = normalize_whitespace(
        record.get("publisher")
        or record.get("source")
        or record.get("organization")
    )

    if direct_source:
        source_names.append(
            direct_source
        )

    direct_url = extract_url(
        record.get("url")
        or record.get("link")
    )

    if direct_url:
        source_urls.append(
            direct_url
        )

    return (
        merge_unique([], source_names),
        merge_unique([], source_urls),
    )


def determine_intelligence_type(
    record: Mapping[str, Any],
    source_kind: str = "",
) -> str:
    explicit_type = normalize_text(
        record.get("record_type")
        or record.get("evidence_type")
        or record.get("intelligence_type")
        or record.get("type")
        or source_kind
    )

    aliases = {
        "incident": "incident",
        "cyber incident": "incident",
        "cyber_incident": "incident",

        "assessment": "assessment",
        "strategic assessment": "assessment",
        "strategic_assessment": "assessment",
        "threat assessment": "assessment",
        "threat_assessment": "assessment",

        "signal": "signal",
        "strategic signal": "signal",
        "strategic_signal": "signal",
        "observation": "signal",
        "threat observation": "signal",
        "threat_observation": "signal",

        "narrative": "narrative",
        "strategic narrative": "narrative",
        "strategic_narrative": "narrative",
    }

    if explicit_type in aliases:
        return aliases[explicit_type]

    if "narrative" in explicit_type:
        return "narrative"

    if "signal" in explicit_type:
        return "signal"

    if "assessment" in explicit_type:
        return "assessment"

    if "incident" in explicit_type:
        return "incident"

    return "signal"


def intelligence_id(
    record: Mapping[str, Any],
    evidence_type: str,
) -> str:
    candidate = normalize_whitespace(
        record.get("incident_id")
        or record.get("signal_id")
        or record.get("narrative_id")
        or record.get("assessment_id")
        or record.get("evidence_id")
        or record.get("id")
        or record.get("uuid")
    )

    if candidate:
        return candidate

    title = normalize_whitespace(
        record.get("title")
        or record.get("name")
        or record.get("headline")
    )

    published = normalize_whitespace(
        record.get("published")
        or record.get("date")
        or record.get("created_at")
    )

    if not title:
        return ""

    digest = hashlib.sha256(
        (
            evidence_type
            + "\x1f"
            + title.casefold()
            + "\x1f"
            + published
        ).encode("utf-8")
    ).hexdigest()[:12].upper()

    prefixes = {
        "incident": "INC",
        "assessment": "ASM",
        "signal": "SIG",
        "narrative": "NAR",
    }

    prefix = prefixes.get(
        evidence_type,
        "INT",
    )

    return f"{prefix}-{digest}"


def normalize_intelligence(
    record: dict[str, Any],
    source_kind: str = "",
) -> Evidence | None:
    evidence_type = determine_intelligence_type(
        record,
        source_kind=source_kind,
    )

    evidence_id = intelligence_id(
        record,
        evidence_type,
    )

    if not evidence_id:
        return None

    title = normalize_whitespace(
        record.get("title")
        or record.get("name")
        or record.get("headline")
        or evidence_id
    )

    description = normalize_whitespace(
        record.get("summary")
        or record.get("description")
        or record.get("abstract")
        or record.get("narrative")
        or record.get("text")
        or record.get("content")
    )

    dates = nested_dict(
        record,
        "dates",
    )

    attack = nested_dict(
        record,
        "attack",
    )

    impact = nested_dict(
        record,
        "impact",
    )

    cyber_physical = nested_dict(
        record,
        "cyber_physical",
    )

    automotive_relevance = nested_dict(
        record,
        "automotive_relevance",
    )

    actors = nested_dict(
        record,
        "actors",
    )

    geography = nested_dict(
        record,
        "geography",
    )

    evidence_metadata = nested_dict(
        record,
        "evidence",
    )

    campaign = nested_dict(
        record,
        "campaign",
    )

    sectors = merge_unique(
        listify(record.get("sectors")),
        listify(record.get("domains")),
    )

    sectors = merge_unique(
        sectors,
        listify(record.get("industry")),
    )

    themes = merge_unique(
        listify(record.get("themes")),
        listify(record.get("theme")),
    )

    themes = merge_unique(
        themes,
        listify(record.get("narrative_tags")),
    )

    tags = merge_unique(
        listify(record.get("tags")),
        listify(record.get("keywords")),
    )

    signal_types = merge_unique(
        listify(record.get("signal_types")),
        listify(record.get("signal_type")),
    )

    assets = merge_unique(
        listify(record.get("assets")),
        listify(record.get("affected_assets")),
    )

    protocols = listify(
        record.get("protocols")
    )

    attack_vectors = merge_unique(
        listify(
            attack.get("attack_vector")
        ),
        listify(
            record.get("attack_vectors")
        ),
    )

    initial_access = listify(
        attack.get("initial_access")
    )

    techniques = merge_unique(
        listify(
            attack.get("techniques")
        ),
        listify(
            record.get("techniques")
        ),
    )

    authentication_issues = listify(
        attack.get(
            "authentication_issues"
        )
    )

    vulnerabilities = listify(
        attack.get("vulnerabilities")
    )

    impact_categories = merge_unique(
        listify(
            impact.get("categories")
        ),
        listify(
            record.get("impacts")
        ),
    )

    actor_names = listify(
        actors.get("names")
    )

    actor_types = listify(
        actors.get("type")
    )

    actor_motivations = listify(
        actors.get("motivation")
    )

    countries = listify(
        geography.get("countries")
    )

    regions = listify(
        geography.get("regions")
    )

    related_signals = merge_unique(
        listify(record.get("signal_ids")),
        listify(record.get("related_signals")),
    )

    related_incidents = merge_unique(
        listify(record.get("incident_ids")),
        listify(record.get("related_incidents")),
    )

    related_narratives = merge_unique(
        listify(record.get("narrative_ids")),
        listify(record.get("related_narratives")),
    )

    linked_cves = merge_unique(
        listify(
            attack.get("cves")
        ),
        listify(
            record.get("cves")
        ),
    )

    linked_cves = merge_unique(
        linked_cves,
        [
            match.upper()
            for match in CVE_ID_RE.findall(
                join_nonempty([
                    title,
                    description,
                    attack.get("description"),
                    impact.get("description"),
                ])
            )
        ],
    )

    source_names, source_urls = (
        collect_source_metadata(
            record
        )
    )

    published = normalize_whitespace(
        dates.get("published")
        or dates.get("first_observed")
        or record.get("published")
        or record.get("date")
        or record.get("created_at")
    )

    primary_url = (
        source_urls[0]
        if source_urls
        else extract_url(
            record.get("url")
        )
    )

    status = normalize_whitespace(
        record.get("status")
    )

    event_type = normalize_whitespace(
        record.get("event_type")
    )

    confidence = normalize_whitespace(
        evidence_metadata.get("confidence")
        or record.get("confidence")
    )

    severity = normalize_whitespace(
        impact.get("severity")
        or record.get("severity")
    )

    narrative_statement = normalize_whitespace(
        record.get("narrative")
        or record.get("strategic_narrative")
        or record.get("interpretation")
        or record.get("analysis")
    )

    rationale = normalize_whitespace(
        record.get("rationale")
        or record.get("reason")
        or record.get("why_it_matters")
    )

    trend = normalize_whitespace(
        record.get("trend")
        or record.get("direction")
    )

    horizon = normalize_whitespace(
        record.get("time_horizon")
        or record.get("horizon")
    )

    scale_terms: list[str] = []

    if geography.get("scope"):
        scale_terms.append(
            normalize_whitespace(
                geography.get("scope")
            )
        )

    for key in (
        "scale",
        "scope",
        "affected_count",
        "affected_locations",
        "affected_organizations",
    ):
        value = record.get(key)

        if value not in (None, "", [], {}):
            scale_terms.append(
                normalize_whitespace(value)
            )

    structured_terms = merge_unique(
        sectors,
        themes,
    )

    structured_terms = merge_unique(
        structured_terms,
        tags,
    )

    structured_terms = merge_unique(
        structured_terms,
        signal_types,
    )

    matched_terms = merge_unique(
        structured_terms,
        impact_categories,
    )

    topics = merge_unique(
        sectors,
        themes,
    )

    topics = merge_unique(
        topics,
        signal_types,
    )

    surfaces = merge_unique(
        assets,
        attack_vectors,
    )

    search_text = join_nonempty([
        evidence_id,
        evidence_type,
        title,
        description,
        narrative_statement,
        rationale,

        status,
        event_type,
        confidence,
        severity,
        trend,
        horizon,

        " ".join(sectors),
        " ".join(themes),
        " ".join(tags),
        " ".join(signal_types),
        " ".join(assets),
        " ".join(protocols),

        " ".join(attack_vectors),
        " ".join(initial_access),
        " ".join(techniques),
        " ".join(authentication_issues),
        " ".join(vulnerabilities),
        " ".join(linked_cves),

        normalize_whitespace(
            attack.get("description")
        ),

        " ".join(
            impact_categories
        ),

        normalize_whitespace(
            impact.get("description")
        ),

        normalize_whitespace(
            cyber_physical.get(
                "digital_action"
            )
        ),

        normalize_whitespace(
            cyber_physical.get(
                "physical_effect"
            )
        ),

        normalize_whitespace(
            cyber_physical.get(
                "potential_consequence"
            )
        ),

        normalize_whitespace(
            automotive_relevance.get(
                "relevance"
            )
        ),

        normalize_whitespace(
            automotive_relevance.get(
                "reason"
            )
        ),

        " ".join(actor_names),
        " ".join(actor_types),
        " ".join(actor_motivations),

        normalize_whitespace(
            actors.get("attribution")
        ),

        normalize_whitespace(
            campaign.get("campaign_name")
        ),

        " ".join(countries),
        " ".join(regions),
        " ".join(scale_terms),

        " ".join(related_incidents),
        " ".join(related_signals),
        " ".join(related_narratives),

        " ".join(source_names),
    ])

    return Evidence(
        evidence_id=evidence_id,
        evidence_type=evidence_type,
        title=title,
        description=description,
        published=published,
        url=primary_url,
        severity=severity,
        sources=source_names,
        matched_terms=matched_terms,
        topics=topics,
        surfaces=surfaces,
        linked_cves=linked_cves,
        search_text=search_text,
        tokens=tokenize(
            search_text
        ),
        features=extract_features(
            search_text
        ),
        raw_record=record,
    )

def normalize_incident(
    record: dict[str, Any],
) -> Evidence | None:
    return normalize_intelligence(
        record,
        source_kind="incident",
    )


def normalize_signal(
    record: dict[str, Any],
) -> Evidence | None:
    return normalize_intelligence(
        record,
        source_kind="signal",
    )


def normalize_narrative(
    record: dict[str, Any],
) -> Evidence | None:
    return normalize_intelligence(
        record,
        source_kind="narrative",
    )


def load_evidence(
    cve_path: Path,
    paper_path: Path,
    repository_path: Path,
    incident_path: Path,
    signal_path: Path,
) -> list[Evidence]:
    cve_data = load_json(
        cve_path
    )

    paper_data = load_json(
        paper_path
    )
    repository_data = load_json(
        repository_path
    )

    incident_data = load_json(
        incident_path
    )

    signal_data = load_json(
        signal_path
    )

    cve_records = records_from_json(
        cve_data,
        (
            "records",
            "cves",
            "items",
        ),
    )

    paper_records = records_from_json(
        paper_data,
        (
            "records",
            "papers",
            "items",
        ),
    )
    repository_records = records_from_json(
        repository_data,
        (
            "repositories",
            "records",
            "items",
        ),
    )

    incident_records = records_from_json(
        incident_data,
        (
            "incidents",
            "records",
            "items",
        ),
    )

    signal_records = records_from_json(
        signal_data,
        (
            "records",
            "signals",
            "items",
        ),
    )

    evidence: list[Evidence] = []

    # ---------------------------------------------------------
    # CVEs
    # ---------------------------------------------------------

    for record in cve_records:
        item = normalize_cve(
            record
        )

        if item:
            evidence.append(
                item
            )

    # ---------------------------------------------------------
    # Academic papers
    # ---------------------------------------------------------

    for record in paper_records:
        item = normalize_paper(
            record
        )

        if item:
            evidence.append(
                item
            )


    # ---------------------------------------------------------
    # GitHub repositories
    # ---------------------------------------------------------
    for record in repository_records:
        item = normalize_repository(
            record
        )

        if item:
            evidence.append(
                item
            )


    # ---------------------------------------------------------
    # Incidents and assessments
    # ---------------------------------------------------------

    for record in incident_records:
        item = normalize_incident(
            record
        )

        if item:
            evidence.append(
                item
            )

    # ---------------------------------------------------------
    # Strategic signals
    # ---------------------------------------------------------

    for record in signal_records:
        item = normalize_signal(
            record
        )

        if item:
            evidence.append(
                item
            )

    LOGGER.info( 
        (
            "Loaded %d CVEs, %d papers, %d repositories, "
            "%d incident/intelligence records, "
            "and %d signals"
        ),
        len(cve_records),
        len(paper_records),
        len(repository_records),
        len(incident_records),
        len(signal_records),
    )

    # ---------------------------------------------------------
    # Deduplication
    # ---------------------------------------------------------



    deduped: dict[
        tuple[str, str],
        Evidence,
    ] = {}

    duplicates: Counter[
        tuple[str, str]
    ] = Counter()

    for item in evidence:
        key = (
            item.evidence_type,
            item.evidence_id.casefold(),
        )
        duplicates[key] += 1

        current = deduped.get(
            key
        )

        if (
            current is None
            or len(item.search_text)
            > len(current.search_text)
        ):
            deduped[key] = item

    duplicate_count = sum(
        count - 1
        for count in duplicates.values()
        if count > 1
    )

    if duplicate_count:
        LOGGER.warning(
            "Removed %d duplicate evidence records",
            duplicate_count,
        )

    result = list(
        deduped.values()
    )

    result.sort(
        key=lambda evidence_item: (
            evidence_item.evidence_type,
            evidence_item.evidence_id.casefold(),
        )
    )

    return result


def build_idf(threats: Sequence[Threat], evidence: Sequence[Evidence]) -> dict[str, float]:
    documents = [item.tokens for item in threats] + [item.tokens for item in evidence]
    total = max(len(documents), 1)
    document_frequency: Counter[str] = Counter()
    for tokens in documents:
        document_frequency.update(tokens)
    return {
        token: math.log((1 + total) / (1 + frequency)) + 1.0
        for token, frequency in document_frequency.items()
    }


def weighted_jaccard(left: set[str], right: set[str], idf: Mapping[str, float]) -> float:
    if not left or not right:
        return 0.0
    union = left | right
    intersection = left & right
    denominator = sum(idf.get(token, 1.0) for token in union)
    if denominator <= 0:
        return 0.0
    return sum(idf.get(token, 1.0) for token in intersection) / denominator


def coverage_score(evidence_tokens: set[str], threat_tokens: set[str], idf: Mapping[str, float]) -> float:
    if not evidence_tokens or not threat_tokens:
        return 0.0
    common = evidence_tokens & threat_tokens
    denominator = sum(idf.get(t, 1.0) for t in evidence_tokens)
    if denominator <= 0:
        return 0.0
    return sum(idf.get(t, 1.0) for t in common) / denominator


def set_overlap(left: set[str], right: set[str]) -> float:
    if not left or not right:
        return 0.0
    return len(left & right) / len(left | right)


def matched_feature_map(evidence: Evidence, threat: Threat) -> dict[str, list[str]]:
    result: dict[str, list[str]] = {}
    for axis in FEATURE_TERMS:
        matches = sorted(evidence.features.get(axis, set()) & threat.features.get(axis, set()))
        if matches:
            result[axis] = matches
    return result


def apply_rules(evidence: Evidence, threat: Threat) -> tuple[float, list[str]]:
    bonus = 0.0
    fired: list[str] = []
    for rule in THREAT_RULES:
        threat_bonus = rule["boost"].get(threat.threat_id, 0.0)
        if threat_bonus and has_any(evidence.search_text, rule["requires_any"]):
            bonus += float(threat_bonus)
            fired.append(rule["name"])
    return min(bonus, 0.35), fired


def context_penalty(evidence: Evidence, threat: Threat) -> float:
    requirements = THREAT_CONTEXT_REQUIREMENTS.get(threat.threat_id)
    if not requirements:
        return 0.0
    if has_any(evidence.search_text, requirements):
        return 0.0
    # Broad/systemic threats need explicit scenario context.
    return 0.12


def evidence_is_generic(evidence: Evidence) -> bool:
    automotive_indicators = [
        "vehicle", "automotive", "car", "ev charging", "charging station", "ocpp", "v2g",
        "can bus", "ecu", "infotainment", "headunit", "telematics", "gnss", "adas",
        "roadside", "fleet", "keyless", "traction", "charging infrastructure"
    ]
    return not has_any(evidence.search_text, automotive_indicators)

def type_specific_correlation_floor(
    evidence: Evidence,
    threat: Threat,
    feature_matches: Mapping[str, list[str]],
    rules_fired: Sequence[str],
) -> tuple[float, list[str]]:
    """
    Determine a conservative minimum relevance score based on the
    evidence type.

    A type-specific floor is only activated when the evidence also
    contains threat-specific context or structured feature overlap.

    The floor is a minimum score, not an additive bonus.
    """

    evidence_type = normalize_text(
        evidence.evidence_type
    )

    if evidence_type not in {
        "incident",
        "assessment",
        "signal",
        "narrative",
        "repository",
    }:
        return 0.0, []

    requirements = THREAT_CONTEXT_REQUIREMENTS.get(
        threat.threat_id,
        [],
    )

    context_match = bool(
        requirements
        and has_any(
            evidence.search_text,
            requirements,
        )
    )

    matched_axes = {
        axis
        for axis, values in feature_matches.items()
        if values
    }

    matched_axis_count = len(
        matched_axes
    )

    asset_match = bool(
        feature_matches.get("assets")
    )

    protocol_match = bool(
        feature_matches.get("protocols")
    )

    attack_vector_match = bool(
        feature_matches.get("attack_vectors")
    )

    impact_match = bool(
        feature_matches.get("impacts")
    )

    lifecycle_match = bool(
        feature_matches.get("lifecycle")
    )

    deterministic_rule_match = bool(
        rules_fired
    )

    floor = 0.0
    reasons: list[str] = []

    # ---------------------------------------------------------
    # INCIDENT
    #
    # Real-world incidents are strong observations, but an
    # incident must still contain threat-specific context or
    # structured technical/impact evidence before receiving a
    # correlation floor.
    # ---------------------------------------------------------

    if evidence_type == "incident":
        if (
            context_match
            and matched_axis_count >= 1
        ):
            floor = max(
                floor,
                0.40,
            )

            reasons.append(
                "incident_context_and_feature"
            )

        if (
            context_match
            and matched_axis_count >= 2
        ):
            floor = max(
                floor,
                0.48,
            )

            reasons.append(
                "incident_multi_axis"
            )

        if (
            context_match
            and asset_match
            and impact_match
        ):
            floor = max(
                floor,
                0.54,
            )

            reasons.append(
                "incident_asset_and_impact"
            )

        if (
            deterministic_rule_match
            and matched_axis_count >= 1
        ):
            floor = max(
                floor,
                0.48,
            )

            reasons.append(
                "incident_rule_and_feature"
            )

        if (
            deterministic_rule_match
            and asset_match
            and (
                protocol_match
                or attack_vector_match
                or impact_match
            )
        ):
            floor = max(
                floor,
                0.56,
            )

            reasons.append(
                "incident_strong_structured_match"
            )

    # ---------------------------------------------------------
    # ASSESSMENT
    #
    # Assessments are authoritative interpretations rather than
    # direct observations. They may enter review when strategic
    # context and structured overlap support the relationship.
    # ---------------------------------------------------------

    elif evidence_type == "assessment":
        if (
            context_match
            and matched_axis_count >= 1
        ):
            floor = max(
                floor,
                0.40,
            )

            reasons.append(
                "assessment_context_and_feature"
            )

        if (
            context_match
            and matched_axis_count >= 2
        ):
            floor = max(
                floor,
                0.48,
            )

            reasons.append(
                "assessment_multi_axis"
            )

        if (
            context_match
            and asset_match
            and impact_match
        ):
            floor = max(
                floor,
                0.52,
            )

            reasons.append(
                "assessment_asset_and_impact"
            )

        if (
            context_match
            and matched_axis_count >= 2
            and (
                protocol_match
                or attack_vector_match
            )
        ):
            floor = max(
                floor,
                0.54,
            )

            reasons.append(
                "assessment_technical_context"
            )

    # ---------------------------------------------------------
    # SIGNAL
    #
    # Signals describe emerging observations. They may create a
    # review candidate but should remain below automatic
    # acceptance unless the normal calculated score is stronger.
    # ---------------------------------------------------------

    elif evidence_type == "signal":
        if (
            context_match
            and matched_axis_count >= 1
        ):
            floor = max(
                floor,
                0.39,
            )

            reasons.append(
                "signal_context_and_feature"
            )

        if (
            context_match
            and matched_axis_count >= 2
        ):
            floor = max(
                floor,
                0.44,
            )

            reasons.append(
                "signal_multi_axis"
            )

        if (
            deterministic_rule_match
            and matched_axis_count >= 1
        ):
            floor = max(
                floor,
                0.46,
            )

            reasons.append(
                "signal_rule_and_feature"
            )

        if (
            context_match
            and asset_match
            and impact_match
        ):
            floor = max(
                floor,
                0.48,
            )

            reasons.append(
                "signal_asset_and_impact"
            )

    # ---------------------------------------------------------
    # REPOSITORY
    # ---------------------------------------------------------
    elif evidence_type == "repository":
        if (
            deterministic_rule_match
            and matched_axis_count >= 1
        ):
            floor = max(
                floor,
                0.40,
            )
            reasons.append(
                "repository_rule_and_feature"
            )

        if (
            matched_axis_count >= 2
            and (
                protocol_match
                or attack_vector_match
            )
        ):
            floor = max(
                floor,
                0.42,
            )
            reasons.append(
                "repository_multi_axis_technical"
            )

        if (
            deterministic_rule_match
            and asset_match
            and (
                protocol_match
                or attack_vector_match
                or lifecycle_match
            )
        ):
            floor = max(
                floor,
                0.48,
            )
            reasons.append(
                "repository_strong_structured_match"
            )



    # ---------------------------------------------------------
    # NARRATIVE
    #
    # Narratives are analytical syntheses. They require multiple
    # independent semantic axes or explicit threat context before
    # receiving a review floor.
    # ---------------------------------------------------------

    elif evidence_type == "narrative":
        if (
            context_match
            and matched_axis_count >= 2
        ):
            floor = max(
                floor,
                0.40,
            )

            reasons.append(
                "narrative_context_and_multi_axis"
            )

        if (
            context_match
            and matched_axis_count >= 3
        ):
            floor = max(
                floor,
                0.46,
            )

            reasons.append(
                "narrative_broad_structured_support"
            )

        if (
            context_match
            and asset_match
            and impact_match
            and (
                attack_vector_match
                or protocol_match
                or lifecycle_match
            )
        ):
            floor = max(
                floor,
                0.50,
            )

            reasons.append(
                "narrative_pathway_support"
            )

    return floor, reasons

def score_pair(
    evidence: Evidence,
    threat: Threat,
    idf: Mapping[str, float],
) -> tuple[ScoreBreakdown, dict[str, list[str]], list[str], list[str]]:
    feature_matches = matched_feature_map(evidence, threat)
    lexical = weighted_jaccard(evidence.tokens, threat.tokens, idf)
    evidence_coverage = coverage_score(evidence.tokens, threat.tokens, idf)
    lexical_combined = min(1.0, 0.65 * lexical + 0.35 * evidence_coverage)

    title_tokens = tokenize(threat.title + " " + threat.title_nl)
    title_score = coverage_score(evidence.tokens, title_tokens, idf)

    axis_scores = {
        axis: set_overlap(evidence.features.get(axis, set()), threat.features.get(axis, set()))
        for axis in FEATURE_TERMS
    }
    structured_axes = [axis for axis, score in axis_scores.items() if score > 0]
    feature_score = (
        0.24 * axis_scores["assets"]
        + 0.20 * axis_scores["protocols"]
        + 0.14 * axis_scores["attack_vectors"]
        + 0.22 * axis_scores["impacts"]
        + 0.12 * axis_scores["lifecycle"]
        + 0.08 * max(
            set_overlap(set(map(normalize_text, threat.secondary_domains)), set()),
            0.0,
        )
    )

    domain_text = normalize_text(" ".join([threat.category, threat.primary_domain, *threat.secondary_domains]))
    domain_tokens = tokenize(domain_text)
    domain_score = coverage_score(evidence.tokens, domain_tokens, idf)

    rule_bonus, rules_fired = apply_rules(evidence, threat)
    ctx_penalty = context_penalty(evidence, threat)
    generic_penalty = 0.16 if evidence_is_generic(evidence) and not rules_fired else 0.0

    # Structured signals and exact rules dominate; lexical similarity is useful
    # but deliberately not sufficient by itself.
    base = (
        0.22 * lexical_combined
        + 0.12 * title_score
        + 0.30 * feature_score
        + 0.10 * axis_scores["assets"]
        + 0.08 * axis_scores["protocols"]
        + 0.06 * axis_scores["attack_vectors"]
        + 0.07 * axis_scores["impacts"]
        + 0.03 * axis_scores["lifecycle"]
        + 0.02 * domain_score
    )
    # A pair with several independently matching axes gets a modest bonus.
    multi_axis_bonus = min(
        0.10,
        max(0, len(structured_axes) - 1) * 0.025
    )

    calculated_score = (
        base
        + rule_bonus
        + multi_axis_bonus
        - ctx_penalty
        - generic_penalty
    )

    # ---------------------------------------------------------
    # RULE FLOOR
    #
    # Strong deterministic technical matches must not disappear
    # merely because a short CVE description has little lexical
    # overlap with a long Threat Register description.
    # ---------------------------------------------------------

    rule_floor = 0.0
    # A rule-only match is a useful signal, but is not by itself
    # sufficient to force the correlation into manual review.
    #
    # The normal calculated score can still push the result above
    # the review threshold when additional lexical/contextual
    # evidence exists.
    if rules_fired:
        rule_floor = 0.30
   
    # ---------------------------------------------------------
    # CVE-SPECIFIC RULE FLOOR
    # ---------------------------------------------------------

    if evidence.evidence_type == "cve":

        # Rule + asset identifies a plausible CVE correlation,
        # but is not sufficient for automatic acceptance.
        if (
            rules_fired
            and feature_matches.get("assets")
        ):
            rule_floor = max(
                rule_floor,
                0.48
            )    


        strong_cve_match = False

        # Charging infrastructure plus a concrete attack path
        # or an explicit charging protocol.
        if (
            "ev_charging_general" in rules_fired
            and "ev_charging" in feature_matches.get(
                "assets",
                []
            )
            and (
                "ocpp" in feature_matches.get(
                    "protocols",
                    []
                )
                or feature_matches.get(
                    "attack_vectors"
                )
            )
        ):
            strong_cve_match = True

        # Infotainment CVE plus a concrete attack vector
        # or operational impact.
        if (
            "infotainment_headunit" in rules_fired
            and "infotainment" in feature_matches.get(
                "assets",
                []
            )
            and (
                feature_matches.get(
                    "attack_vectors"
                )
                or feature_matches.get(
                    "impacts"
                )
            )
        ):
            strong_cve_match = True

        # Update-system CVE plus a concrete supply-chain,
        # privilege, integrity, or availability dimension.
        if (
            "vehicle_update_compromise" in rules_fired
            and "update_system" in feature_matches.get(
                "assets",
                []
            )
            and (
                feature_matches.get(
                    "attack_vectors"
                )
                or feature_matches.get(
                    "impacts"
                )
            )
        ):
            strong_cve_match = True

        # Vehicle theft requires explicit keyless/RKE context.
        if (
            "digital_vehicle_theft" in rules_fired
            and (
                "rke" in feature_matches.get(
                    "protocols",
                    []
                )
                or "theft" in feature_matches.get(
                    "impacts",
                    []
       
         )
            )
        ):
            strong_cve_match = True

        if strong_cve_match:
            rule_floor = max(
                rule_floor,
                0.62
            )
    # ---------------------------------------------------------
    # TYPE-SPECIFIC CORRELATION FLOOR
    # ---------------------------------------------------------

    type_floor, type_floor_reasons = (
        type_specific_correlation_floor(
            evidence=evidence,
            threat=threat,
            feature_matches=feature_matches,
            rules_fired=rules_fired,
        )
    )

    if type_floor > 0.0:
        rule_floor = max(
            rule_floor,
            type_floor,
        )

        rules_fired.extend(
            reason
            for reason in type_floor_reasons
            if reason not in rules_fired
        )

   


    # The rule floor is a minimum, not an additional bonus.
    final = max(
        0.0,
        min(
            1.0,
            max(
                calculated_score, 
                rule_floor
            )
        )
    )
    common_tokens = sorted(
        evidence.tokens & threat.tokens,
        key=lambda token: (-idf.get(token, 1.0), token),
    )[:15]

    breakdown = ScoreBreakdown(
        lexical=lexical_combined,
        title=title_score,
        features=feature_score,
        assets=axis_scores["assets"],
        protocols=axis_scores["protocols"],
        attack_vectors=axis_scores["attack_vectors"],
        impacts=axis_scores["impacts"],
        lifecycle=axis_scores["lifecycle"],
        domain=domain_score,
        rule_bonus=rule_bonus + multi_axis_bonus,
        context_penalty=ctx_penalty,
        generic_penalty=generic_penalty,
        final=final,
    )
    return breakdown, feature_matches, common_tokens, rules_fired


def classify_paper_strength(evidence: Evidence) -> tuple[float, str]:
    text = evidence.search_text
    if has_any(text, ["real-world malicious activity", "real world attack", "production charging station", "deployed in an exposed environment"]):
        return 0.86, "Real-world observation"
    if has_any(text, ["experimental analysis", "experiment", "proof of concept", "proof-of-concept", "prototype", "validated by detecting"]):
        return 0.82, "Experimental demonstration"
    if has_any(text, ["case study", "measurement study", "empirical study", "dataset analysis"]):
        return 0.70, "Empirical study"
    if has_any(text, ["simulation", "simulated", "digital twin"]):
        return 0.52, "Simulation"
    if has_any(text, ["survey", "systematic review", "literature review"]):
        return 0.34, "Survey/review"
    if has_any(text, ["framework", "methodology", "model", "conceptual"]):
        return 0.42, "Conceptual or methodological"
    return 0.56, "Technical analysis"

def classify_evidence_strength(
    evidence: Evidence,
) -> tuple[float, str]:
    """
    Determine the intrinsic strength of an evidence or intelligence record.

    This is separate from threat relevance. A record may be authoritative
    or strong evidence while still having little relevance to a specific
    T-xxx threat.
    """

    # ---------------------------------------------------------
    # Academic research
    # ---------------------------------------------------------

    if evidence.evidence_type == "paper":
        return classify_paper_strength(
            evidence
        )

    # ---------------------------------------------------------
    # Strategic assessment
    # ---------------------------------------------------------

    if evidence.evidence_type == "assessment":
        score = 0.68
        label = "Authoritative strategic assessment"

        if evidence.description:
            score += 0.05

        if evidence.sources:
            score += 0.05

        return (
            min(
                0.88,
                max(
                    0.40,
                    score,
                ),
            ),
            label,
        )

    # ---------------------------------------------------------
    # Real-world incident
    # ---------------------------------------------------------

    if evidence.evidence_type == "incident":
        score = 0.70
        label = "Real-world incident report"

        if evidence.description:
            score += 0.05

        if evidence.sources:
            score += 0.05

        return (
            min(
                0.90,
                max(
                    0.40,
                    score,
                ),
            ),
            label,
        )

    # ---------------------------------------------------------
    # Intelligence signal
    # ---------------------------------------------------------

    if evidence.evidence_type == "signal":
        score = 0.54
        label = "Emerging intelligence signal"

        if evidence.description:
            score += 0.05

        if evidence.sources:
            score += 0.05

        return (
            min(
                0.78,
                max(
                    0.30,
                    score,
                ),
            ),
            label,
        )

    # ---------------------------------------------------------
    # Strategic narrative
    # ---------------------------------------------------------

    if evidence.evidence_type == "narrative":
        score = 0.58
        label = "Synthesized intelligence narrative"

        if evidence.description:
            score += 0.05

        if evidence.sources:
            score += 0.03

        return (
            min(
                0.80,
                max(
                    0.30,
                    score,
                ),
            ),
            label,
        )


    # ---------------------------------------------------------
    # GitHub repository
    # ---------------------------------------------------------
    if evidence.evidence_type == "repository":
        raw = evidence.raw_record

        relevance = raw.get("relevance")
        if not isinstance(relevance, dict):
            relevance = {}

        readme_metadata = raw.get(
            "readme_metadata"
        )
        if not isinstance(
            readme_metadata,
            dict,
        ):
            readme_metadata = {}

        level = normalize_text(
            relevance.get("level")
        )

        score = 0.46
        label = "Technical repository"

        if level == "direct":
            score += 0.08
            label = (
                "Directly relevant technical repository"
            )

        elif level == "strong":
            score += 0.04
            label = (
                "Strongly relevant technical repository"
            )

        if readme_metadata.get(
            "available"
        ):
            score += 0.05

        if evidence.topics:
            score += 0.03

        if evidence.linked_cves:
            score += 0.06

        stars = safe_int(
            raw.get("stars")
        ) or 0

        if stars >= 100:
            score += 0.04
        elif stars >= 20:
            score += 0.02

        if raw.get("archived"):
            score -= 0.05

        return (
            min(
                0.76,
                max(
                    0.30,
                    score,
                ),
            ),
            label,
        )


    # ---------------------------------------------------------
    # CVE / vulnerability evidence
    # ---------------------------------------------------------

    score = 0.62
    label = "Confirmed vulnerability record"

    if evidence.pending_analysis:
        score -= 0.12
        label = "Pending vulnerability analysis"

    if evidence.cvss is not None:
        if evidence.cvss >= 9.0:
            score += 0.10

        elif evidence.cvss >= 7.0:
            score += 0.07

        elif evidence.cvss >= 4.0:
            score += 0.03

    if evidence.description:
        score += 0.07

    if has_any(
        evidence.search_text,
        [
            "allows attackers",
            "authentication bypass",
            "remote code execution",
            "arbitrary code",
        ],
    ):
        score += 0.06

    return (
        min(
            0.92,
            max(
                0.20,
                score,
            ),
        ),
        label,
    )

def confidence_for(
    score: float,
    feature_matches: Mapping[str, list[str]],
    rules_fired: Sequence[str],
    common_tokens: Sequence[str],
) -> str:
    axes = len(feature_matches)
    if score >= 0.72 and (rules_fired or axes >= 2):
        return "high"
    if score >= 0.52 and (axes >= 1 or len(common_tokens) >= 3):
        return "medium"
    return "low"


def status_for(score: float, accept_threshold: float, review_threshold: float) -> str:
    if score >= accept_threshold:
        return "accepted"
    if score >= review_threshold:
        return "review"
    return "rejected"


def explain_match(
    evidence: Evidence,
    threat: Threat,
    feature_matches: Mapping[str, list[str]],
    common_tokens: Sequence[str],
    rules_fired: Sequence[str],
    status: str,
) -> str:
    components: list[str] = []
    if feature_matches:
        readable = []
        for axis, values in feature_matches.items():
            readable.append(f"{axis}: {', '.join(values)}")
        components.append("structured overlap in " + "; ".join(readable))
    if rules_fired:
        components.append("specific rules: " + ", ".join(rules_fired))
    if common_tokens:
        components.append("shared terms: " + ", ".join(common_tokens[:8]))
    if not components:
        components.append("limited lexical similarity")
    prefix = {
        "accepted": "Strong automatic match",
        "review": "Plausible match requiring review",
        "rejected": "Insufficient match",
    }[status]
    return f"{prefix} between {evidence.evidence_id} and {threat.threat_id}: " + "; ".join(components) + "."


def build_match_result(
    evidence: Evidence,
    threat: Threat,
    breakdown: ScoreBreakdown,
    feature_matches: dict[str, list[str]],
    common_tokens: list[str],
    rules_fired: list[str],
    accept_threshold: float,
    review_threshold: float,
) -> MatchResult:
    status = status_for(breakdown.final, accept_threshold, review_threshold)
    strength_score, strength_label = classify_evidence_strength(evidence)
    confidence = confidence_for(breakdown.final, feature_matches, rules_fired, common_tokens)
    matched_axes = sorted(feature_matches)
    return MatchResult(
        evidence_id=evidence.evidence_id,
        evidence_type=evidence.evidence_type,
        threat_id=threat.threat_id,
        relevance_score=round(breakdown.final, 4),
        evidence_strength_score=round(strength_score, 4),
        evidence_strength=strength_label,
        confidence=confidence,
        status=status,
        title=evidence.title,
        url=evidence.url,
        published=evidence.published,
        matched_axes=matched_axes,
        matched_features=feature_matches,
        matched_terms=common_tokens,
        rules_fired=rules_fired,
        explanation=explain_match(
            evidence, threat, feature_matches, common_tokens, rules_fired, status
        ),
        score_breakdown={
            key: round(value, 4)
            for key, value in asdict(breakdown).items()
        },
    )


def candidate_rank(
    evidence: Evidence,
    threat: Threat,
    idf: Mapping[str, float],
) -> float:
    lexical = coverage_score(evidence.tokens, threat.tokens, idf)
    feature_matches = matched_feature_map(evidence, threat)
    feature_count = sum(len(values) for values in feature_matches.values())
    rule_bonus, _ = apply_rules(evidence, threat)
    return lexical + min(0.5, feature_count * 0.08) + rule_bonus


def evidence_type_allowed(
    threat: Threat,
    evidence_type: str,
) -> bool:
    """Return whether this evidence type is enabled for the threat profile."""

    allowed = {
        normalize_text(value)
        for value in threat.evidence_types
        if value
    }

    if not allowed:
        return False

    normalized_type = normalize_text(
        evidence_type
    )

    if normalized_type == "assessment":
        return (
            "assessment" in allowed
            or "signal" in allowed
        )

    if normalized_type in {
        "narrative",
        "threat observation",
    }:
        return "signal" in allowed

    if normalized_type == "repository":
        return (
            "repository" in allowed
            or "paper" in allowed
            or "cve" in allowed
        )

    return normalized_type in allowed

def correlate(
    threats: Sequence[Threat],
    evidence_items: Sequence[Evidence],
    idf: Mapping[str, float],
    args: argparse.Namespace,
) -> tuple[list[MatchResult], list[MatchResult]]:
    accepted_or_review: list[MatchResult] = []
    rejected: list[MatchResult] = []
    total = len(evidence_items)

    for index, evidence in enumerate(evidence_items, start=1):
        eligible_threats = [
            threat for threat in threats
            if evidence_type_allowed(threat, evidence.evidence_type)
        ]
        ranked = sorted(
            eligible_threats,
            key=lambda threat: candidate_rank(evidence, threat, idf),
            reverse=True,
        )
        candidates = ranked[: args.top_k]
        item_results: list[MatchResult] = []

        for threat in candidates:
            breakdown, features, terms, rules = score_pair(evidence, threat, idf)
            result = build_match_result(
                evidence,
                threat,
                breakdown,
                features,
                terms,
                rules,
                args.accept_threshold,
                args.review_threshold,
            )
            item_results.append(result)
            if args.explain:
                LOGGER.debug(
                    "%s -> %s status=%s final=%.4f lexical=%.4f features=%.4f rule=%.4f penalties=%.4f",
                    evidence.evidence_id,
                    threat.threat_id,
                    result.status,
                    breakdown.final,
                    breakdown.lexical,
                    breakdown.features,
                    breakdown.rule_bonus,
                    breakdown.context_penalty + breakdown.generic_penalty,
                )

        item_results.sort(key=lambda result: result.relevance_score, reverse=True)
        accepted_count = 0
        for result in item_results:
            if result.status == "accepted":
                if accepted_count < args.max_accepted_per_evidence:
                    accepted_or_review.append(result)
                    accepted_count += 1
                else:
                    result.status = "review"
                    result.explanation = "Additional plausible match moved to review because the accepted-match limit was reached."
                    accepted_or_review.append(result)
            elif result.status == "review":
                accepted_or_review.append(result)
            elif args.include_rejected:
                rejected.append(result)

        if index % 100 == 0 or index == total:
            LOGGER.info("Processed %d/%d evidence records", index, total)

    accepted_or_review.sort(
        key=lambda result: (result.threat_id, -result.relevance_score, result.evidence_id.casefold())
    )
    rejected.sort(
        key=lambda result: (result.evidence_id.casefold(), -result.relevance_score, result.threat_id)
    )
    return accepted_or_review, rejected


def sha256_file(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open("rb") as handle:
        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
            digest.update(chunk)
    return digest.hexdigest()


def iso_now() -> str:
    return datetime.now(timezone.utc).astimezone().isoformat()


def json_ready_match(result: MatchResult) -> dict[str, Any]:
    return asdict(result)


def threat_summary(threat: Threat) -> dict[str, Any]:
    return {
        "id": threat.threat_id,
        "title": threat.title,
        "title_nl": threat.title_nl,
        "category": threat.category,
        "primary_domain": threat.primary_domain,
        "secondary_domains": threat.secondary_domains,
        "tags": threat.tags,
        "terms": threat.terms,
        "surfaces": threat.surfaces,
        "evidence_types": threat.evidence_types,
        "signal_queries": threat.signal_queries,
        "risk_rating": threat.risk_rating,
    }


def evidence_summary(evidence: Evidence) -> dict[str, Any]:
    data: dict[str, Any] = {
        "id": evidence.evidence_id,
        "type": evidence.evidence_type,
        "title": evidence.title,
        "description": evidence.description,
        "published": evidence.published,
        "url": evidence.url,
        "matched_terms": evidence.matched_terms,
        "topics": evidence.topics,
        "vendors": evidence.vendors,
        "surfaces": evidence.surfaces,
        "linked_cves": evidence.linked_cves,
    }
    if evidence.evidence_type == "cve":
        data.update({
            "severity": evidence.severity,
            "cvss": evidence.cvss,
            "cvss_version": evidence.cvss_version,
            "pending_analysis": evidence.pending_analysis,
        })
    else:
        data.update({
            "authors": evidence.authors,
            "venue": evidence.venue,
            "year": evidence.year,
            "doi": evidence.doi,
            "sources": evidence.sources,
            "cited_by": evidence.cited_by,
            "open_access": evidence.open_access,
        })
    return data


def build_threat_correlations(
    threats: Sequence[Threat],
    results: Sequence[MatchResult],
) -> dict[str, Any]:
    grouped: dict[str, list[MatchResult]] = defaultdict(list)
    for result in results:
        grouped[result.threat_id].append(result)

    threat_map: dict[str, Any] = {}
    for threat in threats:
        matches = grouped.get(threat.threat_id, [])
        accepted = [match for match in matches if match.status == "accepted"]
        review = [match for match in matches if match.status == "review"]
        threat_map[threat.threat_id] = {
            **threat_summary(threat),
            "statistics": {
                "accepted": len(accepted),
                "review": len(review),
                "cves": sum(1 for match in accepted if match.evidence_type == "cve"),
                "papers": sum(1 for match in accepted if match.evidence_type == "paper"),
                "repositories": sum(
                    1
                    for match in accepted
                    if match.evidence_type == "repository"
                ),
                "signals": sum(1 for match in accepted if match.evidence_type == "signal"),
                "incidents": sum(1 for match in accepted if match.evidence_type == "incident"),
                "assessments": sum(1 for match in accepted if match.evidence_type == "assessment"),
                "narratives": sum(1 for match in accepted if match.evidence_type == "narrative"),
            },
            "evidence": [json_ready_match(match) for match in matches],
        }

    return {
        "schema_version": SCHEMA_VERSION,
        "generated_at": iso_now(),
        "threats": threat_map,
    }


def build_evidence_index(
    evidence_items: Sequence[Evidence],
    results: Sequence[MatchResult],
) -> dict[str, Any]:
    grouped: dict[tuple[str, str], list[MatchResult]] = defaultdict(list)
    for result in results:
        grouped[(result.evidence_type, result.evidence_id.casefold())].append(result)

    records: list[dict[str, Any]] = []
    for evidence in evidence_items:
        matches = sorted(
            grouped.get((evidence.evidence_type, evidence.evidence_id.casefold()), []),
            key=lambda match: match.relevance_score,
            reverse=True,
        )
        records.append({
            **evidence_summary(evidence),
            "matches": [
                {
                    "threat_id": match.threat_id,
                    "relevance_score": match.relevance_score,
                    "evidence_strength_score": match.evidence_strength_score,
                    "confidence": match.confidence,
                    "status": match.status,
                    "explanation": match.explanation,
                }
                for match in matches
            ],
        })

    return {
        "schema_version": SCHEMA_VERSION,
        "generated_at": iso_now(),
        "records": records,
    }


def build_review_output(
    results: Sequence[MatchResult],
    rejected: Sequence[MatchResult],
) -> dict[str, Any]:
    review = [result for result in results if result.status == "review"]
    accepted = [result for result in results if result.status == "accepted"]
    return {
        "schema_version": SCHEMA_VERSION,
        "generated_at": iso_now(),
        "statistics": {
            "accepted": len(accepted),
            "review_required": len(review),
            "rejected_included": len(rejected),
        },
        "review_required": [json_ready_match(result) for result in review],
        "rejected_candidates": [json_ready_match(result) for result in rejected],
    }


def atomic_write_json(path: Path, data: Any) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    fd, temporary_name = tempfile.mkstemp(
        prefix=f".{path.name}.", suffix=".tmp", dir=str(path.parent)
    )
    temporary_path = Path(temporary_name)
    try:
        with os.fdopen(fd, "w", encoding="utf-8") as handle:
            json.dump(data, handle, indent=2, ensure_ascii=False, allow_nan=False)
            handle.write("\n")
            handle.flush()
            os.fsync(handle.fileno())
        os.replace(temporary_path, path)
    except Exception:
        try:
            temporary_path.unlink(missing_ok=True)
        finally:
            raise


def validate_paths(args: argparse.Namespace) -> None:
    input_paths = [args.threats, args.taxonomy, args.cves, args.papers, args.repositories, args.incidents, args.signals]
    resolved_inputs = {path.resolve() for path in input_paths if path.exists()}
    output_files = {
        (args.output_dir / "threat_correlations.json").resolve(),
        (args.output_dir / "evidence_index.json").resolve(),
        (args.output_dir / "correlation_review.json").resolve(),
        (args.output_dir / "correlation_run.json").resolve(),
    }
    conflict = resolved_inputs & output_files
    if conflict:
        raise ValueError(f"Output path conflicts with input: {next(iter(conflict))}")


def validate_loaded_data(threats: Sequence[Threat], evidence: Sequence[Evidence]) -> list[str]:
    warnings: list[str] = []
    ids = [int(threat.threat_id.split("-")[1]) for threat in threats]
    if ids:
        missing = sorted(set(range(min(ids), max(ids) + 1)) - set(ids))
        if missing:
            warnings.append("Missing threat IDs: " + ", ".join(f"T-{value:03d}" for value in missing))
    papers_without_abstract = sum(
        1 for item in evidence if item.evidence_type == "paper" and not item.description
    )
    cves_without_description = sum(
        1 for item in evidence if item.evidence_type == "cve" and not item.description
    )
    if papers_without_abstract:
        warnings.append(f"Papers without abstract: {papers_without_abstract}")
    if cves_without_description:
        warnings.append(f"CVEs without vulnerability description: {cves_without_description}")
    return warnings


def main() -> int:
    args = parse_args()
    configure_logging(args.log_level)
    start = time.time()

    try:
        validate_paths(args)
        LOGGER.info("Loading threat profiles from %s and %s", args.threats, args.taxonomy)
        threats = load_threats(args.threats, args.taxonomy)
        LOGGER.info("Loaded %d unique threats", len(threats))

        LOGGER.info("Loading CVEs from %s", args.cves)
        LOGGER.info("Loading papers from %s", args.papers)
        LOGGER.info(
            "Loading repositories from %s",
            args.repositories,
        )
        evidence_items = load_evidence(args.cves, args.papers, args.repositories, args.incidents, args.signals)
        cve_count = sum(item.evidence_type == "cve" for item in evidence_items)
        paper_count = sum(item.evidence_type == "paper" for item in evidence_items)
        repository_count = sum(
            item.evidence_type == "repository"
            for item in evidence_items
        )
        signal_count = sum(item.evidence_type == "signal" for item in evidence_items)
 
        incident_count = sum(item.evidence_type == "incident" for item in evidence_items)
        assessment_count = sum(item.evidence_type == "assessment" for item in evidence_items)
        narrative_count = sum(item.evidence_type == "narrative" for item in evidence_items)
        LOGGER.info(
            "Loaded %d CVEs, %d papers, %d repositories, %d signals, %d incidents, %d assessments and %d narratives",
            cve_count,
            paper_count,
            repository_count,
            signal_count,
            incident_count,
            assessment_count,
            narrative_count,
        )

        if args.threat_id:
            threats = [threat for threat in threats if threat.threat_id == args.threat_id]
            if not threats:
                raise ValueError(f"Threat not found: {args.threat_id}")
        if args.evidence_id:
            needle = args.evidence_id.casefold()
            evidence_items = [
                item for item in evidence_items if item.evidence_id.casefold() == needle
            ]
            if not evidence_items:
                raise ValueError(f"Evidence not found: {args.evidence_id}")

        warnings = validate_loaded_data(threats, evidence_items)
        for warning in warnings:
            LOGGER.warning(warning)

        if args.validate_only:
            LOGGER.info("Validation completed successfully; no output written")
            return 0

        args.output_dir.mkdir(parents=True, exist_ok=True)
        output_paths = {
            "correlations": args.output_dir / "threat_correlations.json",
            "index": args.output_dir / "evidence_index.json",
            "review": args.output_dir / "correlation_review.json",
            "run": args.output_dir / "correlation_run.json",
        }
        if not args.force:
            existing = [str(path) for path in output_paths.values() if path.exists()]
            if existing:
                raise FileExistsError(
                    "Output files already exist. Use --force to overwrite: " + ", ".join(existing)
                )

        LOGGER.info("Building document-frequency index")
        idf = build_idf(threats, evidence_items)
        results, rejected = correlate(threats, evidence_items, idf, args)

        accepted_count = sum(result.status == "accepted" for result in results)
        review_count = sum(result.status == "review" for result in results)
        unmatched_evidence = 0
        matched_ids = {(r.evidence_type, r.evidence_id.casefold()) for r in results}
        for item in evidence_items:
            if (item.evidence_type, item.evidence_id.casefold()) not in matched_ids:
                unmatched_evidence += 1

        correlations_output = build_threat_correlations(threats, results)
        index_output = build_evidence_index(evidence_items, results)
        review_output = build_review_output(results, rejected)
        run_output = {
            "schema_version": SCHEMA_VERSION,
            "generated_at": iso_now(),
            "duration_seconds": round(time.time() - start, 3),
            "inputs": {
                "threats": {"path": str(args.threats), "sha256": sha256_file(args.threats)},
                "taxonomy": {"path": str(args.taxonomy), "sha256": sha256_file(args.taxonomy)},
                "cves": {"path": str(args.cves), "sha256": sha256_file(args.cves)},
                "papers": {"path": str(args.papers), "sha256": sha256_file(args.papers)},
                "repositories": {
                    "path": str(args.repositories),
                    "sha256": sha256_file(
                        args.repositories
                    ),
                },
                "incidents": {"path": str(args.incidents), "sha256": sha256_file(args.incidents)},
                "signals": {"path": str(args.signals), "sha256": sha256_file(args.signals)},
            },
            "configuration": {
                "accept_threshold": args.accept_threshold,
                "review_threshold": args.review_threshold,
                "top_k": args.top_k,
                "max_accepted_per_evidence": args.max_accepted_per_evidence,
                "include_rejected": args.include_rejected,
                "threat_filter": args.threat_id,
                "evidence_filter": args.evidence_id,
                "matching_mode": "deterministic_explainable_v1",
            },
            "statistics": {
                "threats_loaded": len(threats),
                "cves_loaded": cve_count,
                "papers_loaded": paper_count,
                "repositories_loaded": repository_count,
                "signals_loaded": signal_count,
                "incidents_loaded": incident_count,
                "assessments_loaded": assessment_count,
                "narratives_loaded": narrative_count,
                "evidence_loaded": len(evidence_items),
                "accepted_matches": accepted_count,
                "review_matches": review_count,
                "rejected_candidates_included": len(rejected),
                "unmatched_evidence": unmatched_evidence,
            },
            "warnings": warnings,
            "outputs": {key: str(value) for key, value in output_paths.items()},
        }

        atomic_write_json(output_paths["correlations"], correlations_output)
        atomic_write_json(output_paths["index"], index_output)
        atomic_write_json(output_paths["review"], review_output)
        atomic_write_json(output_paths["run"], run_output)

        LOGGER.info("Accepted matches: %d", accepted_count)
        LOGGER.info("Review matches: %d", review_count)
        LOGGER.info("Unmatched evidence records: %d", unmatched_evidence)
        LOGGER.info("Output written to %s", args.output_dir)
        return 0

    except (FileNotFoundError, FileExistsError, ValueError, OSError) as exc:
        LOGGER.error("%s", exc)
        return 2
    except KeyboardInterrupt:
        LOGGER.error("Interrupted")
        return 130
    except Exception:
        LOGGER.exception("Unexpected failure")
        return 1


if __name__ == "__main__":
    sys.exit(main())
