#!/usr/bin/env python3
"""Generate conservative UN R155 review mappings with concept and requirement layers.

V3 preserves the V2-compatible ``mappings`` array for Annex 5 Part A candidates,
but adds:

* phrase-aware technical concepts;
* attack-path bridges between a threat consequence and Annex 5 mechanisms;
* negative compatibility gates to reduce obvious false positives;
* R155 CSMS/type-approval relations outside Annex 5;
* explicit review-only output. No mapping is automatically accepted.

The output remains suitable for build_r155_test_pages_v2_2.py because the
existing top-level ``mappings`` structure is retained.
"""
from __future__ import annotations

import argparse
import json
import re
import sys
from collections import defaultdict
from pathlib import Path
from typing import Any

STOP = {
    "and", "or", "the", "of", "to", "a", "an", "in", "on", "for", "with",
    "through", "from", "as", "by", "into", "across", "within", "regarding",
    "related", "vehicle", "vehicles", "cyber", "cybersecurity", "attack",
    "attacks", "system", "systems",
}

WEAK_TOKENS = {
    "end", "loss", "lost", "party", "third", "support", "service", "services",
    "information", "data", "used", "using", "related", "external", "hosted",
    "application", "applications", "software", "security", "risk", "risks",
    "management", "manufacturer", "manufacturers", "oem", "impact", "function",
    "functions", "component", "components", "digital", "insufficient", "critical",
}

# Concepts describe meaning, not direct legal conclusions. Each rule needs either
# an exact phrase or at least min_aliases aliases in the same text.
CONCEPTS: dict[str, dict[str, Any]] = {
    "end_of_support": {
        "phrases": ("end of support", "end-of-support", "end of life", "end-of-life"),
        "aliases": {"unsupported", "obsolete", "obsolescence", "legacy", "eol", "eos"},
        "min_aliases": 1,
    },
    "cloud_information_loss": {
        "phrases": ("loss of information in the cloud", "cloud data loss", "cloud information loss"),
        "aliases": {"cloud", "backup", "availability", "loss"},
        "min_aliases": 2,
    },
    "backend_compromise": {
        "phrases": ("back-end server", "backend server", "back end server", "server compromise"),
        "aliases": {"backend", "server", "cloud", "remote", "internet"},
        "min_aliases": 2,
    },
    "backend_disruption": {
        "phrases": ("backend outage", "back-end outage", "server disruption", "services being disrupted"),
        "aliases": {"backend", "server", "outage", "disruption", "availability"},
        "min_aliases": 2,
    },
    "supplier_dependency": {
        "phrases": ("supply chain", "third-party", "third party", "service provider", "tier 1", "tier 2"),
        "aliases": {"supplier", "suppliers", "dependency", "dependencies", "provider", "contracted"},
        "min_aliases": 1,
    },
    "vehicle_control": {
        "phrases": ("vehicle takeover", "loss of control", "gain control", "remote control", "unsafe vehicle"),
        "aliases": {"control", "takeover", "steering", "braking", "accelerating", "immobilize", "disable"},
        "min_aliases": 1,
    },
    "remote_operation": {
        "phrases": ("remotely operate", "remote operation", "remote command"),
        "aliases": {"remote", "operate", "command", "control", "telematics"},
        "min_aliases": 2,
    },
    "autonomous_driving": {
        "phrases": ("autonomous driving", "automated driving", "self driving", "driving functions"),
        "aliases": {"autonomous", "automated", "adas", "perception", "lidar", "camera"},
        "min_aliases": 1,
    },
    "sensor_manipulation": {
        "phrases": ("sensor input", "sensor inputs", "sensor spoofing", "physical manipulation"),
        "aliases": {"sensor", "sensors", "camera", "lidar", "radar", "perception", "manipulation", "spoofing"},
        "min_aliases": 2,
    },
    "infotainment_entry": {
        "phrases": ("infotainment system", "infotainment unit", "head unit"),
        "aliases": {"infotainment", "bluetooth", "usb", "wifi", "wi-fi", "media"},
        "min_aliases": 1,
    },
    "external_connectivity": {
        "phrases": ("external connectivity", "external interface", "diagnostic port", "obd port"),
        "aliases": {"bluetooth", "usb", "wifi", "telematics", "diagnostic", "obd", "interface", "port"},
        "min_aliases": 1,
    },
    "communication_channel": {
        "phrases": ("communication channel", "communications channel", "session hijacking", "man in the middle"),
        "aliases": {"communication", "channel", "message", "messages", "network", "replay", "spoofing"},
        "min_aliases": 2,
    },
    "code_data_manipulation": {
        "phrases": ("code injection", "data injection", "manipulation of vehicle data", "manipulation of vehicle code"),
        "aliases": {"injection", "overwrite", "erasure", "modify", "modified", "code", "firmware"},
        "min_aliases": 2,
    },
    "network_separation": {
        "phrases": ("network separation", "network segmentation", "circumvention of network separation"),
        "aliases": {"segmentation", "separation", "gateway", "pivot", "isolation", "circumvention"},
        "min_aliases": 1,
    },
    "unauthorized_access": {
        "phrases": ("unauthorized access", "unauthorised access", "abuse of privileges", "privilege abuse"),
        "aliases": {"unauthorized", "unauthorised", "privilege", "credentials", "access", "backdoor"},
        "min_aliases": 2,
    },
    "data_exfiltration": {
        "phrases": ("data exfiltration", "extract data", "information disclosure", "eavesdropping"),
        "aliases": {"exfiltration", "extract", "disclosure", "eavesdropping", "espionage"},
        "min_aliases": 1,
    },
    "privacy_surveillance": {
        "phrases": ("personal data", "privacy information", "location tracking", "vehicle occupants", "behavioral tracking"),
        "aliases": {"privacy", "tracking", "surveillance", "occupant", "occupants", "location", "monitoring"},
        "min_aliases": 1,
    },
    "botnet_distributed_attack": {
        "phrases": ("use in botnets", "vehicle botnet", "distributed attack", "distributed denial of service"),
        "aliases": {"botnet", "botnets", "distributed", "ddos", "scalable", "endpoints"},
        "min_aliases": 1,
    },
    "service_availability": {
        "phrases": ("service outage", "services outage", "mobility services", "stop functioning"),
        "aliases": {"outage", "availability", "disruption", "interrupt", "interruption", "services"},
        "min_aliases": 1,
    },
    "cryptographic_weakness": {
        "phrases": ("short encryption keys", "cryptographic compromise", "long validity", "weak cryptography"),
        "aliases": {"encryption", "cryptographic", "cryptography", "keys", "certificate", "certificates"},
        "min_aliases": 2,
    },
    "physical_sensor_hardware": {
        "phrases": ("electronic hardware", "replacement of electronic hardware", "physical manipulation"),
        "aliases": {"hardware", "physical", "replacement", "sensor", "sensors"},
        "min_aliases": 2,
    },
    "sensitive_data_loss": {
        "phrases": ("sensitive data", "loss or compromise of sensitive data", "data breach"),
        "aliases": {"sensitive", "confidentiality", "privacy", "data", "theft"},
        "min_aliases": 2,
    },
}

# A threat concept can reach a node concept through a plausible attack path.
# Values are conservative boosts, not acceptance decisions.
ATTACK_PATHS: dict[str, dict[str, float]] = {
    "infotainment_entry": {
        "external_connectivity": 0.30,
        "communication_channel": 0.26,
        "code_data_manipulation": 0.28,
        "network_separation": 0.30,
        "unauthorized_access": 0.22,
    },
    "vehicle_control": {
        "remote_operation": 0.28,
        "network_separation": 0.26,
        "code_data_manipulation": 0.24,
        "communication_channel": 0.18,
        "sensor_manipulation": 0.20,
    },
    "autonomous_driving": {
        "sensor_manipulation": 0.32,
        "network_separation": 0.23,
        "code_data_manipulation": 0.24,
        "communication_channel": 0.18,
        "cryptographic_weakness": 0.12,
    },
    "botnet_distributed_attack": {
        "unauthorized_access": 0.24,
        "external_connectivity": 0.24,
        "backend_compromise": 0.20,
        "code_data_manipulation": 0.18,
        "communication_channel": 0.18,
    },
    "service_availability": {
        "backend_disruption": 0.34,
        "backend_compromise": 0.22,
        "communication_channel": 0.18,
        "external_connectivity": 0.14,
    },
    "data_exfiltration": {
        "backend_compromise": 0.28,
        "communication_channel": 0.24,
        "unauthorized_access": 0.26,
        "external_connectivity": 0.18,
    },
    "privacy_surveillance": {
        "data_exfiltration": 0.30,
        "unauthorized_access": 0.24,
        "communication_channel": 0.18,
        "backend_compromise": 0.18,
    },
    "supplier_dependency": {
        "backend_compromise": 0.16,
        "unauthorized_access": 0.14,
        "code_data_manipulation": 0.12,
    },
}

# These pairs are usually consequence-incompatible without additional evidence.
INCOMPATIBLE_CONCEPTS = {
    ("vehicle_control", "sensitive_data_loss"),
    ("autonomous_driving", "sensitive_data_loss"),
    ("service_availability", "sensitive_data_loss"),
    ("botnet_distributed_attack", "sensitive_data_loss"),
}

AREA_TO_CATEGORY = {f"4.3.{number}": f"4.3.{number}" for number in range(1, 8)}

# Main-text R155 relations. These are review candidates distinct from Annex 5.
REQUIREMENT_RULES: tuple[dict[str, Any], ...] = (
    {
        "relation_id": "R155-CSMS-GOVERNANCE",
        "layer": "csms",
        "references": ["2.3", "7.2.2.2(a-d)"],
        "title": "Cybersecurity governance, responsibilities and risk management processes",
        "phrases": ("insufficient priority", "regulation and oversight", "safety/security", "governance"),
        "aliases": {"oversight", "governance", "priority", "compliance", "regulation", "manufacturer"},
        "min_aliases": 2,
    },
    {
        "relation_id": "R155-AUTHORITY-COMPETENCE",
        "layer": "type_approval",
        "references": ["5.3.1"],
        "title": "Competent personnel and uniform evaluation by Approval Authorities and Technical Services",
        "phrases": ("insufficient oversight", "vehicle authority", "technical service", "market surveillance"),
        "aliases": {"oversight", "authority", "technical", "service", "competence", "knowledge", "complexity"},
        "min_aliases": 2,
    },
    {
        "relation_id": "R155-PRE-APPROVAL-TESTING",
        "layer": "type_approval",
        "references": ["5.1.2", "5.1.3(d)", "7.3.6"],
        "title": "Appropriate and sufficient testing before type approval",
        "phrases": ("inadequate testing prior to market approval", "testing prior to approval", "insufficient testing"),
        "aliases": {"testing", "test", "approval", "market", "verification"},
        "min_aliases": 2,
    },
    {
        "relation_id": "R155-SUPPLIER-DEPENDENCIES",
        "layer": "csms",
        "references": ["5.1.1(a)", "7.2.2.5", "7.3.2"],
        "title": "Management of supplier and service-provider dependencies",
        "phrases": ("vehicle ecosystem", "supply chain", "through suppliers", "supplier dependencies"),
        "aliases": {"supplier", "suppliers", "ecosystem", "dependency", "dependencies", "provider"},
        "min_aliases": 1,
    },
    {
        "relation_id": "R155-RISK-ASSESSMENT-OTHER-THREATS",
        "layer": "vehicle_type",
        "references": ["7.3.3"],
        "title": "Exhaustive vehicle-type risk assessment including external systems and other relevant risks",
        "phrases": ("societal disruption", "systemic risk", "shared dependency", "cascading effects"),
        "aliases": {"societal", "systemic", "cascading", "external", "interactions", "fleet"},
        "min_aliases": 2,
    },
    {
        "relation_id": "R155-MONITORING-RESPONSE",
        "layer": "csms",
        "references": ["7.2.2.2(g-h)", "7.2.2.4", "7.4"],
        "title": "Continual monitoring, response and reporting over the vehicle lifecycle",
        "phrases": ("monitoring", "incident response", "post-production", "new cyber threats"),
        "aliases": {"monitoring", "detect", "response", "incident", "reporting", "lifecycle"},
        "min_aliases": 2,
    },
)


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description="Build UN R155 V3 review mappings")
    parser.add_argument("--register", type=Path, default=Path("data/threats_register.json"))
    parser.add_argument("--annex5", type=Path, default=Path("regulatory/r155/annex5.json"))
    parser.add_argument("--output", type=Path, default=Path("regulatory/r155/r155_threat_mappings.json"))
    parser.add_argument("--min-score", type=float, default=0.20)
    parser.add_argument("--max-candidates", type=int, default=8)
    return parser.parse_args()


def load(path: Path) -> dict[str, Any]:
    try:
        data = json.loads(path.read_text(encoding="utf-8"))
    except FileNotFoundError as exc:
        raise ValueError(f"Input file not found: {path}") from exc
    except json.JSONDecodeError as exc:
        raise ValueError(f"Invalid JSON in {path}: {exc}") from exc
    if not isinstance(data, dict):
        raise ValueError(f"Expected JSON object in {path}")
    return data


def normalize(value: Any) -> str:
    text = str(value or "").lower().replace("–", "-").replace("—", "-")
    text = re.sub(r"[-_/]+", " ", text)
    return re.sub(r"\s+", " ", text).strip()


def tokens(value: Any) -> set[str]:
    return {
        word for word in re.findall(r"[a-z0-9]+", normalize(value))
        if len(word) > 2 and word not in STOP
    }


def threat_text(threat: dict[str, Any]) -> str:
    values = [
        threat.get("title", ""), threat.get("title_nl", ""), threat.get("description", ""),
        threat.get("category", ""), threat.get("annex5_area", ""),
        threat.get("domain_rationale", ""), threat.get("impact_rationale", ""),
        threat.get("likelihood_rationale", ""), threat.get("risk_rationale", ""),
        threat.get("priority", ""), " ".join(threat.get("terms", [])),
        " ".join(threat.get("surfaces", [])), " ".join(threat.get("tags", [])),
    ]
    return " ".join(str(value) for value in values if value)


def node_text(node: dict[str, Any]) -> str:
    return " ".join(
        str(value) for value in (
            node.get("label", ""), node.get("category_title", ""),
            node.get("description", ""), node.get("example", ""),
        ) if value
    )


def rule_hits(text: str, rule: dict[str, Any]) -> bool:
    normalized = normalize(text)
    if any(normalize(phrase) in normalized for phrase in rule.get("phrases", ())):
        return True
    alias_hits = tokens(normalized) & set(rule.get("aliases", set()))
    return len(alias_hits) >= int(rule.get("min_aliases", 2))


def concept_hits(text: str) -> set[str]:
    return {name for name, rule in CONCEPTS.items() if rule_hits(text, rule)}


def explicit_area(threat: dict[str, Any]) -> str | None:
    raw = str(threat.get("annex5_area") or "")
    return next((category for area, category in AREA_TO_CATEGORY.items() if area in raw), None)


def best_attack_path(threat_concepts: set[str], node_concepts: set[str]) -> tuple[float, list[str]]:
    score = 0.0
    paths: list[str] = []
    for source in sorted(threat_concepts):
        for target in sorted(node_concepts):
            boost = ATTACK_PATHS.get(source, {}).get(target, 0.0)
            if boost > score:
                score = boost
                paths = [f"{source}->{target}"]
            elif boost and boost == score:
                paths.append(f"{source}->{target}")
    return score, paths


def incompatible(threat_concepts: set[str], node_concepts: set[str]) -> bool:
    return any((left, right) in INCOMPATIBLE_CONCEPTS for left in threat_concepts for right in node_concepts)


def candidate_score(
    threat: dict[str, Any], node: dict[str, Any], explicit: str | None
) -> tuple[float, list[str], list[str], list[str], str] | None:
    threat_raw = threat_text(threat)
    node_raw = node_text(node)
    threat_tokens = tokens(threat_raw)
    node_tokens = tokens(node_raw)
    overlap = threat_tokens & node_tokens
    strong_overlap = overlap - WEAK_TOKENS
    threat_concepts = concept_hits(threat_raw)
    node_concepts = concept_hits(node_raw)
    shared_concepts = threat_concepts & node_concepts

    if explicit and node.get("category_reference") == explicit:
        return 0.70, sorted(overlap), sorted(shared_concepts), [], "explicit_annex5_area"

    path_score, paths = best_attack_path(threat_concepts, node_concepts)

    # Reject obvious consequence mismatch unless stronger evidence exists.
    if incompatible(threat_concepts, node_concepts) and not shared_concepts and len(strong_overlap) < 2 and not paths:
        return None

    if shared_concepts:
        score = 0.50 + min(0.18, 0.07 * len(shared_concepts))
        score += min(0.10, 0.03 * len(strong_overlap))
        return min(score, 0.99), sorted(overlap), sorted(shared_concepts), paths, "shared_semantic_concept"

    if path_score:
        score = 0.40 + path_score + min(0.09, 0.03 * len(strong_overlap))
        return min(score, 0.95), sorted(overlap), [], paths, "attack_path_bridge"

    if len(strong_overlap) < 2:
        return None

    union = threat_tokens | node_tokens
    jaccard = len(strong_overlap) / len(union) if union else 0.0
    score = jaccard + min(0.24, 0.08 * len(strong_overlap))
    return min(score, 0.80), sorted(overlap), [], [], "strong_lexical_seed"


def requirement_relations(threat: dict[str, Any]) -> list[dict[str, Any]]:
    raw = threat_text(threat)
    relations: list[dict[str, Any]] = []
    for rule in REQUIREMENT_RULES:
        if not rule_hits(raw, rule):
            continue
        relations.append({
            "relation_id": f"{rule['relation_id']}-{threat['id']}",
            "threat_id": threat["id"],
            "threat_title": threat.get("title", ""),
            "regulation_id": "UN-R155",
            "layer": rule["layer"],
            "references": list(rule["references"]),
            "title": rule["title"],
            "status": "review",
            "relationship": "related",
            "rationale": "Candidate generated from threat semantics and the R155 requirement layer; expert review required.",
            "review": {"decision": None, "reviewed_by": None, "reviewed_at": None, "note": None},
        })
    return relations


def main() -> int:
    config = parse_args()
    register = load(config.register)
    annex = load(config.annex5)
    threats = register.get("threats", [])
    nodes = [node for node in annex.get("nodes", []) if isinstance(node, dict) and node.get("part") == "A"]
    if not isinstance(threats, list) or not nodes:
        raise ValueError("Threat register or Annex 5 Part A nodes are empty")

    rows: list[dict[str, Any]] = []
    relations: list[dict[str, Any]] = []
    rejected_pairs = 0
    counts: dict[str, int] = {}
    relation_counts: dict[str, int] = {}

    for threat in threats:
        if not isinstance(threat, dict) or not threat.get("id"):
            continue
        tid = str(threat["id"])
        explicit = explicit_area(threat)
        candidates = []
        for node in nodes:
            result = candidate_score(threat, node, explicit)
            if result is None:
                rejected_pairs += 1
                continue
            score, overlap, concepts, paths, basis = result
            if score < config.min_score:
                continue
            candidates.append((score, node, overlap, concepts, paths, basis))

        candidates.sort(
            key=lambda item: (item[0], len(item[4]), len(item[3]), len(item[2])),
            reverse=True,
        )
        selected = candidates if explicit else candidates[: config.max_candidates]
        counts[tid] = len(selected)

        for score, node, overlap, concepts, paths, basis in selected:
            rows.append({
                "mapping_id": f"R155-{tid}-{str(node['reference']).replace('.', '_')}",
                "threat_id": tid,
                "threat_title": threat.get("title", ""),
                "regulatory_node_id": node["id"],
                "regulatory_reference": node["reference"],
                "regulatory_category": node.get("category_reference"),
                "relationship": "related",
                "status": "review",
                "score": round(score, 3),
                "basis": basis,
                "matched_terms": overlap,
                "matched_concepts": concepts,
                "attack_paths": paths,
                "mitigation_nodes": node.get("related_mitigations", []),
                "rationale": (
                    f"Threat register explicitly identifies Annex 5 area {explicit}."
                    if basis == "explicit_annex5_area"
                    else "Candidate generated from shared technical concepts, an attack-path bridge, or multiple non-generic terms; expert review required."
                ),
                "review": {"decision": None, "reviewed_by": None, "reviewed_at": None, "note": None},
            })

        current_relations = requirement_relations(threat)
        relation_counts[tid] = len(current_relations)
        relations.extend(current_relations)

    threat_ids = [str(t["id"]) for t in threats if isinstance(t, dict) and t.get("id")]
    assessments = []
    for tid in threat_ids:
        annex_count = counts.get(tid, 0)
        requirement_count = relation_counts.get(tid, 0)
        if annex_count:
            classification = "annex5_candidates_generated"
        elif requirement_count:
            classification = "r155_requirement_relation_without_annex5_candidate"
        else:
            classification = "no_direct_mapping_generated"
        assessments.append({
            "threat_id": tid,
            "classification": classification,
            "annex5_candidate_count": annex_count,
            "r155_requirement_relation_count": requirement_count,
            "expert_review_required": True,
        })

    output = {
        "schema_version": "2.0",
        "mapping_set_id": "NCRC-R155-REVIEW-3",
        "regulation_id": "UN-R155",
        "annex_id": "UN-R155-ANNEX-5",
        "status": "review",
        "method": "phrase-aware concepts, conservative attack-path bridges, compatibility gates, and separate R155 requirement-layer candidate generation",
        "source_threat_register": {"file": str(config.register), "threat_count": len(threat_ids)},
        "regulatory_source": {"file": str(config.annex5), "part_a_reference_count": len(nodes)},
        "rules": {
            "accepted_created": False,
            "legal_compliance_inference": False,
            "expert_review_required": True,
            "recommended_independent_expert_reviews": 3,
            "weak_tokens_cannot_create_candidate": sorted(WEAK_TOKENS),
            "minimum_non_generic_overlap": 2,
        },
        "summary": {
            "annex5_mapping_count": len(rows),
            "r155_requirement_relation_count": len(relations),
            "threats_with_annex5_candidates": sum(value > 0 for value in counts.values()),
            "threats_with_requirement_relations": sum(value > 0 for value in relation_counts.values()),
            "threats_without_any_candidate": [
                tid for tid in threat_ids if not counts.get(tid, 0) and not relation_counts.get(tid, 0)
            ],
            "candidate_pairs_rejected": rejected_pairs,
        },
        "mappings": rows,
        "regulatory_relations": relations,
        "threat_assessments": assessments,
    }

    config.output.parent.mkdir(parents=True, exist_ok=True)
    config.output.write_text(json.dumps(output, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
    print(json.dumps(output["summary"], indent=2))
    return 0


if __name__ == "__main__":
    try:
        raise SystemExit(main())
    except (ValueError, OSError) as exc:
        print(f"ERROR: {exc}", file=sys.stderr)
        raise SystemExit(2)
