#!/usr/bin/env python3
"""
Turn the two free-text analyses per repo (document_pipeline + intelligence_pipeline)
into structured JSON and merge it into dataset.json.

Existing fields (name, score, confidence, surfaces) are kept untouched, so the current
dashboard pages keep working. New fields are added next to them.

Usage:
  python3 enrich_dataset.py --raw /path/to/raw_jsons --dataset dataset.json --out dataset.rich.json

--raw can be a folder with one .json per repo (file name = repo name) or a single
.json file holding a list or a {name: {...}} mapping.
"""
import argparse, json, re, sys
from pathlib import Path

LEVELS = ["low", "medium", "high", "critical"]
LEVEL_RE = re.compile(r"\b(critical|high|medium|low)\b", re.I)

# ---------------------------------------------------------------- parsing
def clean(s):
    s = re.sub(r"\*\*|__|`", "", s)
    return re.sub(r"\s+", " ", s).strip(" :-\t")

def split_sections(text):
    """Split markdown on headings -> list of (normalised_title, body)."""
    parts, title, buf = [], None, []
    for line in text.splitlines():
        m = re.match(r"^\s*#{1,6}\s+(.*\S)\s*$", line)
        if m:
            if title is not None or buf:
                parts.append((title or "", "\n".join(buf)))
            title, buf = clean(re.sub(r"^\d+[.)]\s*", "", m.group(1))).lower(), []
        else:
            buf.append(line)
    parts.append((title or "", "\n".join(buf)))
    return parts

ITEM_RE = re.compile(r"^\s*(?:[-*\u2022]|\d+[.)])\s+(.*)$")
BOLD_LEAD_RE = re.compile(r"^\*\*(.+?)\*\*\s*:?\s*(.*)$|^(.{2,60}?):\s+(.*)$")

def items(body):
    """Bullet/numbered items -> [{'title','text'}]. Empty parent bullets (pure headers) are dropped."""
    out = []
    for line in body.splitlines():
        m = ITEM_RE.match(line)
        if m:
            raw = m.group(1).strip()
            b = BOLD_LEAD_RE.match(raw)
            if b:
                title = clean(b.group(1) or b.group(3))
                text = clean(b.group(2) if b.group(1) else b.group(4))
            else:
                title, text = "", clean(raw)
            out.append({"title": title.rstrip(":"), "text": text})
        elif line.strip() and out and not line.lstrip().startswith("#"):
            out[-1]["text"] = (out[-1]["text"] + " " + clean(line)).strip()   # wrapped line
    # a bullet with a title but no text is a header like "Fuzzing Scenarios:" -> drop it
    return [i for i in out if i["text"]]

def kv(body):
    d = {}
    for i in items(body):
        if i["title"]:
            d[i["title"].lower()] = i["text"]
    return d

def level_in(s, last=False):
    m = LEVEL_RE.findall(s or "")
    return (m[-1] if last else m[0]).capitalize() if m else None

L = r"(critical|high|medium|low)"
VERDICT_PATTERNS = [
    r"(?:adjust|rais|elevat|increas|lower|reduc|downgrad|upgrad|revis|chang|updat|set|rat|escalat)(?:ed|ing)?\s+(?:the\s+)?(?:(?:risk|threat|final)\s+)?(?:score\s+|rating\s+)?(?:up\s+|down\s+)?(?:to|as)\s+\**" + L + r"\b",
    r"(?:risk|threat|final)\s+(?:score|rating|level)\s*(?:is|=|:|of|remains|stays|stands at|should be)?\s*(?:now\s+)?\**" + L + r"\b",
    r"(?:verdict|conclusion|classified as|categori[sz]ed as|assessed as|rated)\s*:?\s*(?:as\s+)?\**" + L + r"\b",
    r"(?:remains?|stays?|stay|agree[sd]? (?:with|on)[^.]{0,40}?)\s+\**" + L + r"\b",
]

def verdict_in(text, allow_bold=False):
    """Explicit verdict of the second model, or None. Bare level words in prose never count."""
    hits = []
    for pat in VERDICT_PATTERNS:
        for m in re.finditer(pat, text or "", re.I):
            hits.append((m.start(), m.group(1)))
    if allow_bold:
        for m in re.finditer(r"\*\*" + L + r"\*\*", text or "", re.I):
            hits.append((m.start(), m.group(1)))
    return max(hits)[1].capitalize() if hits else None

def pick(sections, *keys):
    for t, b in sections:
        if any(k in t for k in keys):
            return b
    return None

def csv(s):
    return [clean(x) for x in re.split(r",|;| and (?=[A-Z])", s or "") if clean(x)]

def titles_only(lst):                       # for compact counts / lists
    return [i["title"] or i["text"][:60] for i in lst]

# ---------------------------------------------------------------- per repo
def parse_repo(doc, intel):
    out = {}
    ds, ints = split_sections(doc or ""), split_sections(intel or "")

    # --- protocol classification
    cls = kv(pick(ds, "protocol classification", "classification") or "")
    out["primary_focus"] = cls.get("primary focus")
    out["secondary_focus"] = csv(cls.get("secondary focus"))
    out["technologies"] = csv(cls.get("technologies used") or cls.get("technologies"))

    # --- first score
    sb = pick(ds, "threat score", "risk score") or ""
    sc_line = next((l for l in sb.splitlines() if re.search(r"threat score|score", l, re.I)), sb)
    out["score_initial"] = level_in(sc_line) or level_in(sb)
    jm = re.search(r"justification\s*:?\*{0,2}\s*(.+)", sb, re.I | re.S)
    out["justification"] = clean(jm.group(1)) if jm else clean(sb)

    # --- vulnerability / analysis lists
    out["vulnerabilities"] = items(pick(ds, "vulnerabilit") or "")
    out["implementation_challenges"] = [
        i for i in items(pick(ds, "deep analysis") or "")
        if re.search(r"challenge|scalab|integration", i["title"], re.I)]
    out["fuzzing"] = [dict(i, source="initial") for i in items(pick(ds, "fuzzing") or "")]
    out["next_steps"] = [i["title"] + (": " if i["title"] and i["text"] else "") + i["text"]
                         for i in items(pick(ds, "next steps", "recommend") or "")]

    # --- intelligence pipeline
    out["insights"] = items(pick(ints, "additional insights", "insights") or "")
    out["hidden_attack_surfaces"] = items(pick(ints, "hidden attack surface") or "")
    out["weaknesses"] = items(pick(ints, "weakness") or "")
    out["fuzzing"] += [dict(i, source="advanced") for i in items(pick(ints, "fuzzing") or "")]

    # The second model only gives a verdict when it disagrees; no verdict = it agrees with the first.
    fb = pick(ints, "final risk score", "final score", "risk score justification", "verdict") or ""
    verdict = verdict_in(fb, allow_bold=True) if fb else verdict_in(intel)
    out["score_verdict"] = verdict
    out["final_justification"] = clean(fb) if verdict else None
    out["score_final"] = verdict or out["score_initial"]
    out["score_source"] = "intelligence" if verdict else "document"
    out["second_agreed"] = verdict is None or verdict == out["score_initial"]

    # --- derived
    i, f = out["score_initial"], out["score_final"]
    rank = lambda x: LEVELS.index(x.lower())
    out["score_changed"] = bool(i and f and i != f)
    out["escalated"] = bool(i and f and rank(f) > rank(i))
    out["downgraded"] = bool(i and f and rank(f) < rank(i))
    out["threat_count"] = len(out["vulnerabilities"]) + len(out["hidden_attack_surfaces"]) + len(out["weaknesses"])
    out["fuzzing_count"] = len(out["fuzzing"])
    return out


# ---------------------------------------------------------------- attack surfaces
# Matched with word boundaries against the first model's "Protocol Classification"
# (primary focus, secondary focus, technologies) and the repo part of the name.
# Edit freely: (label, regex, case_sensitive_for_text). Names are always matched ignoring case.
SURFACE_RULES = [
    ("CAN Bus",              r"\bCAN\b|\bCAN[- ]?FD\b|\bCAN[- ]?bus\b|\bSocketCAN\b|\bCANbus\b", True),
    ("UDS Diagnostics",      r"\bUDS\b|ISO[- ]?14229|unified diagnostic", False),
    ("ISO-TP",               r"ISO[- ]?TP|ISO[- ]?15765|\bISOTP\b", False),
    ("DoIP",                 r"\bDoIP\b|ISO[- ]?13400", False),
    ("SOME/IP",              r"SOME[/ -]?IP|\bSOMEIP\b", False),
    ("J1939 / Heavy Vehicle", r"J1939|\bISOBUS\b|heavy[- ]vehicle", False),
    ("OBD-II / J2534",       r"\bOBD[- ]?(?:II|2)?\b|J2534|pass[- ]?thru", False),
    ("LIN / FlexRay",        r"\bLIN\b|\bFlexRay\b", True),
    ("ECU Firmware",         r"\bECUs?\b|firmware|bootloader|\bAUTOSAR\b|\bSTM32\b", False),
    ("Fuzzing Engine",       r"fuzz", False),
    ("KeyFob / RF",          r"key[- ]?fob|\bRKE\b|\bPEPS\b|immobili[sz]er|\bRF\b|\bSDR\b|software[- ]defined radio", False),
    ("Bluetooth / Wi-Fi",    r"\bBluetooth\b|\bBLE\b|blue[- ]?tap|\bWi-?Fi\b", False),
    ("Cellular / V2X",       r"\bV2X\b|\bV2V\b|\bV2I\b|\bDSRC\b|\bLTE\b|\b[3-6]G\b|cellular", False),
    ("Telematics / Routing", r"telematics|\brouting\b|\bGPS\b|\bGNSS\b|\bTCU\b", False),
    ("Infotainment",         r"infotainment|android auto|carplay|\bIVI\b|head unit", False),
    ("OTA Updates",          r"\bOTA\b|over[- ]the[- ]air|\bUptane\b", False),
    ("Hardware Security",    r"\bHSM\b|secure element|secure boot|\bTPM\b|trustzone", False),
    ("Diagnostic Gateway",   r"gateway", False),
    ("Automotive AI",        r"\bAI\b|\bML\b|\bLLM\b|machine learning|deep learning|neural", False),
    ("Automotive Blockchain", r"blockchain", False),
    ("Dashboard Cluster",    r"instrument cluster|dashboard cluster|speedometer", False),
]
GENERIC_SURFACE = "General Automotive Surface"

def detect_surfaces(name, rich):
    """Attack surfaces from the analysed content, not from substrings of the name."""
    repo = name.split("__", 1)[-1]
    name_text = re.sub(r"[_\-.]+", " ", repo)          # 'python-can' -> 'python can'
    text = " ".join([rich.get("primary_focus") or "", *(rich.get("secondary_focus") or []),
                     *(rich.get("technologies") or [])])
    found = []
    for label, pat, cs in SURFACE_RULES:
        if re.search(pat, text, 0 if cs else re.I) or re.search(pat, name_text, re.I):
            found.append(label)
    return found or [GENERIC_SURFACE]

# ---------------------------------------------------------------- IO
def load_raw(path):
    p, res = Path(path), {}
    if p.is_dir():
        for f in sorted(p.glob("*.json")):
            try:
                d = json.loads(f.read_text(encoding="utf-8"))
            except Exception as e:
                print(f"! cannot read {f.name}: {e}", file=sys.stderr); continue
            res[d.get("name", f.stem) if isinstance(d, dict) else f.stem] = d
    else:
        d = json.loads(p.read_text(encoding="utf-8"))
        if isinstance(d, list):
            res = {x["name"]: x for x in d}
        elif "document_pipeline" in d:
            res = {p.stem: d}
        else:
            res = d
    return res

def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--raw", required=True)
    ap.add_argument("--dataset", default="dataset.json")
    ap.add_argument("--out", default="dataset.rich.json")
    ap.add_argument("--sync-score", action="store_true",
                    help="overwrite 'score' with the final score (second model's verdict, else the first model's)")
    a = ap.parse_args()

    raw = load_raw(a.raw)
    base = json.loads(Path(a.dataset).read_text(encoding="utf-8")) if Path(a.dataset).exists() else []
    base = base if isinstance(base, list) else base.get("repos", [])
    by_name = {r["name"]: r for r in base}

    merged, no_raw, new = [], [], []
    for name, r in by_name.items():
        if name in raw:
            rich = parse_repo(raw[name].get("document_pipeline"), raw[name].get("intelligence_pipeline"))
            merged.append({**r, "owner": name.split("__")[0], **rich})
        else:
            merged.append({**r, "owner": name.split("__")[0]}); no_raw.append(name)
    for name in raw.keys() - by_name.keys():
        rich = parse_repo(raw[name].get("document_pipeline"), raw[name].get("intelligence_pipeline"))
        merged.append({"name": name, "owner": name.split("__")[0],
                       "score": rich["score_final"] or rich["score_initial"], **rich}); new.append(name)

    if a.sync_score:
        for m in merged:
            if m.get("score_final"): m["score"] = m["score_final"]
    Path(a.out).write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding="utf-8")

    # --- report so you can see what the parser could not read
    rich_rows = [m for m in merged if "score_initial" in m]
    def miss(k): return [m["name"] for m in rich_rows if not m.get(k)]
    print(f"{len(merged)} repos written to {a.out}")
    print(f"  parsed from raw: {len(rich_rows)}   without raw file: {len(no_raw)}   only in raw: {len(new)}")
    for k in ("score_initial", "score_final", "vulnerabilities", "fuzzing", "next_steps", "primary_focus"):
        m = miss(k)
        if m: print(f"  missing {k}: {len(m)}  e.g. {m[:3]}")
    n = len(rich_rows) or 1
    gave = sum(1 for m in rich_rows if m.get("score_verdict"))
    esc = sum(1 for m in rich_rows if m.get("escalated"))
    down = sum(1 for m in rich_rows if m.get("downgraded"))
    print(f"  second model gave a verdict for {gave}/{n}; no verdict (agrees) for {n - gave}")
    print(f"  of those verdicts: {esc} raised the score, {down} lowered it, {gave - esc - down} confirmed it")
    if not a.sync_score:
        off = [m["name"] for m in rich_rows if m.get("score_final") and m["score"] != m["score_final"]]
        if off: print(f"  ! 'score' in dataset.json differs from the final score for {len(off)} repos, e.g. {off[:3]}"
                      f"  (use --sync-score to fix)")

if __name__ == "__main__":
    main()
