#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
evidence_intel.py - joins CVEs, papers and GitHub repos into EVIDENCE for the NCRC threat register

Inputs (all optional, whatever exists is used):
  zero_day_threats/zero_day.json   from zerod.py
  papers_threats/papers.json       from papers_intel.py
  github_threats/github.json       from github_intel.py
  dataset.json                     your existing (LLM-analysed) repository dataset
  threat_register.json             YOUR threat register (build_register.py creates it from the workbook; edit it)

Outputs:
  evidence_threats/register.json         threats + linked evidence + strength + gaps  (-> threat_register.html)
  evidence_threats/dataset_enriched.json dataset.json + verified metadata, fixed surfaces, fallback scores

Matching: a threat's 'surfaces' (technical surface, weaker link) and 'terms' (words in the evidence text, stronger
link) select evidence; 'evidence_types' says which source types can prove the threat at all ([] = organisational).
Every attached item records WHY it matched (match_via), so each link can be checked.

Principle: facts and hypotheses are never mixed.
  grade A  confirmed vulnerability (CVE, CVSS >= 7, scored)
  grade B  CVE medium / awaiting score, or peer-reviewed research (top venue)
  grade C  other research (preprints etc.) or public tooling on GitHub
  grade D  LLM-generated hypothesis (kept, but clearly labelled; never counted as proof)

Run:  python3 evidence_intel.py        (set GITHUB_TOKEN to also fetch repos that are only in dataset.json)
"""

import json
import os
import re
import time
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path

import github_intel as gh
from github_intel import norm, _rx, _hits

# ============================================================
# CONFIG
# ============================================================
CVE_FILE = "zero_day_threats/zero_day.json"
PAPERS_FILE = "papers_threats/papers.json"
GITHUB_FILE = "github_threats/github.json"
DATASET_FILE = os.getenv("DATASET_FILE", "dataset.json")
REGISTER_FILE = "threat_register.json"
OUT_DIR = Path("evidence_threats")
MAX_PER_TYPE = 25            # evidence items kept per threat and per source type
REPO_META_CACHE = Path("cache") / "github_repo_meta_cache.json"
REPO_META_TTL_H = 24

# ============================================================
# SURFACE TAXONOMY (shared by CVEs, papers, repos)
# ============================================================
RULES = dict(gh.SURFACE_RULES)
RULES.update({
    "Bluetooth": ["bluetooth", "ble"],
    "Wi-Fi": ["wifi", "wi fi", "wlan"],
    "GNSS": ["gps", "gnss"],
    "Backend / Cloud": ["backend", "back end", "cloud server", "vehicle cloud"],
})
RX = {s: _rx(k) for s, k in RULES.items()}
GENERAL = "General Automotive"
SURFACES = list(RULES) + [GENERAL]

ALIASES = {  # names used by the older modules -> canonical names
    "doip": "DoIP", "ota": "OTA / Firmware", "ota / firmware": "OTA / Firmware",
    "fleet": "Telematics", "backend": "Backend / Cloud", "identity": "Backend / Cloud",
    "wifi": "Wi-Fi", "general automotive surface": GENERAL, "general automotive": GENERAL,
}
PAPER_TOPIC_SURFACES = {
    "In-vehicle network / CAN": ["CAN Bus"], "V2X / connected": ["V2X"], "EV charging": ["EV Charging"],
    "Keyless / hardware": ["Keyless / Hardware"], "OTA / updates": ["OTA / Firmware"],
    "Telematics / infotainment": ["Telematics", "Infotainment"], "ECU / fuzzing / pentest": ["ECU"],
}
GH_URL_RX = re.compile(r"github\.com/([\w.\-]+/[\w.\-]+)", re.I)


def canon(s):
    k = (s or "").strip().lower()
    if k in ALIASES:
        return ALIASES[k]
    return next((c for c in SURFACES if c.lower() == k), s)


def detect(text):
    t = norm(text)
    return sorted(s for s, rx in RX.items() if rx.search(t))


# ============================================================
# IO
# ============================================================
def load_records(path):
    try:
        with open(path, encoding="utf-8") as f:
            data = json.load(f)
    except Exception:
        return []
    return data if isinstance(data, list) else data.get("records", [])


def slug(s):
    return re.sub(r"[^A-Z0-9]+", "-", s.upper()).strip("-")


def load_register():
    p = Path(REGISTER_FILE)
    if p.exists():
        data = json.loads(p.read_text(encoding="utf-8"))
        return data["threats"] if isinstance(data, dict) else data
    threats = [{
        "id": f"TR-{slug(s)}",
        "title": ("General / unclassified automotive threats" if s == GENERAL else f"Compromise or abuse of {s}"),
        "surfaces": [s], "terms": [], "evidence_types": ["cve", "paper", "repo"], "description": "",
        "stride": "", "atm_tactic": "", "unr155_category": "", "iso21434_asset": "", "owner": "", "notes": "",
    } for s in SURFACES]
    p.write_text(json.dumps({"_comment": "Edit freely: rename, split, add 'terms' (extra keywords) and fill the "
                             "taxonomy columns. 'surfaces' decides which evidence is attached.",
                             "threats": threats}, indent=2, ensure_ascii=False), encoding="utf-8")
    print(f"[EVIDENCE] Created default {REGISTER_FILE} ({len(threats)} threats) - edit it to match your register")
    return threats


# ============================================================
# GITHUB RECORDS (github.json + repos only present in dataset.json)
# ============================================================
def split_name(name):
    owner, _, repo = (name or "").partition("__")
    return f"{owner}/{repo}" if repo else name


def make_repo_record(meta, readme, known_cves):
    c = gh.classify(meta, readme)
    c["linked_cves"] = [x for x in c["cves"] if x in known_cves]
    rec = {k: meta[k] for k in ("full_name", "owner", "name", "description", "url", "stars", "forks", "language",
                                "license", "topics", "created", "pushed", "archived", "open_issues")}
    rec.update(c)
    rec["year"] = int(rec["pushed"][:4]) if rec["pushed"][:4].isdigit() else None
    return rec


def fetch_missing_repos(names, known_cves):
    if not names:
        return {}
    if not gh.GH_TOKEN:
        print(f"[EVIDENCE] {len(names)} dataset repos are not in github.json; set GITHUB_TOKEN to fetch their metadata")
        return {}
    cache = gh.load_json(REPO_META_CACHE)
    readmes = gh.load_json(gh.README_CACHE)
    out = {}
    for full in names:
        item = cache.get(full.lower())
        if item and time.time() - item["ts"] < REPO_META_TTL_H * 3600:
            meta = item["data"]
        else:
            data = gh.http_get(f"{gh.GH_API}/repos/{full}")
            if not data:
                continue
            meta = gh.norm_repo(data)
            cache[full.lower()] = {"ts": time.time(), "data": meta}
            time.sleep(0.1)
        out[full.lower()] = make_repo_record(meta, gh.get_readme(meta, readmes), known_cves)
    gh.save_json(REPO_META_CACHE, cache)
    gh.save_json(gh.README_CACHE, readmes)
    print(f"[EVIDENCE] Fetched {len(out)}/{len(names)} missing dataset repos from GitHub")
    return out


# ============================================================
# DATASET ENRICHMENT (keeps every original field)
# ============================================================
def llm_score(d):
    for key in ("score_final", "score_initial", "score"):
        v = d.get(key)
        if v and v != "Unknown":
            return v, {"score_final": "llm_final", "score_initial": "llm_initial", "score": "llm"}[key]
    return None, None


def enrich_dataset_record(d, g):
    out = dict(d)
    full = split_name(d.get("name", ""))
    out["full_name"] = full
    out["github_match"] = bool(g)
    llm = [canon(s) for s in d.get("surfaces", []) if canon(s) != GENERAL]
    rule = [s for s in (g["surfaces"] if g else []) if s != GENERAL]
    spec = " ".join(f"{x.get('title','')} {x.get('text','')}"
                    for x in (d.get("fuzzing") or []) + (d.get("hidden_attack_surfaces") or []))
    known = set(llm) | set(rule)
    out["surfaces_enriched"] = sorted(known) or [GENERAL]               # evidence-based only
    out["surfaces_hypothesised"] = sorted(s for s in detect(spec) if s not in known)   # LLM speculation, kept apart
    if g:
        out.update(url=g["url"], stars=g["stars"], forks=g["forks"], pushed=g["pushed"], created=g["created"],
                   license=g["license"], language=g["language"], topics=g["topics"], archived=g["archived"],
                   role=g["role"], score_rule=g["threat"], score_rule_points=g["score"],
                   score_reasons=g["reasons"], cves=g["cves"], vendors=g["vendors"])
    score, basis = llm_score(d)
    if not score and g:
        score, basis = g["threat"], "rules"
    out["score_best"], out["score_basis"] = score or "Unknown", basis or "none"
    out["needs_llm_review"] = bool(g and g["threat"] in ("Critical", "High") and not llm_score(d)[0])
    return out


# ============================================================
# EVIDENCE ITEMS
# ============================================================
GRADE_ORDER = {"A": 0, "B": 1, "C": 2, "D": 3}


def short(text, n=240):
    text = re.sub(r"\s+", " ", text or "").strip()
    return text if len(text) <= n else text[:n].rsplit(" ", 1)[0] + "…"


def cve_items(records):
    items = []
    for r in records:
        desc = (r.get("vulnerabilities") or [{}])[0].get("text", "")
        surf = sorted({canon(s) for s in r.get("surfaces", []) if canon(s) != GENERAL} | set(detect(desc)))
        cvss = r.get("cvss")
        items.append({
            "ref": f"cve:{r['name']}", "type": "cve", "kind": "vulnerability", "id": r["name"],
            "title": f"{r['name']} - {short(desc, 110)}", "url": r.get("url", ""),
            "date": (r.get("published") or "")[:10], "severity": r.get("score", "Unknown"), "cvss": cvss,
            "grade": "A" if (cvss or 0) >= 7 and not r.get("pending_analysis") else "B",
            "surfaces": surf or [GENERAL],
            "vendors": [t for t in r.get("matched_terms", []) if gh.OEM_RX.search(norm(t))],
            "summary": short(desc), "cves": [r["name"]], "linked": [],
            "_text": norm(" ".join(v.get("text", "") for v in r.get("vulnerabilities", []))),
        })
    return items


def paper_items(records):
    items = []
    for p in records:
        surf = set(detect(p["title"] + " " + p.get("abstract", "")))
        for t in p.get("topics", []):
            surf.update(PAPER_TOPIC_SURFACES.get(t, []))
        defensive = "Intrusion detection" in p.get("topics", [])
        items.append({
            "ref": f"paper:{p['id']}", "type": "paper", "kind": "mitigation" if defensive else "research",
            "id": p["id"], "title": p["title"], "url": p.get("url", ""), "date": (p.get("date") or str(p.get("year") or ""))[:10],
            "severity": "", "grade": "B" if p.get("top_venue") else "C", "surfaces": sorted(surf) or [GENERAL],
            "vendors": p.get("vendors", []), "venue": p.get("venue", ""), "cited_by": p.get("cited_by", 0),
            "summary": short(p.get("abstract", "")), "cves": p.get("cves", []),
            "gh_mentions": [m.lower().rstrip(".") for m in GH_URL_RX.findall(p["title"] + " " + p.get("abstract", ""))],
            "linked": [], "_text": norm(p["title"] + " " + p.get("abstract", "")),
        })
    return items


def repo_items(gh_records, dataset_by_full):
    items = []
    for g in gh_records.values():
        d = dataset_by_full.get(g["full_name"].lower())
        surf = [s for s in g["surfaces"]] or [GENERAL]
        if d:
            surf = d["surfaces_enriched"]
        item = {
            "ref": f"repo:{g['full_name'].lower()}", "type": "repo",
            "kind": "mitigation" if g["role"] == "defensive" else "tooling", "id": g["full_name"],
            "title": g["full_name"], "url": g["url"], "date": g["pushed"], "severity": g["threat"],
            "score": g["score"], "role": g["role"], "stars": g["stars"], "grade": "C", "surfaces": surf,
            "vendors": g["vendors"], "summary": short(g["description"]), "cves": g["cves"], "linked": [],
            "_text": norm(" ".join([g["name"], g["description"], " ".join(g["topics"])])),
        }
        if d:   # keep the LLM work, clearly labelled as hypothesis (grade D)
            item["llm"] = {
                "grade": "D", "score": d.get("score_best"), "basis": d.get("score_basis"),
                "justification": short(d.get("justification") or d.get("final_justification") or "", 300),
                "vulnerabilities": [v.get("title", "") for v in (d.get("vulnerabilities") or [])][:6],
                "fuzzing_ideas": [v.get("title", "") for v in (d.get("fuzzing") or [])][:6],
                "hypothesised_surfaces": d.get("surfaces_hypothesised", []),
            }
        items.append(item)
    return items


def link_items(items):
    by_ref = {i["ref"]: i for i in items}

    def link(a, b_ref):
        b = by_ref.get(b_ref)
        if b and b_ref != a["ref"]:
            if b_ref not in a["linked"]:
                a["linked"].append(b_ref)
            if a["ref"] not in b["linked"]:
                b["linked"].append(a["ref"])
    for it in items:
        for c in it.get("cves", []):
            link(it, f"cve:{c.upper()}")
        for m in it.get("gh_mentions", []):
            link(it, f"repo:{m}")


# ============================================================
# REGISTER
# ============================================================
def rank(items):
    items = sorted(items, key=lambda i: i["date"] or "", reverse=True)
    return sorted(items, key=lambda i: (GRADE_ORDER[i["grade"]], -(i.get("cvss") or i.get("score") or 0)))


POINTS = {"A": 3.0, "B": 2.0, "C": 1.0}
TYPE_CAP = 9.0                       # one source type can contribute at most this many points


def build_register(threats, items):
    matched_refs, out = set(), []
    for t in threats:
        want = {canon(s) for s in t.get("surfaces", [])}
        term_rx = _rx(t["terms"]) if t.get("terms") else None
        applicable = t.get("evidence_types", ["cve", "paper", "repo"])

        hits = []
        for it in items:
            if it["type"] not in applicable:
                continue
            surf = sorted(want & set(it["surfaces"]))
            terms = _hits(term_rx, it["_text"]) if term_rx else []
            if not surf and not terms:
                continue
            row = {k: v for k, v in it.items() if k != "_text"}
            row["match_via"] = [f"term:{x}" for x in terms[:4]] + [f"surface:{x}" for x in surf]
            row["relevance"] = 2 if terms else 1          # term in the text > only the same technical surface
            hits.append(row)
            matched_refs.add(it["ref"])

        threat_ev = [h for h in hits if h["kind"] != "mitigation"]
        mitig = [h for h in hits if h["kind"] == "mitigation"]
        cves = [h for h in hits if h["type"] == "cve"]
        papers = [h for h in hits if h["type"] == "paper"]
        tooling = [h for h in threat_ev if h["type"] == "repo"]
        research = [h for h in threat_ev if h["type"] == "paper"]

        # strength = weighted points (grade x relevance), capped per source type
        per_type = {}
        for h in threat_ev:
            per_type[h["type"]] = per_type.get(h["type"], 0) + POINTS[h["grade"]] * (1.0 if h["relevance"] == 2 else 0.5)
        total = sum(min(v, TYPE_CAP) for v in per_type.values())
        if not applicable:
            strength = "n/a"
        elif total >= 12 and (len(per_type) >= 2 or len(applicable) == 1):
            strength = "Strong"
        elif total >= 5:
            strength = "Moderate"
        elif total > 0:
            strength = "Weak"
        else:
            strength = "None"

        gaps = []                                          # only for evidence types that make sense for this threat
        if "cve" in applicable and not cves: gaps.append("no CVE evidence")
        if "paper" in applicable and not [p for p in research if p["grade"] == "B"]: gaps.append("no peer-reviewed research")
        if "repo" in applicable and not tooling: gaps.append("no public tooling")
        if ("repo" in applicable or "paper" in applicable) and not mitig: gaps.append("no mitigation evidence")

        def rank_key(i):
            return (-i["relevance"], GRADE_ORDER[i["grade"]], -(i.get("cvss") or i.get("score") or 0))
        keep = {}
        for k, group in (("cve", cves), ("paper", papers), ("repo", [h for h in hits if h["type"] == "repo"])):
            group = sorted(group, key=lambda i: i["date"] or "", reverse=True)
            keep[k] = sorted(group, key=rank_key)[:MAX_PER_TYPE]

        dates = [h["date"] for h in hits if h["date"]]
        row = dict(t)
        row.update({
            "strength": strength, "evidence_points": round(total, 1), "applicable_types": applicable,
            "gaps": gaps, "last_evidence": max(dates) if dates else "",
            "counts": {"cve": len(cves), "cve_high": len([c for c in cves if c["grade"] == "A"]),
                       "paper": len(papers), "paper_top_venue": len([p for p in papers if p["grade"] == "B"]),
                       "tooling": len(tooling), "tooling_high": len([r for r in tooling if r["severity"] in ("Critical", "High")]),
                       "mitigation": len(mitig), "total": len(hits)},
            "evidence": keep,
        })
        out.append(row)
    orphans = [i for i in items if i["ref"] not in matched_refs]
    return out, orphans


# ============================================================
# MAIN
# ============================================================
def write_json(path, obj):
    path.parent.mkdir(parents=True, exist_ok=True)
    tmp = path.with_suffix(".tmp")
    tmp.write_text(json.dumps(obj, indent=2, ensure_ascii=False), encoding="utf-8")
    tmp.replace(path)


def main():
    cves, papers = load_records(CVE_FILE), load_records(PAPERS_FILE)
    gh_list, dataset = load_records(GITHUB_FILE), load_records(DATASET_FILE)
    print(f"[EVIDENCE] inputs: cves={len(cves)} papers={len(papers)} github={len(gh_list)} dataset={len(dataset)}")

    known_cves = {r["name"].upper() for r in cves}
    gh_records = {r["full_name"].lower(): r for r in gh_list}
    missing = [split_name(d["name"]) for d in dataset if split_name(d["name"]).lower() not in gh_records]
    gh_records.update(fetch_missing_repos(missing, known_cves))

    enriched = [enrich_dataset_record(d, gh_records.get(split_name(d["name"]).lower())) for d in dataset]
    dataset_by_full = {e["full_name"].lower(): e for e in enriched}

    items = cve_items(cves) + paper_items(papers) + repo_items(gh_records, dataset_by_full)
    link_items(items)
    for e in enriched:                                   # papers/CVEs per dataset repo
        it = next((i for i in items if i["ref"] == f"repo:{e['full_name'].lower()}"), None)
        e["linked_papers"] = [r[6:] for r in (it["linked"] if it else []) if r.startswith("paper:")]
        e["linked_cves"] = [r[4:] for r in (it["linked"] if it else []) if r.startswith("cve:")]

    threats = load_register()
    register, orphans = build_register(threats, items)
    now = datetime.now(timezone.utc).isoformat()

    write_json(OUT_DIR / "dataset_enriched.json", enriched)
    write_json(OUT_DIR / "register.json", {
        "module": "evidence_register", "generated": time.time(), "generated_iso": now,
        "summary": {
            "threats": len(register),
            "by_strength": dict(Counter(r["strength"] for r in register)),
            "evidence_items": len(items),
            "by_type": dict(Counter(i["type"] for i in items)),
            "orphans": {"count": len(orphans), "by_type": dict(Counter(i["type"] for i in orphans))},
            "dataset": {"repos": len(enriched),
                        "unscored_by_llm": sum(1 for e in enriched if e["score_basis"] in ("rules", "none")),
                        "needs_llm_review": sum(e["needs_llm_review"] for e in enriched)},
        },
        "grades": {"A": "confirmed vulnerability (CVE, CVSS>=7)", "B": "CVE medium/pending or peer-reviewed research",
                   "C": "other research or public tooling", "D": "LLM hypothesis (not proof)"},
        "threats": register,
    })
    print(f"[EVIDENCE] register: {dict(Counter(r['strength'] for r in register))} | orphans={len(orphans)}")
    print(f"[EVIDENCE] wrote {OUT_DIR}/register.json and dataset_enriched.json")


if __name__ == "__main__":
    main()
