#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
papers_intel.py - Automotive Cybersecurity Paper Collector (NCRC)

Sources : OpenAlex, Semantic Scholar, arXiv, DBLP   (metadata only - no PDF downloads)
Output  : papers_threats/papers.json  (read by threat_papers.html)

Pipeline: collect (cached) -> normalize -> merge duplicates (DOI / arXiv id / title)
          -> relevance filter (automotive AND security, word-boundary) -> tag topics,
          vendors, CVE ids, top venues -> write JSON

Environment variables (all optional except where noted):
  OPENALEX_API_KEY   free key, https://openalex.org/settings/api  (OpenAlex is skipped without it)
  S2_API_KEY         Semantic Scholar key (works without, but is rate limited hard)
  CONTACT_EMAIL      put in the User-Agent (polite use of the APIs)
  PAPERS_MIN_YEAR    default 2015
  PAPERS_SNOWBALL    1 (default) / 0 - also fetch papers citing the SEED_DOIS below

Run:  python3 papers_intel.py
"""

import hashlib
import json
import logging
import os
import re
import time
import xml.etree.ElementTree as ET
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path

import requests

# ============================================================
# CONFIG
# ============================================================
MIN_YEAR = int(os.getenv("PAPERS_MIN_YEAR", "2015"))
# Keys: either set the environment variable, or paste the key between the quotes below.
INLINE_OPENALEX_KEY = ""     # e.g. "abc123..."   (do not commit this file if you fill it in)
INLINE_S2_KEY = ""
CONTACT=""

def _key(env_name, inline):
    val = (os.getenv(env_name) or inline or "").strip().strip("\"'")
    return val or None


OPENALEX_KEY = _key("OPENALEX_API_KEY", INLINE_OPENALEX_KEY)
S2_KEY = _key("S2_API_KEY", INLINE_S2_KEY)
CONTACT = os.getenv("CONTACT_EMAIL", "")
SNOWBALL = os.getenv("PAPERS_SNOWBALL", "1") != "0"

TIMEOUT = 30
CACHE_TTL_HOURS = 12
OUTPUT_FILE = "papers_threats/papers.json"
ZERO_DAY_FILE = "zero_day_threats/zero_day.json"     # used to link papers <-> CVEs

CACHE_DIR = Path("cache");  CACHE_DIR.mkdir(exist_ok=True)
LOG_DIR = Path("logs");     LOG_DIR.mkdir(exist_ok=True)
CACHE_FILE = CACHE_DIR / "papers_cache.json"

HEADERS = {"User-Agent": f"NCRC-Paper-Collector/1.0 (mailto:{CONTACT})" if CONTACT
           else "NCRC-Paper-Collector/1.0"}

logging.basicConfig(level=logging.INFO,
                    format="%(asctime)s %(levelname)s %(message)s",
                    handlers=[logging.FileHandler(LOG_DIR / "papers.log"), logging.StreamHandler()])
logger = logging.getLogger("papers")

# Classic papers whose citing works are fetched (snowballing). Add your own seed DOIs.
SEED_DOIS = [
    "10.1109/sp.2010.34",      # Koscher et al., Experimental Security Analysis of a Modern Automobile
]

# ============================================================
# RATE LIMITING
# ============================================================

_LAST_S2_CALL = 0.0

# Semantic Scholar: maximaal 1 request/sec
# Kleine veiligheidsmarge om 429's te voorkomen.
S2_MIN_INTERVAL = 2.0


def s2_wait():
    global _LAST_S2_CALL

    elapsed = time.time() - _LAST_S2_CALL

    if elapsed < S2_MIN_INTERVAL:
        time.sleep(S2_MIN_INTERVAL - elapsed)

    _LAST_S2_CALL = time.time()


# ============================================================
# QUERIES
# ============================================================
OPENALEX_QUERIES = [
    '"CAN bus" AND (attack OR intrusion OR security OR vulnerability)',
    'automotive AND (cybersecurity OR "cyber security" OR attack OR vulnerability)',
    '"in-vehicle network" AND (security OR attack OR intrusion)',
    '("ISO 21434" OR "ISO/SAE 21434" OR "UNECE R155") AND (vehicle OR automotive OR cybersecurity)',
    '("connected vehicle" OR "connected car") AND (security OR attack OR vulnerability)',
    '(V2X OR "vehicle-to-everything") AND (security OR attack OR authentication)',
    '("electric vehicle charging" OR "EV charging" OR OCPP OR EVSE) AND (security OR attack OR vulnerability)',
    '(keyless OR "key fob" OR immobilizer) AND (relay OR attack OR vulnerability)',
    '("over-the-air" OR OTA) AND (vehicle OR automotive) AND (security OR attack)',
    '(ECU OR "electronic control unit") AND (attack OR fuzzing OR vulnerability) AND (vehicle OR automotive)',
    '("autonomous vehicle" OR "autonomous driving") AND (adversarial OR attack OR security)',
    'telematics AND (vulnerability OR attack OR security) AND vehicle',
    'infotainment AND (vulnerability OR attack OR exploit)',
    '(vehicle OR automotive) AND (penetration OR fuzzing OR "threat analysis" OR "risk assessment") AND security',
]

S2_QUERIES = [
    "automotive cybersecurity", "CAN bus intrusion detection", "vehicle ECU attack",
    "ISO 21434 threat analysis risk assessment", "V2X security attack",
    "EV charging security OCPP", "keyless entry relay attack",
    "over-the-air update vehicle security", "connected car vulnerability",
    "in-vehicle network security",
]

ARXIV_QUERIES = [
    'cat:cs.CR AND (abs:automotive OR abs:"in-vehicle" OR abs:"connected vehicle" OR abs:"autonomous vehicle")',
    'abs:"CAN bus" AND (abs:attack OR abs:intrusion OR abs:security)',
    '(abs:"ISO 21434" OR abs:"UNECE R155" OR abs:"ISO/SAE 21434")',
    'abs:V2X AND (abs:security OR abs:attack)',
    '(abs:"EV charging" OR abs:"electric vehicle charging" OR abs:OCPP) AND (abs:security OR abs:attack)',
    'abs:keyless AND (abs:relay OR abs:attack OR abs:vulnerability)',
    '(abs:ECU OR abs:"electronic control unit") AND (abs:attack OR abs:fuzzing) AND (abs:vehicle OR abs:automotive)',
]

DBLP_QUERIES = [   # DBLP matches title words only
    "automotive security", "vehicle security", "vehicle attack", "can bus security",
    "can bus intrusion", "in-vehicle network security", "v2x security", "ecu security",
    "connected vehicle security", "autonomous vehicle attack", "charging station security",
    "vehicle intrusion detection", "automotive fuzzing", "keyless entry attack",
    "vehicle over-the-air security", "automotive cybersecurity",
]

def register_queries(path="threat_register.json"):
    """Plain-language queries per threat category, written by build_register.py (editable in the JSON)."""
    try:
        data = json.load(open(path, encoding="utf-8"))
    except Exception:
        return []
    cats = data.get("categories", {}) if isinstance(data, dict) else {}
    seen, out = set(), []
    for c in cats.values():
        for q in c.get("paper_queries", []):
            if q.lower() not in seen:
                seen.add(q.lower()); out.append(q)
    return out


REGISTER_QUERIES = register_queries()

# ============================================================
# RELEVANCE TERMS (word-boundary matching, like zero_day_intel.py)
# ============================================================
def _rx(terms):
    parts = sorted({t.lower() for t in terms}, key=len, reverse=True)
    return re.compile(r"(?<![a-z0-9])(?:" + "|".join(re.escape(p) for p in parts) + r")(?![a-z0-9])", re.I)


def _hits(rx, text):
    return sorted({m.group(0).lower() for m in rx.finditer(text)})


OEMS = ["tesla", "bmw", "audi", "volkswagen", "porsche", "toyota", "lexus", "hyundai", "kia",
        "honda", "nissan", "volvo cars", "mercedes-benz", "mercedes", "jeep", "stellantis",
        "renault", "polestar", "rivian", "byd", "general motors", "harley-davidson"]

AUTO_STRONG = [
    "automotive", "automobile", "automobiles", "in-vehicle", "in vehicle network",
    "can bus", "controller area network", "can-fd", "flexray", "ecu", "ecus",
    "electronic control unit", "obd", "obd-ii", "iso 21434", "iso/sae 21434", "unece r155",
    "wp.29", "autosar", "v2x", "vehicle-to-everything", "vehicle-to-vehicle",
    "connected vehicle", "connected vehicles", "connected car", "connected cars",
    "autonomous vehicle", "autonomous vehicles", "autonomous driving", "self-driving",
    "driver assistance", "adas", "telematics", "infotainment", "keyless", "immobilizer",
    "immobiliser", "key fob", "tpms", "ocpp", "evse", "ev charging", "electric vehicle charging",
    "charging station", "charging stations", "vehicular", "intelligent transportation",
    "iso 15118", "v2g", "vehicle to grid", "vehicle-to-grid",
] + OEMS
AUTO_WEAK = ["vehicle", "vehicles", "car", "cars", "truck", "trucks"]
SECURITY = [
    "attack", "attacks", "attacker", "attackers", "vulnerability", "vulnerabilities", "vulnerable",
    "exploit", "exploits", "exploitation", "intrusion", "security", "secure", "cybersecurity",
    "cyber-security", "threat", "threats", "adversarial", "spoofing", "jamming", "malware",
    "penetration", "fuzzing", "relay", "authentication", "tampering", "hacking", "hijack",
    "risk assessment", "tara",
    # strategic topics used by the threat register (privacy, supply chain, governance ...)
    "privacy", "surveillance", "espionage", "supply chain", "resilience", "regulation", "sovereignty",
    "personal data", "gdpr", "tracking", "geopolitical", "sbom", "software bill of materials",
]
AUTO_STRONG_RX, AUTO_WEAK_RX, SEC_RX = _rx(AUTO_STRONG), _rx(AUTO_WEAK), _rx(SECURITY)
OEM_RX = _rx(OEMS)

TOPIC_RULES = {
    "In-vehicle network / CAN": ["can bus", "controller area network", "can-fd", "flexray", "in-vehicle", "in vehicle network", "obd", "obd-ii"],
    "Intrusion detection": ["intrusion detection", "anomaly detection", "ids"],
    "V2X / connected": ["v2x", "vehicle-to-everything", "vehicle-to-vehicle", "connected vehicle", "connected vehicles", "connected car", "connected cars", "vehicular"],
    "EV charging": ["ev charging", "electric vehicle charging", "charging station", "charging stations", "ocpp", "evse"],
    "Keyless / hardware": ["keyless", "key fob", "immobilizer", "immobiliser", "tpms", "relay attack"],
    "OTA / updates": ["over-the-air", "ota", "firmware update", "software update"],
    "Autonomous / ADAS": ["autonomous vehicle", "autonomous vehicles", "autonomous driving", "self-driving", "adas", "driver assistance"],
    "Standards / TARA": ["iso 21434", "iso/sae 21434", "unece r155", "wp.29", "tara", "risk assessment", "threat analysis"],
    "Telematics / infotainment": ["telematics", "infotainment"],
    "Privacy / data protection": ["privacy", "personal data", "gdpr", "tracking", "surveillance"],
    "Supply chain / SBOM": ["supply chain", "sbom", "software bill of materials", "supplier"],
    "AI / machine learning": ["machine learning", "large language model", "llm", "deep learning", "adversarial"],
    "Governance / regulation": ["regulation", "unece", "type approval", "compliance", "cyber resilience act"],
    "Wireless / GNSS / SDR": ["gnss", "gps", "spoofing", "jamming", "software defined radio", "sdr"],
    "Espionage / sovereignty": ["espionage", "sovereignty", "geopolitical"],
    "ECU / fuzzing / pentest": ["ecu", "ecus", "electronic control unit", "fuzzing", "penetration", "reverse engineering"],
}
TOPIC_RX = {k: _rx(v) for k, v in TOPIC_RULES.items()}

TOP_VENUES_RX = _rx([
    "usenix security", "security symposium", "symposium on security and privacy", "s&p",
    "computer and communications security", "ccs", "network and distributed system security",
    "ndss", "escar", "vehiclesec", "computer science in cars", "cscs", "acsac", "asiaccs", "raid",
    "dsn", "eurosp", "european symposium on security and privacy", "ches", "tches",
    "ieee transactions on vehicular technology", "intelligent transportation systems",
    "intelligent vehicles symposium", "vehicular communications",
    "dependable and secure computing", "information forensics and security",
    "sae international", "sae technical paper", "vehicular technology conference",
])
CVE_RX = re.compile(r"\bCVE-\d{4}-\d{4,7}\b", re.I)


# ============================================================
# CACHE / HTTP
# ============================================================
def load_cache():
    if not CACHE_FILE.exists():
        return {}
    try:
        return json.loads(CACHE_FILE.read_text(encoding="utf-8"))
    except Exception as e:
        logger.warning("Cache load failed: %s", e)
        return {}


def save_cache(cache):
    tmp = CACHE_FILE.with_suffix(".tmp")
    tmp.write_text(json.dumps(cache), encoding="utf-8")
    tmp.replace(CACHE_FILE)


def http_get(url, params=None, headers=None, retries=6, as_json=True):
    hdrs = dict(HEADERS); hdrs.update(headers or {})
    for attempt in range(1, retries + 1):
        wait = attempt * 2
        try:
            r = requests.get(url, params=params, headers=hdrs, timeout=TIMEOUT)
            if r.status_code == 200:
                return r.json() if as_json else r.text

            logger.warning("HTTP %s: %s", r.status_code, url)  # URL only: no API key in logs

            if r.status_code == 429:
                retry_after = r.headers.get("Retry-After")
                try:
                    header_wait = max(0, int(retry_after))
                except (TypeError, ValueError):
                    header_wait = 0

                # Bounded exponential backoff: 60, 120, 240, 480, 600, 600 seconds.
                backoff_wait = min(600, 60 * (2 ** (attempt - 1)))
                wait = max(header_wait, backoff_wait)

                if attempt == retries:
                    logger.error(
                        "Rate limit persists after %s attempts (%s); giving up this query.",
                        retries,
                        url,
                    )
                    break

                logger.warning(
                    "Rate limit reached (%s). Retry %s/%s in %s seconds.",
                    url,
                    attempt,
                    retries,
                    wait,
                )
                time.sleep(wait)
                continue

            if r.status_code in (403, 503):
                wait = 10 * attempt
        except Exception as e:
            logger.warning("Attempt %s failed (%s): %s", attempt, url, e)

        if attempt < retries:
            time.sleep(wait)
    return None

def cached(cache, key, fn, delay):
    item = cache.get(key)
    if item and time.time() - item["ts"] < CACHE_TTL_HOURS * 3600:
        logger.info("CACHE HIT %s", key)
        return item["data"]
    logger.info("FETCH %s", key)
    data = fn()
    time.sleep(delay)
    if data is None:
        logger.warning("No data for %s (not cached)", key)
        return []
    cache[key] = {"ts": time.time(), "data": data}
    save_cache(cache)
    return data


# ============================================================
# NORMALIZATION
# ============================================================
def clean_doi(d):
    if not d:
        return ""
    return re.sub(r"^(https?://(dx\.)?doi\.org/|doi:)", "", d.strip(), flags=re.I).lower()


def extract_arxiv(*texts):
    for t in texts:
        m = re.search(r"arxiv\.org/(?:abs|pdf)/([0-9]{4}\.[0-9]{4,5}|[a-z\-]+(?:\.[A-Z]{2})?/[0-9]{7})", t or "", re.I)
        if m:
            return m.group(1)
    return ""


def mk(**kw):
    rec = dict(title="", abstract="", authors=[], year=None, date="", venue="", doi="",
               arxiv_id="", url="", pdf_url="", cited_by=0, source="")
    rec.update({k: v for k, v in kw.items() if v is not None})
    rec["title"] = re.sub(r"\s+", " ", rec["title"] or "").strip().rstrip(".")
    rec["abstract"] = re.sub(r"\s+", " ", rec["abstract"] or "").strip()
    return rec


def inverted_to_text(inv):
    if not inv:
        return ""
    pos = {}
    for word, idxs in inv.items():
        for i in idxs:
            pos[i] = word
    return " ".join(pos[i] for i in sorted(pos))


# ---------------- OpenAlex ----------------
OA_URL = "https://api.openalex.org/works"
OA_SELECT = ("id,doi,title,display_name,publication_year,publication_date,abstract_inverted_index,"
             "authorships,primary_location,best_oa_location,open_access,cited_by_count")


def norm_openalex(w):
    loc = w.get("primary_location") or {}
    src = loc.get("source") or {}
    best = w.get("best_oa_location") or {}
    oa = w.get("open_access") or {}
    doi = clean_doi(w.get("doi"))
    landing = loc.get("landing_page_url") or ""
    pdf = best.get("pdf_url") or oa.get("oa_url") or ""
    return mk(
        title=w.get("title") or w.get("display_name"),
        abstract=inverted_to_text(w.get("abstract_inverted_index")),
        authors=[(a.get("author") or {}).get("display_name", "") for a in (w.get("authorships") or [])],
        year=w.get("publication_year"), date=w.get("publication_date") or "",
        venue=src.get("display_name") or "", doi=doi,
        arxiv_id=extract_arxiv(landing, pdf),
        url=f"https://doi.org/{doi}" if doi else (landing or w.get("id", "")),
        pdf_url=pdf, cited_by=w.get("cited_by_count") or 0, source="openalex")


def fetch_openalex(search=None, filt=None, pages=3):
    out, cursor = [], "*"
    flt = [f"from_publication_date:{MIN_YEAR}-01-01"] + ([filt] if filt else [])
    for _ in range(pages):
        params = {"per_page": 100, "cursor": cursor, "select": OA_SELECT,
                  "filter": ",".join(flt), "api_key": OPENALEX_KEY}
        if search:
            params["search"] = search
        data = http_get(OA_URL, params)
        if data is None:
            return None
        out.extend(norm_openalex(w) for w in data.get("results", []))
        cursor = (data.get("meta") or {}).get("next_cursor")
        if not cursor or not data.get("results"):
            break
        time.sleep(0.2)
    return out


def fetch_openalex_citing(doi):
    """Papers that cite a seed DOI (snowballing)."""
    work = http_get(f"{OA_URL}/doi:{doi}", {"select": "id", "api_key": OPENALEX_KEY})
    if not work or not work.get("id"):
        logger.warning("Seed DOI not found in OpenAlex: %s", doi)
        return None
    wid = work["id"].rsplit("/", 1)[-1]
    return fetch_openalex(filt=f"cites:{wid}", pages=3)


# ---------------- Semantic Scholar ----------------
S2_URL = "https://api.semanticscholar.org/graph/v1/paper/search"
S2_FIELDS = "title,abstract,year,venue,publicationDate,authors,externalIds,openAccessPdf,citationCount,url"


def fetch_s2(query):
    s2_wait()
    headers = {"x-api-key": S2_KEY} if S2_KEY else {}
    data = http_get(S2_URL, {"query": query, "limit": 100, "year": f"{MIN_YEAR}-", "fields": S2_FIELDS}, headers)
    if data is None:
        return None
    out = []
    for p in data.get("data", []):
        ext = p.get("externalIds") or {}
        doi = clean_doi(ext.get("DOI"))
        out.append(mk(
            title=p.get("title"), abstract=p.get("abstract"),
            authors=[a.get("name", "") for a in p.get("authors") or []],
            year=p.get("year"), date=p.get("publicationDate") or "",
            venue=p.get("venue") or "", doi=doi, arxiv_id=ext.get("ArXiv") or "",
            url=f"https://doi.org/{doi}" if doi else (p.get("url") or ""),
            pdf_url=(p.get("openAccessPdf") or {}).get("url") or "",
            cited_by=p.get("citationCount") or 0, source="semanticscholar"))
    return out


# ---------------- arXiv ----------------
ARXIV_URL = "https://export.arxiv.org/api/query"
NS = {"a": "http://www.w3.org/2005/Atom", "ax": "http://arxiv.org/schemas/atom"}


def parse_arxiv(xml_text):
    out = []
    root = ET.fromstring(xml_text)
    for e in root.findall("a:entry", NS):
        def t(path):
            el = e.find(path, NS)
            return (el.text or "") if el is not None else ""
        abs_url = t("a:id")
        pdf = ""
        for l in e.findall("a:link", NS):
            if l.get("title") == "pdf":
                pdf = l.get("href", "")
        arx = extract_arxiv(abs_url) or re.sub(r"v\d+$", "", abs_url.rsplit("/abs/", 1)[-1])
        doi = clean_doi(t("ax:doi"))
        out.append(mk(
            title=t("a:title"), abstract=t("a:summary"),
            authors=[(a.find("a:name", NS).text or "") for a in e.findall("a:author", NS)],
            year=int(t("a:published")[:4]) if t("a:published")[:4].isdigit() else None,
            date=t("a:published")[:10], venue=t("ax:journal_ref") or "arXiv", doi=doi, arxiv_id=arx,
            url=f"https://doi.org/{doi}" if doi else f"https://arxiv.org/abs/{arx}",
            pdf_url=pdf or f"https://arxiv.org/pdf/{arx}", source="arxiv"))
    return out


def fetch_arxiv(query, pages=2):
    out = []
    for page in range(pages):
        txt = http_get(ARXIV_URL, {"search_query": query, "start": page * 100, "max_results": 100,
                                   "sortBy": "submittedDate", "sortOrder": "descending"}, as_json=False)
        if txt is None:
            return None
        try:
            batch = parse_arxiv(txt)
        except ET.ParseError as e:
            logger.warning("arXiv XML parse error: %s", e)
            return None
        out.extend(batch)
        if len(batch) < 100:
            break
        time.sleep(3.1)                                   # arXiv asks for >= 3 s between calls
    return out


# ---------------- DBLP ----------------
DBLP_URL = "https://dblp.org/search/publ/api"


def fetch_dblp(query):
    data = http_get(DBLP_URL, {"q": query, "format": "json", "h": 500})
    if data is None:
        return None
    hits = ((data.get("result") or {}).get("hits") or {}).get("hit") or []
    out = []
    for h in hits:
        info = h.get("info") or {}
        authors = (info.get("authors") or {}).get("author") or []
        if isinstance(authors, dict):
            authors = [authors]
        venue = info.get("venue") or ""
        if isinstance(venue, list):
            venue = ", ".join(venue)
        year = int(info["year"]) if str(info.get("year", "")).isdigit() else None
        if year and year < MIN_YEAR:
            continue
        doi = clean_doi(info.get("doi"))
        ee = info.get("ee") or ""
        if isinstance(ee, list):
            ee = ee[0] if ee else ""
        out.append(mk(title=info.get("title"), authors=[a.get("text", "") for a in authors],
                      year=year, venue="arXiv" if venue == "CoRR" else venue, doi=doi,
                      arxiv_id=extract_arxiv(ee),
                      url=f"https://doi.org/{doi}" if doi else ee, source="dblp"))
    return out


# ============================================================
# MERGE / DEDUPE
# ============================================================
def title_key(title):
    return re.sub(r"[^a-z0-9]", "", (title or "").lower())


def is_preprint_venue(v):
    return (v or "").strip().lower() in ("", "arxiv", "corr", "arxiv.org")


def merge_into(base, new):
    for k in ("doi", "arxiv_id", "url", "pdf_url", "year"):
        if not base.get(k) and new.get(k):
            base[k] = new[k]
    if not base["venue"] and new["venue"]:
        base["venue"] = new["venue"]
    if is_preprint_venue(base["venue"]) and not is_preprint_venue(new["venue"]):
        base["venue"] = new["venue"]
        if new.get("doi"):
            base["url"] = new["url"]
    if len(new["abstract"]) > len(base["abstract"]):
        base["abstract"] = new["abstract"]
    if len(new["authors"]) > len(base["authors"]):
        base["authors"] = new["authors"]
    if new["date"] and (not base["date"] or new["date"] < base["date"]):
        base["date"] = new["date"]
    base["cited_by"] = max(base["cited_by"], new["cited_by"])
    base["sources"] = sorted(set(base["sources"]) | {new["source"]})


def dedupe(records):
    index, out = {}, []
    for r in records:
        if len(r["title"]) < 8:
            continue
        r["sources"] = [r["source"]]
        keys = [k for k in (("doi", r["doi"]), ("arxiv", r["arxiv_id"].lower()),
                            ("title", title_key(r["title"]))) if k[1]]
        base = next((index[k] for k in keys if k in index), None)
        if base is None:
            base = r
            out.append(base)
        else:
            merge_into(base, r)
        for k in keys:
            index[k] = base
        for k in (("doi", base["doi"]), ("arxiv", base["arxiv_id"].lower())):
            if k[1]:
                index[k] = base
    return out


# ============================================================
# RELEVANCE + TAGGING
# ============================================================
def assess(rec):
    """Returns matched terms if the paper is automotive-security relevant, else []."""
    title, text = rec["title"], rec["title"] + " " + rec["abstract"]
    strong = _hits(AUTO_STRONG_RX, text)
    weak_title = _hits(AUTO_WEAK_RX, title)
    sec_any = _hits(SEC_RX, text)
    sec_title = _hits(SEC_RX, title)
    if not sec_any:
        return []
    if strong:
        return sorted(set(strong) | set(sec_any[:3]))
    if weak_title:                                         # "vehicle" only counts in the title
        return sorted(set(weak_title) | set(sec_any[:3]))
    if sec_title and _hits(AUTO_WEAK_RX, rec["abstract"]):
        return sorted(set(sec_title) | set(_hits(AUTO_WEAK_RX, rec["abstract"])))
    return []


def finalize(rec, matched, known_cves):
    text = rec["title"] + " " + rec["abstract"]
    cves = sorted({c.upper() for c in CVE_RX.findall(text)})
    year = rec["year"] or (int(rec["date"][:4]) if rec["date"][:4].isdigit() else None)
    pid = (f"doi:{rec['doi']}" if rec["doi"] else f"arxiv:{rec['arxiv_id']}" if rec["arxiv_id"]
           else "t:" + hashlib.sha1(title_key(rec["title"]).encode()).hexdigest()[:12])
    return {
        "id": pid, "title": rec["title"], "authors": rec["authors"][:12],
        "author_count": len(rec["authors"]), "year": year, "date": rec["date"],
        "venue": rec["venue"], "abstract": rec["abstract"], "doi": rec["doi"],
        "arxiv_id": rec["arxiv_id"], "url": rec["url"], "pdf_url": rec["pdf_url"],
        "open_access": bool(rec["pdf_url"]), "cited_by": rec["cited_by"],
        "sources": rec["sources"], "matched_terms": matched,
        "topics": [t for t, rx in TOPIC_RX.items() if rx.search(text)],
        "vendors": _hits(OEM_RX, text),
        "top_venue": bool(TOP_VENUES_RX.search(rec["venue"] or "")),
        "cves": cves, "linked_cves": [c for c in cves if c in known_cves],
    }


def load_known_cves():
    try:
        with open(ZERO_DAY_FILE, encoding="utf-8") as f:
            return {r.get("name", "").upper() for r in json.load(f).get("records", [])}
    except Exception:
        return set()


# ============================================================
# PIPELINE
# ============================================================
def collect_raw():
    cache = load_cache()
    raw, per_source = [], Counter()

    def add(source, items):
        raw.extend(items); per_source[source] += len(items)

    logger.info("OpenAlex key: %s | Semantic Scholar key: %s",
                f"set ({len(OPENALEX_KEY)} chars)" if OPENALEX_KEY else "NOT set",
                "set" if S2_KEY else "not set")
    if OPENALEX_KEY:
        for q in OPENALEX_QUERIES:
            add("openalex", cached(cache, f"oa:{q}", lambda q=q: fetch_openalex(search=q), 0.3))
        if SNOWBALL:
            for doi in SEED_DOIS:
                add("openalex", cached(cache, f"oa-cites:{doi}", lambda d=doi: fetch_openalex_citing(d), 0.3))
    else:
        logger.warning("No OpenAlex key found - skipping OpenAlex. Set env var OPENALEX_API_KEY in the "
                       "SAME terminal you run this from, or paste it into INLINE_OPENALEX_KEY.")

    if REGISTER_QUERIES:
        logger.info("%s extra queries from threat_register.json", len(REGISTER_QUERIES))
        if OPENALEX_KEY:
            for q in REGISTER_QUERIES:
                add("openalex", cached(cache, f"oa-reg:{q}", lambda q=q: fetch_openalex(search=q, pages=1), 0.3))
    for q in S2_QUERIES + REGISTER_QUERIES:
        add("semanticscholar", cached(cache, f"s2:{q}", lambda q=q: fetch_s2(q), 0 ))
    for q in ARXIV_QUERIES:
        add("arxiv", cached(cache, f"arxiv:{q}", lambda q=q: fetch_arxiv(q), 3.1))
    for q in DBLP_QUERIES:
        add("dblp", cached(cache, f"dblp:{q}", lambda q=q: fetch_dblp(q), 1.5))
    return raw, per_source


def run_papers():
    print("[PAPERS] Collecting...")
    raw, per_source = collect_raw()
    print(f"[PAPERS] Raw records per source: {dict(per_source)}")

    raw = [r for r in raw if r and r.get("title")]
    unique = dedupe(raw)
    known = load_known_cves()

    records, dropped_year, dropped_rel = [], 0, 0
    for r in unique:
        year = r["year"] or (int(r["date"][:4]) if r["date"][:4].isdigit() else None)
        if year and year < MIN_YEAR:
            dropped_year += 1; continue
        matched = assess(r)
        if not matched:
            dropped_rel += 1; continue
        records.append(finalize(r, matched, known))

    records.sort(key=lambda r: (r["date"] or f"{r['year'] or 0}-01-01"), reverse=True)
    print(f"[PAPERS] raw={len(raw)} unique={len(unique)} too_old={dropped_year} "
          f"not_relevant={dropped_rel} kept={len(records)}")
    return records


def main():
    records = run_papers()
    out = {
        "module": "papers",
        "generated": time.time(),
        "generated_iso": datetime.now(timezone.utc).isoformat(),
        "summary": {"by_year": dict(sorted(Counter(str(r["year"]) for r in records).items(), reverse=True)),
                    "by_source": dict(Counter(s for r in records for s in r["sources"])),
                    "top_venue": sum(r["top_venue"] for r in records),
                    "open_access": sum(r["open_access"] for r in records),
                    "cve_linked": sum(bool(r["cves"]) for r in records)},
        "records": records,
    }
    os.makedirs(os.path.dirname(OUTPUT_FILE), exist_ok=True)
    tmp = OUTPUT_FILE + ".tmp"
    with open(tmp, "w", encoding="utf-8") as f:
        json.dump(out, f, indent=2, ensure_ascii=False)
    os.replace(tmp, OUTPUT_FILE)
    print(f"[PAPERS] Wrote {len(records)} records to {OUTPUT_FILE}")


if __name__ == "__main__":
    main()
