#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
github_cve_fixes.py - enrich automotive CVEs with public GitHub remediation evidence.

Design goals
------------
* Keep the existing CVE pipeline untouched.
* Read zero_day_threats/zero_day.json.
* Write a separate zero_day_threats/cve_fixes.json.
* Read GITHUB_TOKEN from the environment; never write or print the token.
* Use only the Python standard library.
* Cache GitHub API responses and write output atomically.
* Distinguish mere CVE mentions from possible and verified fixes.

Evidence sources
----------------
1. GitHub Global Security Advisories, queried by CVE ID.
2. GitHub commit search, queried by CVE ID.
3. GitHub issue/pull-request search, queried by CVE ID.
4. URLs referenced by advisories, including commits, pull requests and releases.

Classification is deliberately conservative:
* verified_fix: explicit patched version in a GitHub advisory, an explicit fixing
  commit, or a merged pull request with explicit fix language.
* fix_candidate: fix/patch/remediation language is present but verification is
  incomplete.
* mention_only: the CVE is mentioned, but there is no reliable fix evidence.
* no_public_fix_found: no usable GitHub evidence was found.
* lookup_error: one or more required lookups failed and no stronger result exists.

Usage
-----
    python3 github_cve_fixes.py
    python3 github_cve_fixes.py --limit 10 --verbose
    python3 github_cve_fixes.py --refresh
    python3 github_cve_fixes.py --input zero_day_threats/zero_day.json \
        --output zero_day_threats/cve_fixes.json

Environment
-----------
    GITHUB_TOKEN=github_pat_...

The token needs access only to the GitHub resources you want to query. Public
advisory lookup itself can work without authentication, but this script requires
GITHUB_TOKEN so all searches use an authenticated request budget.
"""

from __future__ import annotations

import argparse
import hashlib
import json
import os
import random
import re
import sys
import time
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode, urlparse
from urllib.request import Request, urlopen


API_ROOT = "https://api.github.com"
API_VERSION = "2022-11-28"
DEFAULT_INPUT = Path("zero_day_threats/zero_day.json")
DEFAULT_OUTPUT = Path("zero_day_threats/cve_fixes.json")
DEFAULT_CACHE = Path("zero_day_threats/github_cve_cache.json")
DEFAULT_CACHE_TTL_HOURS = 168
DEFAULT_PER_PAGE = 30
DEFAULT_MAX_COMMITS = 15
DEFAULT_MAX_ISSUES = 15
DEFAULT_REQUEST_DELAY = 1.1
USER_AGENT = "NCRC-github-cve-fixes/1.0"

CVE_RX = re.compile(r"^CVE-\d{4}-\d{4,}$", re.I)
COMMIT_URL_RX = re.compile(
    r"^https://github\.com/([^/]+)/([^/]+)/commit/([0-9a-f]{7,64})(?:[/?#].*)?$",
    re.I,
)
PULL_URL_RX = re.compile(
    r"^https://github\.com/([^/]+)/([^/]+)/(?:pull|pulls)/(\d+)(?:[/?#].*)?$",
    re.I,
)
ISSUE_URL_RX = re.compile(
    r"^https://github\.com/([^/]+)/([^/]+)/issues/(\d+)(?:[/?#].*)?$",
    re.I,
)
RELEASE_URL_RX = re.compile(
    r"^https://github\.com/([^/]+)/([^/]+)/releases(?:/tag/([^/?#]+))?(?:[/?#].*)?$",
    re.I,
)
COMPARE_URL_RX = re.compile(
    r"^https://github\.com/([^/]+)/([^/]+)/compare/([^/?#]+)(?:[/?#].*)?$",
    re.I,
)

FIX_RX = re.compile(
    r"\b(?:fix(?:e[ds])?|fixed|fixing|patch(?:e[ds])?|patched|patching|"
    r"remediat(?:e[ds]?|ion)|resolv(?:e[ds]?|ing)|mitigat(?:e[ds]?|ion)|"
    r"security\s+fix|backport(?:ed|ing)?|hotfix|correct(?:ed|ion)|"
    r"address(?:e[ds]|ing)?|prevent(?:ed|s|ing)?)\b",
    re.I,
)
NEGATIVE_FIX_RX = re.compile(
    r"\b(?:not\s+fixed|unfixed|no\s+fix|without\s+(?:a\s+)?fix|"
    r"workaround\s+only|false\s+positive|revert(?:ed|ing)?\s+(?:the\s+)?fix)\b",
    re.I,
)
SECURITY_RX = re.compile(
    r"\b(?:security|vulnerab(?:ility|le)|exploit|cve|ghsa|advisory|"
    r"overflow|use-after-free|injection|authentication|authorization|"
    r"privilege|denial.of.service|memory.corruption)\b",
    re.I,
)
TEST_RX = re.compile(
    r"\b(?:regression\s+test|security\s+test|test\s+case|unit\s+test|"
    r"integration\s+test|reproducer|proof.of.concept|poc)\b",
    re.I,
)
VERSION_RX = re.compile(
    r"\b(?:fixed|patched|resolved|available)\s+(?:in|since|from)?\s*"
    r"v?\d+(?:\.\d+){1,3}(?:[-+._a-z0-9]*)\b",
    re.I,
)


class GitHubAPIError(RuntimeError):
    """A GitHub API request failed."""


class RateLimitError(GitHubAPIError):
    """GitHub refused a request because a rate limit was reached."""


class JsonCache:
    def __init__(self, path: Path, ttl_hours: int, refresh: bool = False) -> None:
        self.path = path
        self.ttl_seconds = max(0, ttl_hours) * 3600
        self.refresh = refresh
        self.entries: dict[str, dict[str, Any]] = {}
        self.dirty = False
        self.load()

    def load(self) -> None:
        if not self.path.exists():
            return
        try:
            with self.path.open("r", encoding="utf-8") as handle:
                raw = json.load(handle)
            if isinstance(raw, dict) and isinstance(raw.get("entries"), dict):
                self.entries = raw["entries"]
        except (OSError, json.JSONDecodeError):
            self.entries = {}

    @staticmethod
    def key(url: str) -> str:
        return hashlib.sha256(url.encode("utf-8")).hexdigest()

    def get(self, url: str) -> Any | None:
        if self.refresh:
            return None
        entry = self.entries.get(self.key(url))
        if not isinstance(entry, dict):
            return None
        stored_at = entry.get("stored_at")
        if not isinstance(stored_at, (int, float)):
            return None
        if self.ttl_seconds and time.time() - stored_at > self.ttl_seconds:
            return None
        return entry.get("data")

    def set(self, url: str, data: Any) -> None:
        self.entries[self.key(url)] = {
            "url": url,
            "stored_at": time.time(),
            "data": data,
        }
        self.dirty = True

    def save(self) -> None:
        if not self.dirty:
            return
        self.path.parent.mkdir(parents=True, exist_ok=True)
        payload = {
            "schema_version": 1,
            "generated_iso": utc_now(),
            "entries": self.entries,
        }
        atomic_write_json(self.path, payload)
        self.dirty = False


class GitHubClient:
    def __init__(
        self,
        token: str,
        cache: JsonCache,
        request_delay: float,
        timeout: int,
        retries: int,
        verbose: bool = False,
    ) -> None:
        self.token = token
        self.cache = cache
        self.request_delay = max(0.0, request_delay)
        self.timeout = max(5, timeout)
        self.retries = max(0, retries)
        self.verbose = verbose
        self.last_request_at = 0.0
        self.requests = 0
        self.cache_hits = 0
        self.rate_limit_remaining: int | None = None
        self.rate_limit_reset: int | None = None

    def _log(self, message: str) -> None:
        if self.verbose:
            print(f"[github] {message}")

    def _pace(self) -> None:
        elapsed = time.monotonic() - self.last_request_at
        wait = self.request_delay - elapsed
        if wait > 0:
            time.sleep(wait)

    def _headers(self) -> dict[str, str]:
        return {
            "Accept": "application/vnd.github+json",
            "Authorization": f"Bearer {self.token}",
            "X-GitHub-Api-Version": API_VERSION,
            "User-Agent": USER_AGENT,
        }

    def get(self, path: str, params: dict[str, Any] | None = None) -> Any:
        query = urlencode(
            [(key, value) for key, value in (params or {}).items() if value is not None],
            doseq=True,
        )
        url = f"{API_ROOT}{path}"
        if query:
            url += "?" + query

        cached = self.cache.get(url)
        if cached is not None:
            self.cache_hits += 1
            self._log(f"cache hit: {path}")
            return cached

        last_error: Exception | None = None
        for attempt in range(self.retries + 1):
            self._pace()
            request = Request(url, headers=self._headers(), method="GET")
            try:
                self._log(f"GET {path}")
                with urlopen(request, timeout=self.timeout) as response:
                    self.last_request_at = time.monotonic()
                    self.requests += 1
                    self._capture_rate_limit(response.headers)
                    raw = response.read().decode("utf-8")
                    data = json.loads(raw) if raw else None
                    self.cache.set(url, data)
                    return data
            except HTTPError as exc:
                self.last_request_at = time.monotonic()
                self.requests += 1
                self._capture_rate_limit(exc.headers)
                body = self._error_body(exc)
                message = self._error_message(body, exc.reason)

                if exc.code in (403, 429) and self._is_rate_limited(exc, message):
                    reset = self.rate_limit_reset
                    reset_text = (
                        datetime.fromtimestamp(reset, tz=timezone.utc).isoformat()
                        if reset
                        else "unknown"
                    )
                    raise RateLimitError(
                        f"GitHub rate limit reached for {path}; reset={reset_text}; "
                        f"message={message}"
                    ) from exc

                if exc.code >= 500 and attempt < self.retries:
                    last_error = exc
                    self._retry_sleep(attempt)
                    continue

                raise GitHubAPIError(
                    f"GitHub API returned HTTP {exc.code} for {path}: {message}"
                ) from exc
            except (URLError, TimeoutError, json.JSONDecodeError) as exc:
                self.last_request_at = time.monotonic()
                last_error = exc
                if attempt < self.retries:
                    self._retry_sleep(attempt)
                    continue
                raise GitHubAPIError(f"GitHub request failed for {path}: {exc}") from exc

        raise GitHubAPIError(f"GitHub request failed for {path}: {last_error}")

    def _retry_sleep(self, attempt: int) -> None:
        delay = min(30.0, (2 ** attempt) + random.random())
        self._log(f"temporary failure; retry in {delay:.1f}s")
        time.sleep(delay)

    def _capture_rate_limit(self, headers: Any) -> None:
        if headers is None:
            return
        try:
            remaining = headers.get("X-RateLimit-Remaining")
            reset = headers.get("X-RateLimit-Reset")
            self.rate_limit_remaining = int(remaining) if remaining is not None else None
            self.rate_limit_reset = int(reset) if reset is not None else None
        except (TypeError, ValueError):
            pass

    @staticmethod
    def _error_body(exc: HTTPError) -> Any:
        try:
            raw = exc.read().decode("utf-8", errors="replace")
            return json.loads(raw) if raw else {}
        except (OSError, json.JSONDecodeError):
            return {}

    @staticmethod
    def _error_message(body: Any, fallback: Any) -> str:
        if isinstance(body, dict) and body.get("message"):
            return str(body["message"])
        return str(fallback or "unknown error")

    def _is_rate_limited(self, exc: HTTPError, message: str) -> bool:
        remaining = None
        try:
            remaining = exc.headers.get("X-RateLimit-Remaining")
        except AttributeError:
            pass
        return (
            str(remaining) == "0"
            or "rate limit" in message.casefold()
            or "secondary rate limit" in message.casefold()
        )


# ---------------------------------------------------------------------------
# Generic helpers
# ---------------------------------------------------------------------------


def utc_now() -> str:
    return datetime.now(timezone.utc).isoformat()


def clean(value: Any) -> str:
    return re.sub(r"\s+", " ", str(value or "")).strip()


def short_text(value: Any, limit: int = 500) -> str:
    text = clean(value)
    if len(text) <= limit:
        return text
    return text[: limit - 3].rsplit(" ", 1)[0].rstrip() + "..."


def atomic_write_json(path: Path, payload: Any) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    temporary = path.with_name(path.name + ".tmp")
    with temporary.open("w", encoding="utf-8") as handle:
        json.dump(payload, handle, indent=2, ensure_ascii=False, sort_keys=False)
        handle.write("\n")
        handle.flush()
        os.fsync(handle.fileno())
    os.replace(temporary, path)


def load_json(path: Path) -> dict[str, Any]:
    if not path.exists():
        raise FileNotFoundError(f"Input file not found: {path}")
    with path.open("r", encoding="utf-8") as handle:
        data = json.load(handle)
    if not isinstance(data, dict):
        raise ValueError(f"Expected a JSON object in {path}")
    return data


def as_dict(value: Any) -> dict[str, Any]:
    """Return value as a dict, or an empty dict for any other JSON shape."""
    return value if isinstance(value, dict) else {}


def as_list(value: Any) -> list[Any]:
    """Return value as a list; wrap scalar/dict values and ignore null."""
    if value is None:
        return []
    if isinstance(value, list):
        return value
    if isinstance(value, tuple):
        return list(value)
    return [value]


def scalar_text(value: Any, *preferred_keys: str) -> str:
    """Safely normalize strings, numbers, booleans and common API objects."""
    if value is None:
        return ""
    if isinstance(value, str):
        return clean(value)
    if isinstance(value, (int, float, bool)):
        return clean(value)
    if isinstance(value, dict):
        keys = preferred_keys or (
            "identifier", "name", "value", "url", "html_url", "login", "full_name"
        )
        for key in keys:
            if key in value and value[key] is not None:
                text = scalar_text(value[key])
                if text:
                    return text
        return ""
    if isinstance(value, (list, tuple, set)):
        return ", ".join(unique_strings(scalar_text(x) for x in value))
    return clean(value)


def dict_text(value: Any, key: str, *fallback_keys: str) -> str:
    obj = as_dict(value)
    for candidate in (key, *fallback_keys):
        if candidate in obj:
            text = scalar_text(obj.get(candidate))
            if text:
                return text
    return ""


def nested_text(value: Any, *path: str) -> str:
    current = value
    for key in path:
        if not isinstance(current, dict):
            return ""
        current = current.get(key)
    return scalar_text(current)


def cve_id_from_record(record: dict[str, Any]) -> str:
    record = as_dict(record)
    value = scalar_text(
        record.get("name")
        or record.get("cve_id")
        or record.get("id")
        or record.get("cve"),
        "id",
        "name",
        "cve_id",
    ).upper()
    return value if CVE_RX.fullmatch(value) else ""


def input_records(data: dict[str, Any]) -> list[dict[str, Any]]:
    for key in ("records", "cves", "items", "vulnerabilities"):
        value = data.get(key)
        if isinstance(value, list):
            return [item for item in value if isinstance(item, dict)]
    raise ValueError("No CVE records found; expected records/cves/items/vulnerabilities")


def unique_strings(values: Iterable[Any]) -> list[str]:
    output: list[str] = []
    seen: set[str] = set()
    for value in values:
        text = clean(value)
        key = text.casefold()
        if text and key not in seen:
            output.append(text)
            seen.add(key)
    return output


def safe_float(value: Any) -> float:
    try:
        return float(value)
    except (TypeError, ValueError):
        return 0.0


def has_explicit_cve(text: str, cve_id: str) -> bool:
    return cve_id.casefold() in text.casefold()


def fix_language(text: str) -> bool:
    return bool(FIX_RX.search(text)) and not bool(NEGATIVE_FIX_RX.search(text))


def get_repo_from_api_item(item: dict[str, Any]) -> str:
    item = as_dict(item)
    repository = item.get("repository")
    if isinstance(repository, dict):
        full_name = scalar_text(repository.get("full_name"), "full_name", "name")
        if full_name:
            return full_name
    elif isinstance(repository, str) and "/" in repository:
        return clean(repository)

    repo_url = scalar_text(item.get("repository_url"), "url")
    match = re.search(r"/repos/([^/]+/[^/]+)$", repo_url)
    return match.group(1) if match else ""


def classify_reference_url(url: str) -> dict[str, Any] | None:
    url = clean(url)
    if not url:
        return None

    match = COMMIT_URL_RX.match(url)
    if match:
        owner, repo, sha = match.groups()
        return {
            "kind": "commit_reference",
            "repository": f"{owner}/{repo}",
            "sha": sha,
            "url": url,
        }

    match = PULL_URL_RX.match(url)
    if match:
        owner, repo, number = match.groups()
        return {
            "kind": "pull_request_reference",
            "repository": f"{owner}/{repo}",
            "number": int(number),
            "url": url,
        }

    match = ISSUE_URL_RX.match(url)
    if match:
        owner, repo, number = match.groups()
        return {
            "kind": "issue_reference",
            "repository": f"{owner}/{repo}",
            "number": int(number),
            "url": url,
        }

    match = RELEASE_URL_RX.match(url)
    if match:
        owner, repo, tag = match.groups()
        return {
            "kind": "release_reference",
            "repository": f"{owner}/{repo}",
            "tag": tag or "",
            "url": url,
        }

    match = COMPARE_URL_RX.match(url)
    if match:
        owner, repo, comparison = match.groups()
        return {
            "kind": "compare_reference",
            "repository": f"{owner}/{repo}",
            "comparison": comparison,
            "url": url,
        }

    return {
        "kind": "external_reference",
        "host": urlparse(url).netloc.casefold(),
        "url": url,
    }


def evidence_key(item: dict[str, Any]) -> tuple[Any, ...]:
    return (
        item.get("kind"),
        item.get("url"),
        item.get("repository"),
        item.get("sha"),
        item.get("number"),
        item.get("ghsa_id"),
    )


def deduplicate_evidence(items: Iterable[dict[str, Any]]) -> list[dict[str, Any]]:
    output: list[dict[str, Any]] = []
    seen: set[tuple[Any, ...]] = set()
    for item in items:
        key = evidence_key(item)
        if key in seen:
            continue
        seen.add(key)
        output.append(item)
    return output


# Repositories that usually mirror CVE metadata, PoCs, reports or bulk updates.
# Results from these repositories remain visible as mentions, but cannot by
# themselves promote a CVE to verified_fix.
NON_PRODUCT_REPO_RX = re.compile(
    r"(?:^|/)(?:"
    r"cvelistv5|cvelistv5-cveservicesadaptertest|cve[_-]?reports?|"
    r"poc-cve-collection|cve-poc|vulnerability-poc|threatvault|"
    r"linux-cve-announce(?:\\.0)?|linux_security_vulns|"
    r"cve-research|cve-updates|cybervault-cve-intel|"
    r"mitre_mapping_site|web-csirt"
    r")(?:$|[-_/])",
    re.I,
)

GENERIC_UPDATE_RX = re.compile(
    r"\\b(?:sync|bulk|database|dataset|feed|catalog|cvelist|threat[ -]?vault|"
    r"layer updates?|update layers?|base\\.lock|metadata|mirror)\\b",
    re.I,
)


def repository_is_non_product(repository: Any) -> bool:
    return bool(NON_PRODUCT_REPO_RX.search(clean(repository)))


def direct_cve_fix_language(message: Any, cve_id: str) -> bool:
    """Require fix/patch wording to be directly connected to this CVE."""
    text = clean(message)
    escaped = re.escape(cve_id)
    fix = r"(?:fix(?:e[ds])?|patch(?:e[ds])?|patched|backport(?:ed|ing)?)"
    patterns = (
        rf"\\b{fix}\\b[^\\n.]{{0,80}}\\b{escaped}\\b",
        rf"\\b{escaped}\\b[^\\n.]{{0,80}}\\b{fix}\\b",
    )
    return any(re.search(pattern, text, re.I) for pattern in patterns)


def commit_repositories(commit: dict[str, Any]) -> list[str]:
    values = as_list(commit.get("repositories")) + [commit.get("repository")]
    return sorted(unique_strings(values), key=str.casefold)


def preferred_product_repository(commit: dict[str, Any]) -> str:
    repositories = commit_repositories(commit)
    product = [repo for repo in repositories if not repository_is_non_product(repo)]
    if product:
        # Prefer the shortest path as a deterministic proxy for the least
        # nested/mirror-like repository name. The full alternative list remains.
        return sorted(product, key=lambda value: (len(value), value.casefold()))[0]
    return repositories[0] if repositories else ""


def commit_fix_score(commit: dict[str, Any], cve_id: str) -> tuple[int, list[str]]:
    """Score a SHA-deduplicated commit as remediation evidence."""
    message = clean(commit.get("message"))
    repository = preferred_product_repository(commit)
    reasons: list[str] = []
    score = 0

    if not commit.get("explicit_cve_reference") or cve_id.casefold() not in message.casefold():
        return 0, ["no_explicit_cve_in_commit_message"]

    score += 15
    reasons.append("explicit_cve_in_commit_message")

    if direct_cve_fix_language(message, cve_id):
        score += 45
        reasons.append("fix_language_directly_linked_to_cve")
    elif commit.get("fix_language"):
        score += 10
        reasons.append("generic_fix_language")

    if commit.get("security_context"):
        score += 5
        reasons.append("security_context")
    if commit.get("regression_test_language"):
        score += 5
        reasons.append("regression_test_language")

    if repository_is_non_product(repository):
        score -= 60
        reasons.append("non_product_repository")
    else:
        score += 10
        reasons.append("product_or_upstream_repository_candidate")

    if GENERIC_UPDATE_RX.search(message):
        score -= 25
        reasons.append("generic_or_bulk_update_message")

    if len(message) <= 300:
        score += 5
        reasons.append("focused_commit_message")

    return max(0, min(100, score)), reasons


def advisory_commit_shas(references: list[dict[str, Any]]) -> set[str]:
    return {
        clean(item.get("sha")).casefold()
        for item in references
        if isinstance(item, dict)
        and item.get("kind") == "commit_reference"
        and clean(item.get("sha"))
    }


def select_primary_fix(
    cve_id: str,
    advisories: list[dict[str, Any]],
    commits: list[dict[str, Any]],
    issues: list[dict[str, Any]],
    references: list[dict[str, Any]],
) -> tuple[dict[str, Any] | None, list[dict[str, Any]]]:
    """Return one primary remediation object plus ranked commit candidates."""
    reviewed_versions: list[dict[str, Any]] = []
    for advisory in advisories:
        if advisory.get("advisory_type") != "reviewed":
            continue
        for version in as_list(advisory.get("fixed_versions")):
            if isinstance(version, dict) and clean(version.get("version")):
                reviewed_versions.append({
                    "type": "patched_version",
                    "verification_level": "verified",
                    "score": 100,
                    "cve_id": cve_id,
                    "ghsa_id": clean(advisory.get("ghsa_id")),
                    "advisory_url": clean(advisory.get("url")),
                    "ecosystem": clean(version.get("ecosystem")),
                    "package": clean(version.get("package")),
                    "version": clean(version.get("version")),
                    "reasons": ["reviewed_github_advisory_documents_patched_version"],
                })

    referenced_shas = advisory_commit_shas(references)
    ranked: list[dict[str, Any]] = []
    for commit in commits:
        if not isinstance(commit, dict):
            continue
        score, reasons = commit_fix_score(commit, cve_id)
        sha = clean(commit.get("sha"))
        advisory_reference = bool(sha and sha.casefold() in referenced_shas)
        if advisory_reference:
            score = max(score, 95)
            reasons.append("github_advisory_references_exact_sha")

        repository = preferred_product_repository(commit)
        verification_level = (
            "verified"
            if advisory_reference and direct_cve_fix_language(commit.get("message"), cve_id)
            else "candidate"
            if score >= 60 and direct_cve_fix_language(commit.get("message"), cve_id)
            else "mention"
        )
        ranked.append({
            "type": "commit",
            "verification_level": verification_level,
            "score": score,
            "cve_id": cve_id,
            "repository": repository,
            "repositories": commit_repositories(commit),
            "sha": sha,
            "url": clean(commit.get("url")),
            "message": clean(commit.get("message")),
            "duplicate_count": int(commit.get("duplicate_count", 1) or 1),
            "reasons": unique_strings(reasons),
        })

    ranked.sort(
        key=lambda item: (
            item.get("verification_level") == "verified",
            item.get("verification_level") == "candidate",
            int(item.get("score", 0)),
            clean(item.get("sha")),
        ),
        reverse=True,
    )

    if reviewed_versions:
        reviewed_versions.sort(
            key=lambda item: (
                clean(item.get("package")).casefold(),
                clean(item.get("version")).casefold(),
            )
        )
        return reviewed_versions[0], ranked

    primary_commit = next(
        (item for item in ranked if item.get("verification_level") in {"verified", "candidate"}),
        None,
    )
    return primary_commit, ranked


def strict_remediation_status(
    primary_fix: dict[str, Any] | None,
    advisories: list[dict[str, Any]],
    commits: list[dict[str, Any]],
    issues: list[dict[str, Any]],
    references: list[dict[str, Any]],
    errors: list[str],
) -> tuple[str, str, list[str]]:
    if primary_fix and primary_fix.get("verification_level") == "verified":
        return "verified_fix", "high", unique_strings(primary_fix.get("reasons", []))
    if primary_fix and primary_fix.get("verification_level") == "candidate":
        return "fix_candidate", "medium", unique_strings(primary_fix.get("reasons", []))
    if advisories or commits or issues or references:
        return "mention_only", "low", ["github_mentions_found_without_strictly_verified_fix"]
    if errors:
        return "lookup_error", "unknown", ["lookup_incomplete_due_to_errors"]
    return "no_public_fix_found", "none", ["no_github_evidence_found"]


# ---------------------------------------------------------------------------
# GitHub evidence collectors
# ---------------------------------------------------------------------------


def collect_advisories(client: GitHubClient, cve_id: str) -> tuple[list[dict[str, Any]], list[str]]:
    evidence: list[dict[str, Any]] = []
    errors: list[str] = []

    for advisory_type in ("reviewed", "unreviewed"):
        try:
            response = client.get(
                "/advisories",
                {"cve_id": cve_id, "type": advisory_type, "per_page": 100},
            )
        except GitHubAPIError as exc:
            errors.append(f"advisory:{advisory_type}: {exc}")
            continue

        for raw_advisory in as_list(response):
            advisory = as_dict(raw_advisory)
            if not advisory:
                continue

            vulnerabilities: list[dict[str, str]] = []
            fixed_versions: list[dict[str, str]] = []

            for raw_vulnerability in as_list(advisory.get("vulnerabilities")):
                vulnerability = as_dict(raw_vulnerability)
                if not vulnerability:
                    continue

                raw_package = vulnerability.get("package")
                package = as_dict(raw_package)
                ecosystem = scalar_text(
                    package.get("ecosystem") if package else "",
                    "ecosystem",
                )
                package_name = scalar_text(
                    package.get("name") if package else raw_package,
                    "name",
                    "package",
                )

                raw_patched = vulnerability.get("first_patched_version")
                patched_version = scalar_text(
                    raw_patched,
                    "identifier",
                    "version",
                    "name",
                    "value",
                )

                vulnerable_range = scalar_text(
                    vulnerability.get("vulnerable_version_range"),
                    "identifier",
                    "range",
                    "value",
                )

                item = {
                    "ecosystem": ecosystem,
                    "package": package_name,
                    "vulnerable_version_range": vulnerable_range,
                    "first_patched_version": patched_version,
                }
                vulnerabilities.append(item)

                if patched_version:
                    fixed_versions.append(
                        {
                            "ecosystem": ecosystem,
                            "package": package_name,
                            "version": patched_version,
                            "source": "github_advisory",
                        }
                    )

            references: list[dict[str, Any]] = []
            for raw_reference in as_list(advisory.get("references")):
                if isinstance(raw_reference, dict):
                    ref_url = scalar_text(raw_reference, "url", "html_url", "href")
                else:
                    ref_url = scalar_text(raw_reference)
                classified = classify_reference_url(ref_url)
                if classified:
                    references.append(classified)

            ghsa_id = scalar_text(advisory.get("ghsa_id"), "ghsa_id", "id")
            evidence.append(
                {
                    "kind": "github_advisory",
                    "ghsa_id": ghsa_id,
                    "cve_id": scalar_text(advisory.get("cve_id"), "cve_id", "id") or cve_id,
                    "advisory_type": advisory_type,
                    "severity": scalar_text(advisory.get("severity"), "name", "value"),
                    "summary": short_text(scalar_text(advisory.get("summary")), 400),
                    "description": short_text(scalar_text(advisory.get("description")), 900),
                    "published_at": scalar_text(advisory.get("published_at")),
                    "updated_at": scalar_text(advisory.get("updated_at")),
                    "withdrawn_at": scalar_text(advisory.get("withdrawn_at")),
                    "url": scalar_text(advisory.get("html_url") or advisory.get("url"), "url", "html_url"),
                    "vulnerabilities": vulnerabilities,
                    "fixed_versions": fixed_versions,
                    "references": references,
                    "explicit_cve_reference": True,
                    "fix_language": bool(fixed_versions),
                    "verification": (
                        "patched_version_documented"
                        if fixed_versions
                        else "advisory_without_patched_version"
                    ),
                }
            )

    return evidence, errors


def collect_commits(
    client: GitHubClient,
    cve_id: str,
    max_results: int,
) -> tuple[list[dict[str, Any]], list[str]]:
    evidence: list[dict[str, Any]] = []
    if max_results <= 0:
        return evidence, []

    try:
        response = client.get(
            "/search/commits",
            {
                "q": f'"{cve_id}"',
                "sort": "committer-date",
                "order": "desc",
                "per_page": min(max_results, 100),
            },
        )
    except GitHubAPIError as exc:
        return [], [f"commits: {exc}"]

    response_obj = as_dict(response)
    for raw_item in as_list(response_obj.get("items")):
        item = as_dict(raw_item)
        if not item:
            continue
        commit = as_dict(item.get("commit"))
        author = as_dict(commit.get("author"))
        committer = as_dict(commit.get("committer"))
        message = scalar_text(commit.get("message"))
        explicit = has_explicit_cve(message, cve_id)
        fixing = explicit and fix_language(message)

        evidence.append(
            {
                "kind": "commit_search_result",
                "repository": get_repo_from_api_item(item),
                "sha": scalar_text(item.get("sha"), "sha", "id"),
                "url": scalar_text(item.get("html_url") or item.get("url"), "url", "html_url"),
                "message": short_text(message, 700),
                "author_date": scalar_text(author.get("date")),
                "committer_date": scalar_text(committer.get("date")),
                "explicit_cve_reference": explicit,
                "fix_language": fixing,
                "security_context": bool(SECURITY_RX.search(message)),
                "regression_test_language": bool(TEST_RX.search(message)),
                "verification": "explicit_cve_fix_commit" if fixing else "cve_mention_in_commit",
            }
        )

    return evidence[:max_results], []


def collect_issues_and_pulls(
    client: GitHubClient,
    cve_id: str,
    max_results: int,
) -> tuple[list[dict[str, Any]], list[str]]:
    evidence: list[dict[str, Any]] = []
    if max_results <= 0:
        return evidence, []

    try:
        response = client.get(
            "/search/issues",
            {
                "q": f'"{cve_id}" in:title,body',
                "sort": "updated",
                "order": "desc",
                "per_page": min(max_results, 100),
            },
        )
    except GitHubAPIError as exc:
        return [], [f"issues: {exc}"]

    response_obj = as_dict(response)
    for raw_item in as_list(response_obj.get("items")):
        item = as_dict(raw_item)
        if not item:
            continue

        title = scalar_text(item.get("title"))
        body = scalar_text(item.get("body"))
        text = f"{title} {body}".strip()
        html_url = scalar_text(item.get("html_url") or item.get("url"), "url", "html_url")
        parsed = classify_reference_url(html_url) or {}
        raw_pull = item.get("pull_request")
        pull = as_dict(raw_pull)
        is_pull_request = bool(raw_pull)
        explicit = has_explicit_cve(text, cve_id)
        fixing = explicit and fix_language(text)
        merged_at = scalar_text(pull.get("merged_at"))

        evidence.append(
            {
                "kind": "pull_request_search_result" if is_pull_request else "issue_search_result",
                "repository": scalar_text(parsed.get("repository")),
                "number": item.get("number") if isinstance(item.get("number"), int) else scalar_text(item.get("number")),
                "url": html_url,
                "title": short_text(title, 300),
                "body_excerpt": short_text(body, 700),
                "state": scalar_text(item.get("state"), "name", "value"),
                "created_at": scalar_text(item.get("created_at")),
                "updated_at": scalar_text(item.get("updated_at")),
                "closed_at": scalar_text(item.get("closed_at")),
                "merged_at": merged_at,
                "explicit_cve_reference": explicit,
                "fix_language": fixing,
                "security_context": bool(SECURITY_RX.search(text)),
                "regression_test_language": bool(TEST_RX.search(text)),
                "version_language": bool(VERSION_RX.search(text)),
                "verification": (
                    "merged_explicit_cve_fix_pull_request"
                    if is_pull_request and merged_at and fixing
                    else "explicit_cve_fix_candidate"
                    if fixing
                    else "cve_mention_in_pull_request"
                    if is_pull_request
                    else "cve_mention_in_issue"
                ),
            }
        )

    return evidence[:max_results], []


# ---------------------------------------------------------------------------
# Classification and output shaping
# ---------------------------------------------------------------------------


def flatten_advisory_references(advisories: list[dict[str, Any]]) -> list[dict[str, Any]]:
    output: list[dict[str, Any]] = []
    for advisory in advisories:
        ghsa_id = advisory.get("ghsa_id")
        advisory_type = advisory.get("advisory_type")
        for reference in advisory.get("references", []) or []:
            if not isinstance(reference, dict):
                continue
            item = dict(reference)
            item["source"] = "github_advisory_reference"
            item["ghsa_id"] = ghsa_id
            item["advisory_type"] = advisory_type
            item["explicit_cve_reference"] = True
            output.append(item)
    return output


def remediation_status(
    advisories: list[dict[str, Any]],
    commits: list[dict[str, Any]],
    issues: list[dict[str, Any]],
    references: list[dict[str, Any]],
    errors: list[str],
) -> tuple[str, str, list[str]]:
    reasons: list[str] = []

    reviewed_patched = any(
        item.get("advisory_type") == "reviewed" and item.get("fixed_versions")
        for item in advisories
    )
    any_patched = any(item.get("fixed_versions") for item in advisories)
    explicit_fix_commits = [item for item in commits if item.get("fix_language")]
    merged_fix_prs = [
        item
        for item in issues
        if item.get("kind") == "pull_request_search_result"
        and item.get("merged_at")
        and item.get("fix_language")
    ]
    advisory_commit_refs = [
        item for item in references if item.get("kind") == "commit_reference"
    ]
    advisory_pull_refs = [
        item for item in references if item.get("kind") == "pull_request_reference"
    ]
    candidates = [
        item
        for item in commits + issues
        if item.get("explicit_cve_reference") and item.get("fix_language")
    ]
    mentions = advisories + commits + issues + references

    if reviewed_patched:
        reasons.append("reviewed_github_advisory_documents_patched_version")
    if explicit_fix_commits:
        reasons.append("explicit_cve_fix_commit_found")
    if merged_fix_prs:
        reasons.append("merged_explicit_cve_fix_pull_request_found")
    if advisory_commit_refs:
        reasons.append("github_advisory_references_commit")
    if advisory_pull_refs:
        reasons.append("github_advisory_references_pull_request")

    if reviewed_patched or explicit_fix_commits or merged_fix_prs:
        return "verified_fix", "high", unique_strings(reasons)

    if any_patched:
        reasons.append("unreviewed_advisory_documents_patched_version")
        return "fix_candidate", "medium", unique_strings(reasons)

    if candidates or advisory_commit_refs or advisory_pull_refs:
        if candidates:
            reasons.append("explicit_fix_language_requires_manual_verification")
        return "fix_candidate", "medium", unique_strings(reasons)

    if mentions:
        reasons.append("github_mentions_found_without_verified_fix_evidence")
        return "mention_only", "low", unique_strings(reasons)

    if errors:
        reasons.append("lookup_incomplete_due_to_errors")
        return "lookup_error", "unknown", unique_strings(reasons)

    return "no_public_fix_found", "none", ["no_github_evidence_found"]


def repositories_from_evidence(items: Iterable[dict[str, Any]]) -> list[str]:
    return sorted(
        unique_strings(item.get("repository") for item in items if isinstance(item, dict)),
        key=str.casefold,
    )


def enrich_cve(
    client: GitHubClient,
    source_record: dict[str, Any],
    max_commits: int,
    max_issues: int,
) -> dict[str, Any]:
    cve_id = cve_id_from_record(source_record)
    if not cve_id:
        return {
            "cve_id": "",
            "remediation_status": "invalid_cve_id",
            "confidence": "none",
            "reasons": ["source_record_has_no_valid_cve_id"],
            "errors": [],
        }

    advisories, advisory_errors = collect_advisories(client, cve_id)
    commits, commit_errors = collect_commits(client, cve_id, max_commits)
    issues, issue_errors = collect_issues_and_pulls(client, cve_id, max_issues)
    references = flatten_advisory_references(advisories)
    errors = advisory_errors + commit_errors + issue_errors

    advisories = deduplicate_evidence(advisories)
    commits = deduplicate_evidence(commits)
    issues = deduplicate_evidence(issues)
    references = deduplicate_evidence(references)

    primary_fix, ranked_fix_candidates = select_primary_fix(
        cve_id, advisories, commits, issues, references
    )
    status, confidence, reasons = strict_remediation_status(
        primary_fix, advisories, commits, issues, references, errors
    )

    fixed_versions: list[dict[str, str]] = []
    seen_fixed_versions: set[tuple[str, str, str, str]] = set()
    for advisory in advisories:
        for raw_version in advisory.get("fixed_versions", []) or []:
            if not isinstance(raw_version, dict):
                continue
            item = {
                "ecosystem": clean(raw_version.get("ecosystem")),
                "package": clean(raw_version.get("package")),
                "version": clean(raw_version.get("version")),
                "source": clean(raw_version.get("source")),
            }
            key = (
                item["ecosystem"].casefold(),
                item["package"].casefold(),
                item["version"].casefold(),
                item["source"].casefold(),
            )
            if item["version"] and key not in seen_fixed_versions:
                fixed_versions.append(item)
                seen_fixed_versions.add(key)

    all_evidence = advisories + commits + issues + references
    return {
        "cve_id": cve_id,
        "source": {
            "published": scalar_text(as_dict(source_record).get("published")),
            "severity": scalar_text(as_dict(source_record).get("score"), "name", "value"),
            "cvss": as_dict(source_record).get("cvss"),
            "surfaces": unique_strings(as_list(as_dict(source_record).get("surfaces"))),
            "nvd_url": scalar_text(as_dict(source_record).get("url"), "url", "html_url"),
        },
        "remediation_status": status,
        "confidence": confidence,
        "reasons": reasons,
        "primary_fix": primary_fix,
        "fix_candidates": [
            item for item in ranked_fix_candidates
            if item.get("verification_level") in {"verified", "candidate"}
        ][:10],
        "repositories": repositories_from_evidence(all_evidence),
        "fixed_versions": fixed_versions,
        "counts": {
            "advisories": len(advisories),
            "commits": len(commits),
            "issues_and_pull_requests": len(issues),
            "advisory_references": len(references),
            "fixed_versions": len(fixed_versions),
        },
        "github": {
            "advisories": advisories,
            "commits": commits,
            "issues_and_pull_requests": issues,
            "advisory_references": references,
        },
        "errors": errors,
    }


# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description="Find public GitHub remediation evidence for automotive CVEs."
    )
    parser.add_argument("--input", type=Path, default=DEFAULT_INPUT)
    parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
    parser.add_argument("--cache", type=Path, default=DEFAULT_CACHE)
    parser.add_argument(
        "--cache-ttl-hours", type=int, default=DEFAULT_CACHE_TTL_HOURS
    )
    parser.add_argument(
        "--refresh",
        action="store_true",
        help="Ignore cached GitHub responses and fetch them again.",
    )
    parser.add_argument(
        "--limit",
        type=int,
        default=0,
        help="Process only the first N CVEs; 0 means all CVEs.",
    )
    parser.add_argument(
        "--cve",
        action="append",
        default=[],
        help="Process only this CVE ID; may be repeated.",
    )
    parser.add_argument("--max-commits", type=int, default=DEFAULT_MAX_COMMITS)
    parser.add_argument("--max-issues", type=int, default=DEFAULT_MAX_ISSUES)
    parser.add_argument(
        "--request-delay", type=float, default=DEFAULT_REQUEST_DELAY
    )
    parser.add_argument("--timeout", type=int, default=30)
    parser.add_argument("--retries", type=int, default=2)
    parser.add_argument("--verbose", action="store_true")
    return parser.parse_args()


def validate_args(args: argparse.Namespace) -> None:
    if args.limit < 0:
        raise ValueError("--limit must be 0 or greater")
    if args.max_commits < 0:
        raise ValueError("--max-commits must be 0 or greater")
    if args.max_issues < 0:
        raise ValueError("--max-issues must be 0 or greater")
    if args.cache_ttl_hours < 0:
        raise ValueError("--cache-ttl-hours must be 0 or greater")
    if args.request_delay < 0:
        raise ValueError("--request-delay must be 0 or greater")


def select_records(
    records: list[dict[str, Any]],
    requested_cves: list[str],
    limit: int,
) -> list[dict[str, Any]]:
    requested = {clean(value).upper() for value in requested_cves if clean(value)}
    selected = []
    seen: set[str] = set()

    for record in records:
        cve_id = cve_id_from_record(record)
        if not cve_id or cve_id in seen:
            continue
        if requested and cve_id not in requested:
            continue
        selected.append(record)
        seen.add(cve_id)
        if limit and len(selected) >= limit:
            break

    if requested:
        missing = sorted(requested - seen)
        if missing:
            print(
                "[WARN] Requested CVEs not present in input: " + ", ".join(missing),
                file=sys.stderr,
            )
    return selected


def main() -> int:
    args = parse_args()
    try:
        validate_args(args)
        token = clean(os.environ.get("GITHUB_TOKEN"))
        if not token:
            raise ValueError("GITHUB_TOKEN is not set in the environment")

        source_data = load_json(args.input)
        source_records = input_records(source_data)
        selected = select_records(source_records, args.cve, args.limit)
        if not selected:
            raise ValueError("No valid CVE records selected")

        cache = JsonCache(
            path=args.cache,
            ttl_hours=args.cache_ttl_hours,
            refresh=args.refresh,
        )
        client = GitHubClient(
            token=token,
            cache=cache,
            request_delay=args.request_delay,
            timeout=args.timeout,
            retries=args.retries,
            verbose=args.verbose,
        )

        print(
            f"[GITHUB-CVE] Processing {len(selected)} of "
            f"{len(source_records)} input records"
        )

        output_records: list[dict[str, Any]] = []
        stopped_early = False
        stop_reason = ""

        for index, source_record in enumerate(selected, start=1):
            cve_id = cve_id_from_record(source_record)
            print(f"[GITHUB-CVE] {index}/{len(selected)} {cve_id}")
            try:
                result = enrich_cve(
                    client=client,
                    source_record=source_record,
                    max_commits=args.max_commits,
                    max_issues=args.max_issues,
                )
                output_records.append(result)
                print(
                    f"[GITHUB-CVE] {cve_id}: "
                    f"{result['remediation_status']} "
                    f"({result['confidence']})"
                )
            except RateLimitError as exc:
                stopped_early = True
                stop_reason = str(exc)
                print(f"[WARN] {exc}", file=sys.stderr)
                break
            except Exception as exc:
                output_records.append({
                    "cve_id": cve_id,
                    "remediation_status": "parse_error",
                    "confidence": "unknown",
                    "reasons": ["unexpected_record_shape_or_parser_error"],
                    "primary_fix": None,
                    "fix_candidates": [],
                    "repositories": [],
                    "fixed_versions": [],
                    "counts": {
                        "advisories": 0,
                        "commits": 0,
                        "issues_and_pull_requests": 0,
                        "advisory_references": 0,
                        "fixed_versions": 0,
                    },
                    "github": {
                        "advisories": [],
                        "commits": [],
                        "issues_and_pull_requests": [],
                        "advisory_references": [],
                    },
                    "errors": [f"{type(exc).__name__}: {exc}"],
                })
                print(
                    f"[WARN] {cve_id}: parse_error: {type(exc).__name__}: {exc}",
                    file=sys.stderr,
                )
            finally:
                cache.save()

        summary = Counter(
            clean(record.get("remediation_status")) or "unknown"
            for record in output_records
        )
        confidence_summary = Counter(
            clean(record.get("confidence")) or "unknown"
            for record in output_records
        )

        payload = {
            "schema_version": 1,
            "module": "github_cve_fixes",
            "generated_iso": utc_now(),
            "source_file": str(args.input),
            "source_generated_iso": clean(
                source_data.get("generated_iso") or source_data.get("generated")
            ),
            "complete": not stopped_early and len(output_records) == len(selected),
            "stopped_early": stopped_early,
            "stop_reason": stop_reason,
            "configuration": {
                "max_commits_per_cve": args.max_commits,
                "max_issues_per_cve": args.max_issues,
                "cache_ttl_hours": args.cache_ttl_hours,
                "request_delay_seconds": args.request_delay,
                "refresh": bool(args.refresh),
            },
            "statistics": {
                "source_records": len(source_records),
                "selected_records": len(selected),
                "processed_records": len(output_records),
                "status": dict(sorted(summary.items())),
                "confidence": dict(sorted(confidence_summary.items())),
                "github_api_requests": client.requests,
                "cache_hits": client.cache_hits,
                "rate_limit_remaining": client.rate_limit_remaining,
                "rate_limit_reset": client.rate_limit_reset,
            },
            "records": output_records,
        }

        atomic_write_json(args.output, payload)
        cache.save()

        print(f"[GITHUB-CVE] Wrote {len(output_records)} records to {args.output}")
        print(f"[GITHUB-CVE] Status summary: {dict(summary)}")
        print(
            f"[GITHUB-CVE] API requests={client.requests}, "
            f"cache hits={client.cache_hits}, "
            f"remaining={client.rate_limit_remaining}"
        )
        return 0 if payload["complete"] else 3

    except (FileNotFoundError, ValueError, OSError, json.JSONDecodeError) as exc:
        print(f"ERROR: {exc}", file=sys.stderr)
        return 2
    except KeyboardInterrupt:
        print("ERROR: interrupted by user", file=sys.stderr)
        return 130


if __name__ == "__main__":
    sys.exit(main())
